toolmenu 0.25.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +312 -35
- package/dist/args.d.ts +14 -2
- package/dist/args.js +44 -1
- package/dist/auth-check.d.ts +27 -0
- package/dist/auth-check.js +177 -0
- package/dist/auth.d.ts +7 -0
- package/dist/auth.js +29 -1
- package/dist/auto.d.ts +8 -0
- package/dist/auto.js +9 -2
- package/dist/breakdown.js +14 -10
- package/dist/cli.js +171 -56
- package/dist/clients.d.ts +13 -2
- package/dist/clients.js +65 -11
- package/dist/config.js +6 -0
- package/dist/diff.d.ts +35 -1
- package/dist/diff.js +57 -12
- package/dist/failures.js +4 -3
- package/dist/help.d.ts +1 -1
- package/dist/help.js +45 -17
- package/dist/history.js +5 -0
- package/dist/html.d.ts +29 -0
- package/dist/html.js +360 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3 -2
- package/dist/init.d.ts +14 -0
- package/dist/init.js +20 -6
- package/dist/junit.d.ts +17 -0
- package/dist/junit.js +87 -0
- package/dist/mcp-env.d.ts +6 -0
- package/dist/mcp-env.js +6 -0
- package/dist/mcp.d.ts +10 -0
- package/dist/mcp.js +86 -15
- package/dist/menu.d.ts +6 -0
- package/dist/menu.js +12 -1
- package/dist/options.d.ts +13 -0
- package/dist/options.js +4 -0
- package/dist/report.d.ts +14 -0
- package/dist/report.js +57 -51
- package/dist/review.d.ts +48 -0
- package/dist/review.js +212 -0
- package/dist/rule-docs.json +49 -4
- package/dist/rules/clients.d.ts +11 -0
- package/dist/rules/clients.js +65 -2
- package/dist/rules/determinism.d.ts +3 -0
- package/dist/rules/determinism.js +44 -7
- package/dist/rules/ids.js +5 -1
- package/dist/rules/index.js +5 -2
- package/dist/rules/output.d.ts +21 -0
- package/dist/rules/output.js +49 -0
- package/dist/rules/search.js +1 -1
- package/dist/rules/spec.d.ts +19 -0
- package/dist/rules/spec.js +86 -1
- package/dist/sanitize.d.ts +42 -0
- package/dist/sanitize.js +142 -0
- package/dist/sarif.d.ts +2 -0
- package/dist/sarif.js +2 -2
- package/dist/scan.d.ts +9 -2
- package/dist/scan.js +46 -24
- package/dist/session.d.ts +17 -0
- package/dist/session.js +119 -13
- package/dist/snapshot.js +2 -1
- package/package.json +5 -3
package/README.md
CHANGED
|
@@ -5,24 +5,34 @@
|
|
|
5
5
|
<h1 align="center">toolmenu</h1>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
8
|
-
<b>
|
|
8
|
+
<b>Tests your MCP server the way agents and clients meet it: the menu they read, the calls they make, the results and sign-in they get back, and every release after. Problems caught before your users hit them.</b>
|
|
9
9
|
</p>
|
|
10
10
|
|
|
11
11
|
<p align="center">
|
|
12
12
|
|
|
13
13
|
[](https://www.npmjs.com/package/toolmenu)
|
|
14
|
-
[](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml)
|
|
15
|
-
[](https://github.com/niksa90/toolmenu/blob/main/LICENSE)
|
|
16
|
-
[](https://modelcontextprotocol.io)
|
|
17
|
-
[](https://github.com/niksa90/toolmenu)
|
|
18
|
-
[](https://glama.ai/mcp/servers/niksa90/toolmenu)
|
|
14
|
+
[](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml)
|
|
15
|
+
[](https://github.com/niksa90/toolmenu/blob/main/LICENSE)
|
|
16
|
+
[](https://modelcontextprotocol.io)
|
|
17
|
+
[](https://github.com/niksa90/toolmenu)
|
|
18
|
+
[](https://glama.ai/mcp/servers/niksa90/toolmenu)
|
|
19
|
+
[](https://mcprush.com/to/toolmenu-mcp?ref=badge)
|
|
20
|
+
[](https://mcpservers.org/servers/niksa90/toolmenu)
|
|
19
21
|
|
|
20
22
|
</p>
|
|
21
23
|
|
|
22
24
|
Your MCP server's tool list is the menu every agent reads before it does anything.
|
|
23
|
-
toolmenu
|
|
24
|
-
|
|
25
|
-
the
|
|
25
|
+
toolmenu starts there and follows the agent in. It **connects** as a client does (stdio
|
|
26
|
+
or HTTP, every protocol version from 2024-11-05 to 2026-07-28), **reads** the menu,
|
|
27
|
+
**calls** the tools (read-only by default; every tool, writes included, in a sealed
|
|
28
|
+
sandbox with `--call-everything`), **checks** each result against the spec and the tool's
|
|
29
|
+
own `outputSchema`, **walks** the OAuth discovery a client follows before it logs in, and
|
|
30
|
+
**compares** the menu across steps, connections, processes, clients and releases.
|
|
31
|
+
|
|
32
|
+
It isn't only a linter: a linter grades the text of one snapshot, and much of what
|
|
33
|
+
breaks agents only shows when you call the server, reconnect, or ship the next version.
|
|
34
|
+
No LLM is involved, so the same server gives the same answer on every run and it fits
|
|
35
|
+
in CI. The same checks run [every week on ~3,200 public servers](#tested-on-real-servers).
|
|
26
36
|
|
|
27
37
|
```sh
|
|
28
38
|
npx toolmenu snapshot -- node dist/server.js # or: npx toolmenu snapshot https://example.com/mcp
|
|
@@ -31,28 +41,71 @@ npx toolmenu init -- node dist/server.js # CI: a baseline, a workflow,
|
|
|
31
41
|
|
|
32
42
|

|
|
33
43
|
|
|
44
|
+
> **Status: 0.27.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md),
|
|
45
|
+
> every rule in [docs/RULES.md](docs/RULES.md), protocol versions in [docs/PROTOCOLS.md](docs/PROTOCOLS.md)
|
|
46
|
+
> (written by a test that runs each one), changes in [CHANGELOG.md](CHANGELOG.md).
|
|
47
|
+
|
|
48
|
+
**Contents:** [A finding](#what-a-finding-looks-like) ·
|
|
49
|
+
[What it catches](#what-it-catches) · [Why "when it changes" matters](#why-when-it-changes-matters) ·
|
|
50
|
+
[Tested on real servers](#tested-on-real-servers) · [Start here](#start-here) ·
|
|
51
|
+
[CI](#in-ci-one-pr-comment) · [`session`](#session-the-menu-changing-while-the-agent-works) ·
|
|
52
|
+
[`diff`](#diff-compare-releases) · [`review`](#accepting-a-new-baseline-review) · [`snapshot` rules](#snapshot-rules) ·
|
|
53
|
+
[What it can't catch](#what-toolmenu-cant-catch) · [Options](#options)
|
|
54
|
+
|
|
55
|
+
## What a finding looks like
|
|
56
|
+
|
|
57
|
+
A server whose `unlock_toolset` tool adds two tools to the menu mid-conversation, run
|
|
58
|
+
with `toolmenu session`:
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
step 3: call unlock_toolset {"toolset":"audits"} · answered · menu changed · +2 tools · list_changed received
|
|
62
|
+
ERROR session/mid-insert
|
|
63
|
+
+2 tools inserted at position 1 (list_team_audits, get_team_audit). Invalidates the cached prompt:
|
|
64
|
+
the tool list and the conversation after it are processed again.
|
|
65
|
+
cost: the whole tool list (this server's part: ~406 tokens, estimate) and the conversation after it
|
|
66
|
+
→ Next: Add new tools at the end of the list, not in the middle; better, list them from the start.
|
|
67
|
+
WARN session/unlock-coverage · unsure
|
|
68
|
+
unlock_toolset looks like it unlocks tools, and the run got through 1 of its 2 toolset values. […]
|
|
69
|
+
not unlocked: "reports"
|
|
70
|
+
→ Next: Unlock every value: session --init puts each one in the starter.
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Every finding has the same parts: a **level** (`ERROR`, `WARN`, `INFO`; `--fail-on`
|
|
74
|
+
decides which fail the run), a **rule id** you can look up (`toolmenu explain
|
|
75
|
+
session/mid-insert`) or turn off in the config, **where** it happened (the step, the
|
|
76
|
+
tool, the parameter path), **what toolmenu saw**, and a **`→ Next:`** step.
|
|
77
|
+
|
|
78
|
+
How much to trust a finding depends on what it rests on, and the rule id's prefix says which:
|
|
79
|
+
|
|
80
|
+
| Rule family | What it rests on |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `spec/`, `apps/`, `lists/schema` | The protocol: the official JSON schema for the server's version (pinned in `schemas/`), a MUST/SHOULD in the spec, or the MCP Apps extension |
|
|
83
|
+
| `diff/` | Two menus compared. A **breaking** change is one where a call that passed validation before now fails it |
|
|
84
|
+
| `session/`, `menu/`, `lists/variance`, `auth/`, `catalog/` | Behaviour toolmenu observed: the menu changing after a step, between two calls, between two processes or connections, or served to a bad credential. The finding names the step or the call |
|
|
85
|
+
| `naming/`, `ids/`, `description/`, `write/`, `search/` | How agents read the menu: names, words, descriptions. Tuned on real servers, but heuristics |
|
|
86
|
+
|
|
87
|
+
A finding marked **`· unsure`**, in any family, rests on names and words rather than on
|
|
88
|
+
something toolmenu saw happen (above: "looks like it unlocks tools"). It says what was
|
|
89
|
+
seen, not what it means. No LLM is involved anywhere: the same server gives the same
|
|
90
|
+
findings on every run.
|
|
91
|
+
|
|
34
92
|
## What it catches
|
|
35
93
|
|
|
36
94
|
| | |
|
|
37
95
|
|---|---|
|
|
38
96
|
| **A menu that changes mid-session** | `session` calls your tools and watches the list. A tool added mid-conversation can rewrite the whole cached prompt: one unlock cost 40–57K tokens of cache in my logs. |
|
|
97
|
+
| **Every tool, called** | `session --auto` calls the tools marked read-only; `--call-everything` calls all of them, writes included, with made-up input in a container with no network and no credentials. A call that crashes, hangs or breaks the protocol is reported; a readable tool error is what the spec asks for. |
|
|
98
|
+
| **Results clients reject** | Results are checked against the tool's `outputSchema` (a MUST since 2025-06-18) and, on 2026-07-28, for `resultType`, the way the official SDK client checks them before your agent ever sees the answer. |
|
|
99
|
+
| **Sign-in, before logging in** | `auth check` walks OAuth discovery as a spec-following client does: the 401, protected resource metadata (RFC 9728), authorization server metadata (RFC 8414 or OpenID Connect), PKCE and registration. |
|
|
39
100
|
| **Breaking releases** | `diff` compares two menus: removed tools, new required parameters, narrowed types, token growth. It also says whether your version bump is big enough. |
|
|
40
101
|
| **The 2026-07-28 protocol** | The tool set must not vary per connection, and results need `resultType`. Elicitation, multi round-trip input and `server/discover` are checked too, as are the 2025 handshakes. |
|
|
41
102
|
| **MCP Apps, Tasks, Server Cards** | It reads `ui://` resources as a host does and checks who may see each tool. It catches tasks sent to clients that can't poll them, and Server Cards that contradict the live server. |
|
|
42
|
-
| **Your client's view** | `--client
|
|
103
|
+
| **Your client's view** | `--client` with Claude Code, the Claude API, OpenAI, Codex, VS Code (Copilot Chat), Copilot cloud agent and code review, Gemini CLI, the Gemini API or Antigravity shows what each loads, cuts, can find, refuses, rewrites or leaves out, from each one's own docs. |
|
|
43
104
|
| **What agents misread** | It flags vague `id`s, IDs no tool returns, instructions buried late in descriptions, and writes without annotations or marked read-only. |
|
|
105
|
+
| **Words routing to the wrong tool** | `routes.yml` pins which tool each keyword must (and must not) reach, the way a keyword or BM25 tool search sees the menu. A description edit that lets `get_audit_trail` win "audit" fails the check, with no model and no eval budget. [More](#snapshot-rules) |
|
|
44
106
|
| **Menus that drift** | It catches defaults built from the clock or a hash order, menus that differ per process, per connection or offline, and listings that depend on who asks. |
|
|
45
107
|
|
|
46
|
-
|
|
47
|
-
It runs on the 22-server [bench](bench/README.md) and on hosted servers (GitHub, Atlassian, Sentry,
|
|
48
|
-
Notion, Supabase, Vercel…). What it has found on real servers is in
|
|
49
|
-
[docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
50
|
-
|
|
51
|
-
> **Status: 0.25.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md),
|
|
52
|
-
> every rule in [docs/RULES.md](docs/RULES.md), changes in [CHANGELOG.md](CHANGELOG.md).
|
|
53
|
-
|
|
54
|
-
<details>
|
|
55
|
-
<summary><b>Why "when it changes" matters</b></summary>
|
|
108
|
+
## Why "when it changes" matters
|
|
56
109
|
|
|
57
110
|
The MCP linters I found grade one snapshot of the menu: tool names, descriptions,
|
|
58
111
|
schemas. toolmenu also checks **when** the menu changes:
|
|
@@ -74,10 +127,47 @@ schemas. toolmenu also checks **when** the menu changes:
|
|
|
74
127
|
old versions with today's dependencies, so it records what they serve now
|
|
75
128
|
(FINDINGS F4).
|
|
76
129
|
|
|
77
|
-
|
|
78
|
-
reports no false findings ([bench/](bench/README.md)).
|
|
130
|
+
## Tested on real servers
|
|
79
131
|
|
|
80
|
-
|
|
132
|
+
Every rule change is run against real MCP servers before it ships, and every finding
|
|
133
|
+
that changes on one of them is read by hand; a change has to say why in the
|
|
134
|
+
[changelog](CHANGELOG.md). The corpus is pinned to exact versions, so a rerun checks the
|
|
135
|
+
same code ([bench/README.md](bench/README.md) has the commands):
|
|
136
|
+
|
|
137
|
+
| Corpus | Servers | Checked with |
|
|
138
|
+
|---|---|---|
|
|
139
|
+
| The reference server, `@modelcontextprotocol/server-everything` 2.1.0 (2026-07-28) | 1 | `snapshot --lists`, `session --auto`, and a scenario that answers its elicitation demos. **No false findings**: a change that makes one appear is wrong until shown otherwise |
|
|
140
|
+
| Recent stdio servers from the MCP registry that start without credentials | 86 | `snapshot` |
|
|
141
|
+
| The servers across the stdio lists (registry, popular npm, PyPI) that answer the 2026-07-28 handshake | 39 | `session --auto` |
|
|
142
|
+
| Named servers run in a container: GitHub, Atlassian, Sentry, Notion, Supabase, Playwright, MongoDB… ([`bench/servers.txt`](bench/servers.txt)) | 22 | `snapshot`, `session --init`, `session` |
|
|
143
|
+
| Hosted (Streamable HTTP) servers from the registry that need no headers, sampled with a fixed seed | 200 | `snapshot`, read-only `session --auto --max-calls 10` |
|
|
144
|
+
| Hosted servers behind OAuth: Sentry, GitHub, Atlassian (F14) | 3 | `auth login`, then `snapshot` |
|
|
145
|
+
| **Weekly:** Docker's MCP catalog, every local server at the digest the catalog pins, with a dummy value for each setting it declares (F20) | 237 | `snapshot`, `session --auto`, and the sealed pass: 4,117 of the 4,237 tools listed called |
|
|
146
|
+
| **Weekly:** stdio servers from the registry, npm and PyPI (F17, F18); each week rechecks up to 300 new releases and newly listed servers | 1,090 at baseline | `snapshot`, `session --auto`, and a sealed `--call-everything` pass: 15,550 of 16,518 tools called, every one of the rest accounted for |
|
|
147
|
+
| The servers big vendors publish to run locally: GitHub, AWS Labs, Microsoft, PayPal, Stripe, Atlassian, Notion, HashiCorp, Grafana… as npm, PyPI and container images, with dummy credentials (F19) | 98 | `snapshot`, `session --auto`, and the sealed pass: 2,213 of the 2,292 tools listed called |
|
|
148
|
+
| **Weekly:** hosted servers, 109 vendor endpoints (GitHub, Google, Microsoft, Atlassian, Stripe…) and a seeded 2,000 of the registry's | 2,109 | `snapshot`, read-only `session --auto`, and `auth check` on each that answers 401 |
|
|
149
|
+
|
|
150
|
+
What those runs found is in [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md),
|
|
151
|
+
including what toolmenu itself got wrong and how it was fixed. A few:
|
|
152
|
+
|
|
153
|
+
- **mcp-atlassian serves a different menu every time it starts**: a default built from a
|
|
154
|
+
Python set, in a new order each process, so every restart misses the cache (F12).
|
|
155
|
+
- **Atlassian Rovo served 4 tools to any `Authorization` header**, valid or not, so a
|
|
156
|
+
client whose token expired gets fewer tools instead of a reason to log in again (`auth/invalid-accepted`).
|
|
157
|
+
- **The official filesystem server has reported `0.2.0` for 19 releases**, so a
|
|
158
|
+
server's self-reported version can't be used for semver checks (F5).
|
|
159
|
+
- **Notion's server ships the same 9 unused `$defs` in all 24 tools**: 72% of its menu.
|
|
160
|
+
- **12 of 1,090 stdio servers fail in the official SDK client today** (8 lose every tool,
|
|
161
|
+
4 lose answers), and 22 of 2,000 hosted ones send a 2026-07-28 tool list that client
|
|
162
|
+
rejects (F18).
|
|
163
|
+
- **PayPal's server builds a new menu on every call**: `list_transactions` defaults its dates to
|
|
164
|
+
the moment the list is built, so no cached prompt matches the next one (F19).
|
|
165
|
+
- **Microsoft Sentinel's sign-in can't pass a client that follows MCP 2025-11-25 to the
|
|
166
|
+
letter**: Entra ID supports PKCE but its OpenID Connect metadata doesn't say so, and the
|
|
167
|
+
spec tells clients to refuse (F18).
|
|
168
|
+
|
|
169
|
+
The weekly run compares each week with the last: menus that changed, servers that fixed
|
|
170
|
+
or gained a finding, and the trend ([bench/results/weekly](bench/results/weekly)).
|
|
81
171
|
|
|
82
172
|
## Start here
|
|
83
173
|
|
|
@@ -110,6 +200,7 @@ By hand:
|
|
|
110
200
|
```sh
|
|
111
201
|
npx toolmenu snapshot -- node dist/server.js # the menu, its findings, menu.json
|
|
112
202
|
npx toolmenu diff menu.json new-menu.json # what changed between two releases
|
|
203
|
+
npx toolmenu review menu.json new-menu.json # should the new menu replace the baseline?
|
|
113
204
|
npx toolmenu session --auto -- node dist/server.js # call read-only tools, watch the menu
|
|
114
205
|
```
|
|
115
206
|
|
|
@@ -123,6 +214,12 @@ npx toolmenu auth login https://mcp.example.com/mcp
|
|
|
123
214
|
npx toolmenu snapshot https://mcp.example.com/mcp
|
|
124
215
|
```
|
|
125
216
|
|
|
217
|
+
Before logging in, `toolmenu auth check https://mcp.example.com/mcp` walks the server's
|
|
218
|
+
OAuth discovery the way a client that follows the spec does: the 401 and its
|
|
219
|
+
`WWW-Authenticate`, the protected resource metadata (and whether its `resource` is the
|
|
220
|
+
server's URL), the authorization server's metadata, PKCE and registration. No login, no
|
|
221
|
+
tokens: only the requests a client makes first.
|
|
222
|
+
|
|
126
223
|
Logins are stored per server in `~/.config/toolmenu/auth` (readable only by you);
|
|
127
224
|
`auth list` and `auth logout <url>` manage them. A server that doesn't allow
|
|
128
225
|
dynamic registration needs a pre-registered app: `--client-id` (and
|
|
@@ -133,22 +230,22 @@ CI, where nobody can sign in, pass a token as a header.
|
|
|
133
230
|
Both protocol generations are supported: 2026-07-28 (`server/discover`, stateless)
|
|
134
231
|
and the 2025 `initialize` handshake, through the official TypeScript SDK.
|
|
135
232
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
its default level and how to change it (every rule is in [docs/RULES.md](docs/RULES.md)).
|
|
140
|
-
Every command has its own help: `toolmenu history --help`.
|
|
233
|
+
How to read a finding is [above](#what-a-finding-looks-like). `toolmenu explain <rule>`
|
|
234
|
+
says what a rule checks, its default level and how to change it (every rule is in
|
|
235
|
+
[docs/RULES.md](docs/RULES.md)). Every command has its own help: `toolmenu history --help`.
|
|
141
236
|
|
|
142
237
|
Behind a proxy (`HTTPS_PROXY`), stdio servers get the proxy and CA settings
|
|
143
238
|
(`HTTPS_PROXY`, `NO_PROXY`, `NODE_EXTRA_CA_CERTS`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`…)
|
|
144
239
|
along with the minimal environment, and `NODE_USE_ENV_PROXY=1` so a Node server's
|
|
145
240
|
`fetch` uses the proxy. Without them a server that calls an API can't reach it, and
|
|
146
|
-
fails as if it were broken.
|
|
241
|
+
fails as if it were broken. A proxy URL with a password in it
|
|
242
|
+
(`https://user:pass@proxy:8080`) reaches the server that way too: give a server you
|
|
243
|
+
don't trust a proxy without one, or `--env HTTPS_PROXY=` to leave it out.
|
|
147
244
|
|
|
148
245
|
### Every server you use: `scan`
|
|
149
246
|
|
|
150
247
|
```sh
|
|
151
|
-
npx toolmenu scan ~/Library/Application\ Support/Claude/claude_desktop_config.json
|
|
248
|
+
npx toolmenu scan ~/Library/Application\ Support/Claude/claude_desktop_config.json --allow-env LINEAR_API_KEY
|
|
152
249
|
npx toolmenu scan .mcp.json --format markdown
|
|
153
250
|
```
|
|
154
251
|
|
|
@@ -165,6 +262,13 @@ findings, like this:
|
|
|
165
262
|
linear – – not checked: LINEAR_API_KEY is not set (the config refers to it)
|
|
166
263
|
```
|
|
167
264
|
|
|
265
|
+
`scan` starts every command the list names, so scan only lists you'd run yourself (not
|
|
266
|
+
a `.mcp.json` from a repository you haven't read). A config's `${VAR}` is filled only
|
|
267
|
+
from the variables you name with `--allow-env` (comma-separated, or `'*'` for all): the
|
|
268
|
+
config decides where each value goes, a URL or a header included, so one from someone
|
|
269
|
+
else could otherwise send your environment to a server of theirs. An entry that uses
|
|
270
|
+
a variable you didn't allow isn't checked, and says which `--allow-env` it needs.
|
|
271
|
+
|
|
168
272
|
`${VAR}`, `${VAR:-default}` and `${env:VAR}` in the config are filled from the
|
|
169
273
|
environment. A server whose variable isn't set isn't started with an empty
|
|
170
274
|
credential: it's a row that says so. A server that couldn't be started or read
|
|
@@ -200,6 +304,33 @@ Relative paths resolve from the folder `toolmenu mcp` runs in, or a call's `cwd`
|
|
|
200
304
|
clients ask before each tool call unless you allow it. toolmenu's own menu passes its
|
|
201
305
|
checks with no findings, on both protocols and in a session (the test suite checks).
|
|
202
306
|
|
|
307
|
+
## As a library
|
|
308
|
+
|
|
309
|
+
Everything the CLI does is exported from the `toolmenu` package, with types, for test
|
|
310
|
+
suites and scripts of your own:
|
|
311
|
+
|
|
312
|
+
```js
|
|
313
|
+
import { snapshot, diffMenus, loadMenu, session, loadScenario } from 'toolmenu';
|
|
314
|
+
|
|
315
|
+
// The menu and its findings, as `toolmenu snapshot` gives them.
|
|
316
|
+
const { menu, findings } = await snapshot({ kind: 'stdio', command: 'node', args: ['dist/server.js'] });
|
|
317
|
+
// or { kind: 'http', url: 'https://example.com/mcp', headers: { Authorization: `Bearer ${token}` } }
|
|
318
|
+
|
|
319
|
+
// What changed since the committed baseline, and the version bump it needs.
|
|
320
|
+
const diff = diffMenus(await loadMenu('menu.json'), menu);
|
|
321
|
+
console.log(diff.suggestedBump, diff.tokens);
|
|
322
|
+
if (diff.findings.some((f) => f.severity === 'error')) process.exitCode = 1;
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
`snapshot`, `diffMenus`, `session` and `history` take the same options as their
|
|
326
|
+
commands (`SnapshotOptions`, `DiffOptions`, `SessionOptions`, `HistoryOptions`) and
|
|
327
|
+
return the findings the reports print: each `Finding` has its `rule`, `severity`,
|
|
328
|
+
`message`, `tool` and the `fix` shown as `→ Next:`. Lower down, `compareMenus(before,
|
|
329
|
+
after)` returns each tool's change (`ToolChange[]`) and `runRules` runs the menu rules
|
|
330
|
+
on a menu you built. The `menu.json` format is the `Menu` type (`toolmenu: 1`);
|
|
331
|
+
a later format will bump that number. Library functions don't print or exit: that's
|
|
332
|
+
the CLI's job.
|
|
333
|
+
|
|
203
334
|
## In CI: one PR comment
|
|
204
335
|
|
|
205
336
|
```yaml
|
|
@@ -216,7 +347,7 @@ jobs:
|
|
|
216
347
|
- uses: actions/setup-node@v7
|
|
217
348
|
with: { node-version: 22 }
|
|
218
349
|
- run: npm ci && npm run build
|
|
219
|
-
- uses: niksa90/toolmenu@v0.
|
|
350
|
+
- uses: niksa90/toolmenu@v0.27.0
|
|
220
351
|
with:
|
|
221
352
|
command: node dist/server.js
|
|
222
353
|
baseline: menu.json # your committed snapshot
|
|
@@ -225,7 +356,8 @@ jobs:
|
|
|
225
356
|
|
|
226
357
|
It snapshots the PR's build, diffs it against the baseline as it is on the PR's base
|
|
227
358
|
branch (or at the previous tag, on a tag push), runs the scenario, and
|
|
228
|
-
posts **one comment that updates on every push**:
|
|
359
|
+
posts **one comment that updates on every push**: first the verdict (can this menu
|
|
360
|
+
replace the baseline as it is?), then breaking changes, the token change
|
|
229
361
|
("this PR adds ~1,240 tokens to every conversation"), the suggested version bump,
|
|
230
362
|
and anything `session` caught. The same report goes to the job summary.
|
|
231
363
|
|
|
@@ -240,6 +372,7 @@ and anything `session` caught. The same report goes to the job summary.
|
|
|
240
372
|
| `baseline-from` | `snapshot` | `session` diffs every tool the scenario saw (the union menu), for servers whose tools appear after an unlock |
|
|
241
373
|
| `release` | `auto` | Release versions for the bump check. `auto` reads `package.json`, `pyproject.toml` or `Cargo.toml` on the base branch and the head, or compares the tag with the previous one on a tag push. Set `"1.4.0..1.5.0"` yourself, or `off` |
|
|
242
374
|
| `fail-on` | `error` | Fail the job on findings at or above this level |
|
|
375
|
+
| `gate` | `findings` | `review` passes or fails the job on the [review verdict](#accepting-a-new-baseline-review) instead of `diff`'s findings: a breaking change the release bump allows passes, new findings at `fail-on` fail ([more](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md#the-verdict-and-gate-review)) |
|
|
243
376
|
| `comment` | `true` | Post and update the PR comment |
|
|
244
377
|
| `sarif` | | A folder to also write the findings to as SARIF, for code scanning (below) |
|
|
245
378
|
| `version` | the Action's own | toolmenu version from npm: by default the one matching the Action tag you pinned (`latest` for the newest). The Action installs toolmenu from npm, so it needs a published version |
|
|
@@ -264,7 +397,7 @@ permissions:
|
|
|
264
397
|
security-events: write # to upload SARIF
|
|
265
398
|
steps:
|
|
266
399
|
# … checkout, build, as above
|
|
267
|
-
- uses: niksa90/toolmenu@v0.
|
|
400
|
+
- uses: niksa90/toolmenu@v0.27.0
|
|
268
401
|
with:
|
|
269
402
|
command: node dist/server.js
|
|
270
403
|
sarif: toolmenu-sarif
|
|
@@ -278,6 +411,50 @@ Outside the Action: `toolmenu snapshot --sarif toolmenu.sarif -- node dist/serve
|
|
|
278
411
|
writes the file beside the usual report (`diff` and `session` take `--sarif` too), or
|
|
279
412
|
`--format sarif` prints it instead.
|
|
280
413
|
|
|
414
|
+
### Other CI systems: JUnit test reports
|
|
415
|
+
|
|
416
|
+
GitLab, Jenkins, Azure DevOps, CircleCI and Buildkite show JUnit XML as test results.
|
|
417
|
+
`--junit <path>` writes the findings of `snapshot`, `diff` or `session` as JUnit beside
|
|
418
|
+
the usual report (`--format junit` prints it instead). New in 0.25.2. Each finding is a test case named
|
|
419
|
+
after its tool, in a class named after its rule (`toolmenu.diff.diff/removed`), so the
|
|
420
|
+
CI system tracks it across runs. Findings at or above `--fail-on` are failures, the rest
|
|
421
|
+
pass with their text, so the test report agrees with the exit code. A clean run is one
|
|
422
|
+
passing case.
|
|
423
|
+
|
|
424
|
+
```yaml
|
|
425
|
+
# .gitlab-ci.yml
|
|
426
|
+
toolmenu:
|
|
427
|
+
image: node:22
|
|
428
|
+
script:
|
|
429
|
+
- npm ci && npm run build
|
|
430
|
+
# Exit 1 means findings: keep going so both reports are written, then fail.
|
|
431
|
+
- npx toolmenu snapshot --out new-menu.json --junit snapshot.xml -- node dist/server.js || code=$?
|
|
432
|
+
- npx toolmenu diff menu.json new-menu.json --junit diff.xml || code=$?
|
|
433
|
+
- exit ${code:-0}
|
|
434
|
+
artifacts:
|
|
435
|
+
when: always
|
|
436
|
+
reports:
|
|
437
|
+
junit: [snapshot.xml, diff.xml]
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
### A report to keep: HTML
|
|
441
|
+
|
|
442
|
+
`--html <path>` writes the run of `snapshot`, `diff` or `session` as one HTML page
|
|
443
|
+
beside the usual report: the numbers that matter at the top (errors, warnings, tools,
|
|
444
|
+
tokens; for `diff`, breaking changes and the suggested bump), the menu as a table in
|
|
445
|
+
server order with each tool's share of the tokens, and the findings with their next
|
|
446
|
+
steps. `diff` shows each tool before and after, new and removed tools marked. `session`
|
|
447
|
+
has a row per step with the menu after it, tools inserted, appended or edited marked,
|
|
448
|
+
and links from a step to its findings.
|
|
449
|
+
|
|
450
|
+
The page has no script and loads nothing, so it opens offline: keep it as a CI artifact
|
|
451
|
+
(GitLab `artifacts: paths`, `actions/upload-artifact`) or attach it to an issue for a
|
|
452
|
+
server's maintainer. It follows the viewer's light or dark setting.
|
|
453
|
+
|
|
454
|
+
```sh
|
|
455
|
+
npx toolmenu diff menu.json new-menu.json --html diff.html
|
|
456
|
+
```
|
|
457
|
+
|
|
281
458
|
### A badge for your README
|
|
282
459
|
|
|
283
460
|
```sh
|
|
@@ -299,6 +476,22 @@ PRs without secrets are skipped, and a skipped check still passes; the `skipped`
|
|
|
299
476
|
`session` runs a scripted session (the scenario *is* the agent), lists the menu
|
|
300
477
|
after every step, and reports each change with the step that caused it.
|
|
301
478
|
|
|
479
|
+
```
|
|
480
|
+
step the scenario (the agent) the menu the client sees toolmenu
|
|
481
|
+
──── ─────────────────────────────── ─────────────────────────────── ─────────────────────────────
|
|
482
|
+
1 list search_forms, get_form, … baseline: 5 tools
|
|
483
|
+
2 call search_forms (unchanged) no change
|
|
484
|
+
3 call unlock_toolset audits search_forms, list_team_audits, +2 tools at position 1:
|
|
485
|
+
get_team_audit, get_form, … session/mid-insert (error)
|
|
486
|
+
fresh process: not there →
|
|
487
|
+
session/side-effect (warn)
|
|
488
|
+
```
|
|
489
|
+
|
|
490
|
+
Each step's menu is compared with the one before it, so a change is pinned to the call
|
|
491
|
+
that caused it, with where in the list it starts and what it costs the prompt cache.
|
|
492
|
+
toolmenu then starts a second process (or opens a second connection) to see whether
|
|
493
|
+
the change is global, or only this session's, which 2026-07-28 rules out.
|
|
494
|
+
|
|
302
495
|
```yaml
|
|
303
496
|
# scenario.yml
|
|
304
497
|
allow_writes: false # refuse to call tools not marked readOnlyHint
|
|
@@ -348,6 +541,7 @@ serves, and 23 issue and pull-request tools that only the dynamic mode has.
|
|
|
348
541
|
| `session/error-not-flagged` | warn | A tool answered with an HTTP error from the API behind it (`{"status":401,"object":"error",…}`) as a plain result, not marked `isError`, so a client that reads `isError` takes the call for a success (Notion's server). An auth one also counts toward `session/untested`. Tools that take a URL are left out: their answer is the page's (heuristic) |
|
|
349
542
|
| `session/server-unresponsive` | error | The server stopped answering after a step: even `tools/list` timed out. The run stops there with this one finding, instead of every later step waiting out its timeout (a server that polls a device-code sign-in inside a tool call) |
|
|
350
543
|
| `session/handle-roundtrip` | warn | With `--create-handles`: the tool that creates a session, basket or cart was called, and the tools that take it back refused the handle it had just returned (info when its result had no handle to find) |
|
|
544
|
+
| `session/output-schema` | error | A tool that declares an `outputSchema` answered with no `structuredContent`, or one that doesn't match it: from 2025-06-18 a **MUST**, and the official SDK client raises an error instead of the answer |
|
|
351
545
|
| `session/rate-limited` | error | The server kept refusing requests for being too many (a 429) after toolmenu waited about a minute, or a tool not marked `readOnlyHint` said it was rate-limited (it isn't called again). The run stops there, with the steps it didn't check; give the run its own server instance or raise the limit for it |
|
|
352
546
|
| `session/scope-unchecked` | info | A second server process or connection couldn't start mid-session (a server that holds a file or a port), so whether a change was global wasn't checked. Said once for the run |
|
|
353
547
|
|
|
@@ -380,6 +574,14 @@ with why each tool was left out, not a clean pass. On the 22 servers in `bench/`
|
|
|
380
574
|
**`session` calls tools for real.** Without `allow_writes: true` it refuses any tool
|
|
381
575
|
not marked `readOnlyHint: true`. `--plan` prints the steps without connecting.
|
|
382
576
|
|
|
577
|
+
**`--call-everything`** (with `--auto`, stdio only) calls **every** tool once, writes and
|
|
578
|
+
open-world ones included, with a made-up value (`toolmenu-test`, `1`, a one-item list)
|
|
579
|
+
where the schema gives none. It's for a server in a sandbox with nothing to lose: a
|
|
580
|
+
container with no credentials and no network, as `bench/scripts/batch-full.sh` runs
|
|
581
|
+
them with `SEALED=1`. Every tool's answer is then checked: results against their own
|
|
582
|
+
`outputSchema`, errors flagged as errors, calls that hang. It's refused over HTTP,
|
|
583
|
+
because a hosted server's tools act on someone's real account.
|
|
584
|
+
|
|
383
585
|
## `diff`: compare releases
|
|
384
586
|
|
|
385
587
|
```sh
|
|
@@ -458,6 +660,40 @@ with `--server-version-is-release`. `diff` also enforces an optional `tokenBudge
|
|
|
458
660
|
(`diff/token-budget`). Order changes are only a notice here: between releases, prompt caches rebuild. Order
|
|
459
661
|
matters *within* a session.
|
|
460
662
|
|
|
663
|
+
### Accepting a new baseline: `review`
|
|
664
|
+
|
|
665
|
+
`diff` says what changed. Before you commit a refreshed `menu.json`, `review` answers
|
|
666
|
+
the question that follows: should it replace the baseline?
|
|
667
|
+
|
|
668
|
+
```sh
|
|
669
|
+
npx toolmenu snapshot --out new-menu.json -- node dist/server.js
|
|
670
|
+
npx toolmenu review --release 1.4.0..1.5.0 menu.json new-menu.json
|
|
671
|
+
```
|
|
672
|
+
|
|
673
|
+
```
|
|
674
|
+
toolmenu review new-menu.json as the new menu.json
|
|
675
|
+
verdict ✗ don't accept as is: 1 breaking change, and the release's minor bump is too small (release 2.0.0).
|
|
676
|
+
changes 1 breaking (1 tool) · 0 minor · 3 notice (13 tools)
|
|
677
|
+
findings 0 new · 0 gone · 1 changed · 1 unchanged
|
|
678
|
+
```
|
|
679
|
+
|
|
680
|
+
It runs `diff`, and runs the menu rules on both files, so it also says which findings
|
|
681
|
+
the new menu **adds**, which it **clears**, and which **changed** (a summary that now
|
|
682
|
+
names another tool counts as worse). The verdict is one of:
|
|
683
|
+
|
|
684
|
+
| Verdict | Exit | When |
|
|
685
|
+
|---|---|---|
|
|
686
|
+
| `unchanged` | 0 | The same menu and the same findings |
|
|
687
|
+
| `safe` | 0 | Nothing breaks callers, and no finding at or above `--fail-on` is new or worse |
|
|
688
|
+
| `breaking-released` | 0 | Breaking changes, and `--release` shows a bump that allows them |
|
|
689
|
+
| `breaking` | 1 | Breaking changes, and no release that allows them: none given, too small, or not checkable (calendar versions) |
|
|
690
|
+
| `review` | 1 | Nothing breaks, but findings at or above `--fail-on` are new or worse, or `diff`'s own checks fail (`tokenBudget`) |
|
|
691
|
+
|
|
692
|
+
It reads only the two files, so rules that need the live server (a second process,
|
|
693
|
+
the raw `tools/list`) aren't part of it; `snapshot` runs those. `--format markdown`
|
|
694
|
+
gives the same as a PR comment, `--json` for scripts (`verdict`, `accept`, and the
|
|
695
|
+
findings by kind).
|
|
696
|
+
|
|
461
697
|
### Operations behind a search tool (`--catalog`)
|
|
462
698
|
|
|
463
699
|
Big servers increasingly keep most operations out of the menu: a small fixed menu, a
|
|
@@ -629,6 +865,26 @@ client doesn't document is left out, not guessed.
|
|
|
629
865
|
| `claude-code` | tool names; full definitions for tools marked `_meta["anthropic/alwaysLoad"]` | reads a description up to 4,096 characters | 16,384 through search, 4,096 up front | Claude API's |
|
|
630
866
|
| `claude-api-search` | nothing from tools you defer (keep the 3–5 used most loaded) | name, description, argument names and descriptions; 5 results by default | none documented | `^[a-zA-Z0-9_-]{1,128}$` |
|
|
631
867
|
| `openai` | each deferred function's name and description (its schema loads on demand); a deferred MCP server shows only its own name and description | how many results, and which fields, isn't documented, so `search/findable` doesn't run | none documented | `^[a-zA-Z0-9_-]+$`, up to 128 |
|
|
868
|
+
| `codex` | not documented | not documented | none documented | OpenAI's (it sends tools to the Responses API) |
|
|
869
|
+
| `antigravity` | each tool's name; the agent reads a tool's schema before calling it, unless the user marks it `"eager"` | not a search: the agent reads the schema file | none documented | not documented |
|
|
870
|
+
| `gemini-cli` | not documented | not documented | none documented | rewrites: characters outside letters, digits, `_ - . :` become `_`, names over 63 characters are cut in the middle |
|
|
871
|
+
| `gemini-api` | not documented | not documented | none documented | letters, digits, `_ - . :`, up to 128 |
|
|
872
|
+
|
|
873
|
+
Three more check what a client limits rather than what it loads:
|
|
874
|
+
|
|
875
|
+
| `--client` | What it checks |
|
|
876
|
+
|---|---|
|
|
877
|
+
| `vscode` | VS Code (GitHub Copilot Chat) sends at most 128 tools a request: a server over it alone is `menu/client-limit` |
|
|
878
|
+
| `copilot-cloud-agent` | uses MCP tools only, not prompts or resources (`menu/client-features`, with `--lists`) |
|
|
879
|
+
| `copilot-code-review` | the same, and only tools marked `readOnlyHint: true`: the rest are listed |
|
|
880
|
+
|
|
881
|
+
Read, but nothing in them toolmenu can check yet: Copilot CLI defers tools behind a
|
|
882
|
+
search at about 30 (it searches names, descriptions and parameters; how many results
|
|
883
|
+
isn't said); Copilot in JetBrains, Visual Studio, Eclipse and Xcode documents no tool
|
|
884
|
+
limit; JetBrains AI Assistant and Junie document none either (Junie CLI times out any
|
|
885
|
+
request after 60 s, startup and `tools/list` included). Not here, because their docs
|
|
886
|
+
don't state it: Cursor's tool limit (forum posts say 40), Windsurf's (100 in older docs,
|
|
887
|
+
not in the current page), ChatGPT's. A value comes in when the client documents it.
|
|
632
888
|
|
|
633
889
|
The snapshot adds a line such as `as Claude Code: up front ~524 tokens · on demand ~344
|
|
634
890
|
per tool a search loads (largest: update_contacts, ~2,093)`, and `--json` a `client`
|
|
@@ -656,7 +912,10 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
|
|
|
656
912
|
--no-write don't write the menu file
|
|
657
913
|
--routes <path> routes.yml
|
|
658
914
|
--config <path> config file (default: toolmenu.config.json, if present)
|
|
659
|
-
--format <fmt> text (default), json, github (annotations)
|
|
915
|
+
--format <fmt> text (default), json, github (annotations), markdown, sarif
|
|
916
|
+
(code scanning) or junit (CI test reports)
|
|
917
|
+
--junit <path> snapshot, diff, session: also write the findings as JUnit XML
|
|
918
|
+
--html <path> snapshot, diff, session: also write the report as one HTML page
|
|
660
919
|
--fail-on <level> error (default), warn or info
|
|
661
920
|
--header <k: v> HTTP header, repeatable
|
|
662
921
|
--env <K=V> env var for a stdio server, repeatable (it only gets a
|
|
@@ -669,18 +928,36 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
|
|
|
669
928
|
--lists <kinds> snapshot: also read and check prompts, resources (comma-separated);
|
|
670
929
|
saved in the menu file beside the tools
|
|
671
930
|
--client <name> snapshot: check the menu for one client, from its own docs:
|
|
672
|
-
claude-code, claude-api-search
|
|
931
|
+
claude-code, claude-api-search, openai, codex, vscode,
|
|
932
|
+
copilot-cloud-agent, copilot-code-review, gemini-cli,
|
|
933
|
+
gemini-api or antigravity
|
|
673
934
|
--as <name>:<cred> snapshot: list the menu as this identity too, repeatable:
|
|
674
935
|
"reader:Authorization: Bearer …" (HTTP) or "admin:ROLE=admin"
|
|
675
936
|
(stdio). The first is the menu saved; the others are compared
|
|
676
937
|
--offline-check snapshot (stdio): also list the menu with the network cut off
|
|
677
938
|
--concurrency <n> scan: servers checked at a time (default: 4)
|
|
678
939
|
--out-dir <dir> scan: write each server's menu file there, as <name>.json
|
|
940
|
+
--allow-env <names> scan: variables a config's ${VAR} may be filled from (default:
|
|
941
|
+
none; '*' for all)
|
|
679
942
|
```
|
|
680
943
|
|
|
681
944
|
Exit codes: `0` clean · `1` findings at or above `--fail-on` · `2` couldn't connect
|
|
682
945
|
(for `scan`: a server wasn't checked) or bad usage. `toolmenu <command> --help` lists that command's own options.
|
|
683
946
|
|
|
947
|
+
Credentials on the command line (`--header`, `--env`, `--client-secret`) can be read by
|
|
948
|
+
every user of the machine while toolmenu runs (`ps`). To keep them off it, put the
|
|
949
|
+
options in `TOOLMENU_ARGS`, a JSON array of toolmenu options and their values; they go
|
|
950
|
+
before the server command's `--`. A process's environment is readable only by its own
|
|
951
|
+
user (and root), so this keeps them from other users, not from other processes of
|
|
952
|
+
yours. jq builds the array whatever the token holds:
|
|
953
|
+
|
|
954
|
+
```sh
|
|
955
|
+
TOOLMENU_ARGS=$(jq -cn --arg t "$TOKEN" '["--header", "Authorization: Bearer " + $t]') npx toolmenu snapshot https://example.com/mcp
|
|
956
|
+
```
|
|
957
|
+
|
|
958
|
+
The GitHub Action passes its `headers` and `env` this way (to toolmenu 0.26.0 and
|
|
959
|
+
later; an older `version` gets them as arguments, with a warning).
|
|
960
|
+
|
|
684
961
|
## Limitations
|
|
685
962
|
|
|
686
963
|
- **Server order isn't final client order.** Clients merge tools from several servers
|
package/dist/args.d.ts
CHANGED
|
@@ -19,6 +19,16 @@ export declare function fits(value: unknown, schema: JsonSchema | undefined): bo
|
|
|
19
19
|
* tool that needs one is skipped, not called with junk.
|
|
20
20
|
*/
|
|
21
21
|
export declare function synthesize(schema: JsonSchema | undefined, name?: string): Synthesized;
|
|
22
|
+
/** Made-up values are marked as toolmenu's, so they're easy to find in a server's logs. */
|
|
23
|
+
export declare const INVENTED = "toolmenu-test";
|
|
24
|
+
/**
|
|
25
|
+
* A value of the right type where the schema itself vouches for none, for
|
|
26
|
+
* `session --auto --call-everything`: the schema's own value first (synthesize),
|
|
27
|
+
* then a made-up one ("toolmenu-test", 1, a one-item list, an object of invented
|
|
28
|
+
* required fields). A pattern it can't satisfy gets the made-up string anyway: the
|
|
29
|
+
* tool's answer to a value it refuses is part of what's checked.
|
|
30
|
+
*/
|
|
31
|
+
export declare function invent(schema: JsonSchema | undefined, name?: string, depth?: number): unknown;
|
|
22
32
|
/**
|
|
23
33
|
* A value the user gave (--value), made to fit the parameter: `5` is the number 5
|
|
24
34
|
* for an integer and the string "5" for a string; JSON that doesn't fit the type
|
|
@@ -29,7 +39,7 @@ export declare function fitValue(given: {
|
|
|
29
39
|
value: unknown;
|
|
30
40
|
}, schema: JsonSchema | undefined): unknown;
|
|
31
41
|
/** Where a tool's argument came from: the user (--value) or the schema. */
|
|
32
|
-
export type ArgSource = 'value' | 'schema';
|
|
42
|
+
export type ArgSource = 'value' | 'schema' | 'invented';
|
|
33
43
|
/**
|
|
34
44
|
* Arguments for a tool's required parameters, or the ones it can't fill. `given`:
|
|
35
45
|
* values the user supplied for this tool, by parameter name; they come first, and
|
|
@@ -38,7 +48,9 @@ export type ArgSource = 'value' | 'schema';
|
|
|
38
48
|
export declare function synthesizeArgs(schema: JsonSchema | undefined, given?: Record<string, {
|
|
39
49
|
raw?: string;
|
|
40
50
|
value: unknown;
|
|
41
|
-
}
|
|
51
|
+
}>, options?: {
|
|
52
|
+
invent?: boolean;
|
|
53
|
+
}): {
|
|
42
54
|
ok: true;
|
|
43
55
|
args: Record<string, unknown>;
|
|
44
56
|
sources: Record<string, ArgSource>;
|
package/dist/args.js
CHANGED
|
@@ -107,6 +107,45 @@ export function synthesize(schema, name = '') {
|
|
|
107
107
|
return { ok: false };
|
|
108
108
|
}
|
|
109
109
|
}
|
|
110
|
+
/** Made-up values are marked as toolmenu's, so they're easy to find in a server's logs. */
|
|
111
|
+
export const INVENTED = 'toolmenu-test';
|
|
112
|
+
/**
|
|
113
|
+
* A value of the right type where the schema itself vouches for none, for
|
|
114
|
+
* `session --auto --call-everything`: the schema's own value first (synthesize),
|
|
115
|
+
* then a made-up one ("toolmenu-test", 1, a one-item list, an object of invented
|
|
116
|
+
* required fields). A pattern it can't satisfy gets the made-up string anyway: the
|
|
117
|
+
* tool's answer to a value it refuses is part of what's checked.
|
|
118
|
+
*/
|
|
119
|
+
export function invent(schema, name = '', depth = 0) {
|
|
120
|
+
const own = synthesize(schema, name);
|
|
121
|
+
if (own.ok)
|
|
122
|
+
return own.value;
|
|
123
|
+
if (!schema || typeof schema !== 'object' || depth > 8)
|
|
124
|
+
return INVENTED;
|
|
125
|
+
const branches = (schema.anyOf ?? schema.oneOf);
|
|
126
|
+
const type = Array.isArray(schema.type) ? schema.type.find((t) => t !== 'null') : schema.type;
|
|
127
|
+
if (!type && Array.isArray(branches) && branches.length)
|
|
128
|
+
return invent(branches.find((b) => b && typeof b === 'object' && b.type !== 'null') ?? branches[0], name, depth + 1);
|
|
129
|
+
switch (type) {
|
|
130
|
+
case 'boolean':
|
|
131
|
+
return false;
|
|
132
|
+
case 'integer':
|
|
133
|
+
case 'number':
|
|
134
|
+
return 1;
|
|
135
|
+
case 'array':
|
|
136
|
+
return [invent(schema.items, name, depth + 1)];
|
|
137
|
+
case 'object': {
|
|
138
|
+
const out = {};
|
|
139
|
+
for (const p of schema.required ?? [])
|
|
140
|
+
out[p] = invent(schema.properties?.[p], p, depth + 1);
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
case 'null':
|
|
144
|
+
return null;
|
|
145
|
+
default:
|
|
146
|
+
return INVENTED;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
110
149
|
/**
|
|
111
150
|
* A value the user gave (--value), made to fit the parameter: `5` is the number 5
|
|
112
151
|
* for an integer and the string "5" for a string; JSON that doesn't fit the type
|
|
@@ -127,7 +166,7 @@ export function fitValue(given, schema) {
|
|
|
127
166
|
* values the user supplied for this tool, by parameter name; they come first, and
|
|
128
167
|
* one for an optional parameter is passed too.
|
|
129
168
|
*/
|
|
130
|
-
export function synthesizeArgs(schema, given = {}) {
|
|
169
|
+
export function synthesizeArgs(schema, given = {}, options = {}) {
|
|
131
170
|
const args = {};
|
|
132
171
|
const sources = {};
|
|
133
172
|
const missing = [];
|
|
@@ -143,6 +182,10 @@ export function synthesizeArgs(schema, given = {}) {
|
|
|
143
182
|
args[param] = v.value;
|
|
144
183
|
sources[param] = 'schema';
|
|
145
184
|
}
|
|
185
|
+
else if (options.invent) {
|
|
186
|
+
args[param] = invent(schema?.properties?.[param], param);
|
|
187
|
+
sources[param] = 'invented';
|
|
188
|
+
}
|
|
146
189
|
else
|
|
147
190
|
missing.push(param);
|
|
148
191
|
}
|