toolmenu 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -3
- package/dist/args.d.ts +22 -0
- package/dist/args.js +81 -0
- package/dist/auth.d.ts +77 -0
- package/dist/auth.js +304 -0
- package/dist/auto.d.ts +28 -0
- package/dist/auto.js +81 -0
- package/dist/cli.js +96 -7
- package/dist/connect.d.ts +1 -0
- package/dist/connect.js +11 -1
- package/dist/report.js +22 -2
- package/dist/session.d.ts +10 -0
- package/dist/session.js +7 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
|
|
|
29
29
|
(`history` installs old versions with today's dependencies, so it records what they
|
|
30
30
|
serve now, which can differ from what they shipped with: FINDINGS F4.)
|
|
31
31
|
|
|
32
|
-
> **Status: 0.
|
|
32
|
+
> **Status: 0.9, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
|
|
33
33
|
> real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
34
34
|
|
|
35
35
|
## Start here
|
|
@@ -48,6 +48,19 @@ npx toolmenu snapshot -- node dist/server.js
|
|
|
48
48
|
```
|
|
49
49
|
|
|
50
50
|
HTTP servers work the same way: `npx toolmenu snapshot https://example.com/mcp --header "Authorization: Bearer $TOKEN"`.
|
|
51
|
+
Servers behind OAuth (most hosted ones: GitHub, Atlassian, Sentry…): log in once, in a
|
|
52
|
+
browser, and `snapshot` and `session` use that login from then on, refreshing it as
|
|
53
|
+
needed:
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
npx toolmenu auth login https://mcp.example.com/mcp
|
|
57
|
+
npx toolmenu snapshot https://mcp.example.com/mcp
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Logins are stored per server in `~/.config/toolmenu/auth` (readable only by you);
|
|
61
|
+
`auth list` and `auth logout <url>` manage them. A server that doesn't allow
|
|
62
|
+
dynamic registration needs a pre-registered app: `--client-id` (and
|
|
63
|
+
`--client-secret`). In CI, where no browser can open, pass a token as a header.
|
|
51
64
|
Both protocol generations are supported: 2026-07-28 (`server/discover`, stateless)
|
|
52
65
|
and the 2025 `initialize` handshake, through the official TypeScript SDK.
|
|
53
66
|
|
|
@@ -67,7 +80,7 @@ jobs:
|
|
|
67
80
|
- uses: actions/setup-node@v7
|
|
68
81
|
with: { node-version: 22 }
|
|
69
82
|
- run: npm ci && npm run build
|
|
70
|
-
- uses: niksa90/toolmenu@v0.
|
|
83
|
+
- uses: niksa90/toolmenu@v0.9.0
|
|
71
84
|
with:
|
|
72
85
|
command: node dist/server.js
|
|
73
86
|
baseline: menu.json # your committed snapshot
|
|
@@ -86,7 +99,8 @@ and anything `session` caught. The same report goes to the job summary.
|
|
|
86
99
|
| `headers` | | HTTP headers for a `url` server, one `Name: value` per line. Use a secret for keys: `x-api-key: ${{ secrets.MCP_API_KEY }}` |
|
|
87
100
|
| `env` | | Environment variables for a `command` server, one `KEY=value` per line |
|
|
88
101
|
| `baseline` | `menu.json` | The committed snapshot to diff against |
|
|
89
|
-
| `scenario` | | A scenario to run with `session` |
|
|
102
|
+
| `scenario` | | A scenario to run with `session`, or `auto` to build the steps from the menu |
|
|
103
|
+
| `baseline-from` | `snapshot` | `session` diffs every tool the scenario saw (the union menu), for servers whose tools appear after an unlock |
|
|
90
104
|
| `release` | `auto` | Release versions for the bump check. `auto` reads `package.json`, `pyproject.toml` or `Cargo.toml` on the base branch and the head, or compares the tag with the previous one on a tag push. Set `"1.4.0..1.5.0"` yourself, or `off` |
|
|
91
105
|
| `fail-on` | `error` | Fail the job on findings at or above this level |
|
|
92
106
|
| `comment` | `true` | Post and update the PR comment |
|
|
@@ -132,6 +146,16 @@ as its steps, so the starter is a floor, not a ceiling.
|
|
|
132
146
|
| `session/refused`, `session/step-failed` | error | A write the scenario didn't allow, or a call that failed for another reason |
|
|
133
147
|
| `session/tool-error` | warn | A tool that answered with an error for another reason (`isError`): the run tested less than it looks |
|
|
134
148
|
|
|
149
|
+
**No scenario? `session --auto`** builds the steps from the menu: every read-only tool
|
|
150
|
+
whose required arguments the schema itself gives (a `default`, `examples`, an `enum`,
|
|
151
|
+
a type or format), cheapest first, then the first call again. It never guesses an ID:
|
|
152
|
+
a tool that needs `owner`, `repo` or an issue key is skipped, and the report says
|
|
153
|
+
which values were missing (`--save-scenario auto.yml` writes the steps, with the
|
|
154
|
+
skipped tools commented out, to fill in). Tools marked `openWorldHint: true` (web
|
|
155
|
+
search, fetch, scraping, which can cost API credits) are called only with
|
|
156
|
+
`--open-world`. On the 22 servers in `bench/`, `--auto` reaches 60 of 262 read-only
|
|
157
|
+
tools, 84 with `--open-world`.
|
|
158
|
+
|
|
135
159
|
**`session` calls tools for real.** Without `allow_writes: true` it refuses any tool
|
|
136
160
|
not marked `readOnlyHint: true`. `--plan` prints the steps without connecting.
|
|
137
161
|
|
package/dist/args.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { JsonSchema } from './types.js';
|
|
2
|
+
/** A value for a schema, or why there's none. */
|
|
3
|
+
export type Synthesized = {
|
|
4
|
+
ok: true;
|
|
5
|
+
value: unknown;
|
|
6
|
+
} | {
|
|
7
|
+
ok: false;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* A value the schema itself vouches for: `const`, `default`, `examples`, `enum`,
|
|
11
|
+
* then by type and format. Never a guess at an ID or anything with a `pattern`:
|
|
12
|
+
* a tool that needs one is skipped, not called with junk.
|
|
13
|
+
*/
|
|
14
|
+
export declare function synthesize(schema: JsonSchema | undefined, name?: string): Synthesized;
|
|
15
|
+
/** Arguments for a tool's required parameters, or the ones it can't fill. */
|
|
16
|
+
export declare function synthesizeArgs(schema: JsonSchema | undefined): {
|
|
17
|
+
ok: true;
|
|
18
|
+
args: Record<string, unknown>;
|
|
19
|
+
} | {
|
|
20
|
+
ok: false;
|
|
21
|
+
missing: string[];
|
|
22
|
+
};
|
package/dist/args.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// Fixed dates, not "now": the same menu gives the same calls on every run.
|
|
2
|
+
const DATE_TIME = '2026-01-01T00:00:00Z';
|
|
3
|
+
const DATE = '2026-01-01';
|
|
4
|
+
const SEARCH_WORDS = /^(q|query|search|search_?query|text|keywords?|terms?|prompt|question)$/i;
|
|
5
|
+
/**
|
|
6
|
+
* A value the schema itself vouches for: `const`, `default`, `examples`, `enum`,
|
|
7
|
+
* then by type and format. Never a guess at an ID or anything with a `pattern`:
|
|
8
|
+
* a tool that needs one is skipped, not called with junk.
|
|
9
|
+
*/
|
|
10
|
+
export function synthesize(schema, name = '') {
|
|
11
|
+
if (!schema || typeof schema !== 'object')
|
|
12
|
+
return { ok: false };
|
|
13
|
+
if ('const' in schema)
|
|
14
|
+
return { ok: true, value: schema.const };
|
|
15
|
+
if ('default' in schema)
|
|
16
|
+
return { ok: true, value: schema.default };
|
|
17
|
+
if (Array.isArray(schema.examples) && schema.examples.length)
|
|
18
|
+
return { ok: true, value: schema.examples[0] };
|
|
19
|
+
if (schema.enum?.length)
|
|
20
|
+
return { ok: true, value: schema.enum[0] };
|
|
21
|
+
const branches = (schema.anyOf ?? schema.oneOf);
|
|
22
|
+
const type = Array.isArray(schema.type) ? schema.type.find((t) => t !== 'null') : schema.type;
|
|
23
|
+
if (!type && Array.isArray(branches)) {
|
|
24
|
+
for (const b of branches) {
|
|
25
|
+
const v = synthesize(b, name);
|
|
26
|
+
if (v.ok)
|
|
27
|
+
return v;
|
|
28
|
+
}
|
|
29
|
+
return { ok: false };
|
|
30
|
+
}
|
|
31
|
+
switch (type) {
|
|
32
|
+
case 'boolean':
|
|
33
|
+
return { ok: true, value: false };
|
|
34
|
+
case 'integer':
|
|
35
|
+
case 'number': {
|
|
36
|
+
const min = typeof schema.minimum === 'number' ? schema.minimum : typeof schema.exclusiveMinimum === 'number' ? schema.exclusiveMinimum + 1 : undefined;
|
|
37
|
+
return { ok: true, value: min ?? 1 };
|
|
38
|
+
}
|
|
39
|
+
case 'string': {
|
|
40
|
+
if (typeof schema.pattern === 'string')
|
|
41
|
+
return { ok: false };
|
|
42
|
+
const format = typeof schema.format === 'string' ? schema.format : '';
|
|
43
|
+
if (format === 'date-time')
|
|
44
|
+
return { ok: true, value: DATE_TIME };
|
|
45
|
+
if (format === 'date')
|
|
46
|
+
return { ok: true, value: DATE };
|
|
47
|
+
if (format === 'uri' || format === 'url')
|
|
48
|
+
return { ok: true, value: 'https://example.com' };
|
|
49
|
+
if (format === 'email')
|
|
50
|
+
return { ok: true, value: 'user@example.com' };
|
|
51
|
+
if (SEARCH_WORDS.test(name))
|
|
52
|
+
return { ok: true, value: 'test' };
|
|
53
|
+
return { ok: false };
|
|
54
|
+
}
|
|
55
|
+
case 'array': {
|
|
56
|
+
const item = synthesize(schema.items, name);
|
|
57
|
+
if (item.ok)
|
|
58
|
+
return { ok: true, value: [item.value] };
|
|
59
|
+
return typeof schema.minItems === 'number' && schema.minItems > 0 ? { ok: false } : { ok: true, value: [] };
|
|
60
|
+
}
|
|
61
|
+
case 'object': {
|
|
62
|
+
const args = synthesizeArgs(schema);
|
|
63
|
+
return args.ok ? { ok: true, value: args.args } : { ok: false };
|
|
64
|
+
}
|
|
65
|
+
default:
|
|
66
|
+
return { ok: false };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** Arguments for a tool's required parameters, or the ones it can't fill. */
|
|
70
|
+
export function synthesizeArgs(schema) {
|
|
71
|
+
const args = {};
|
|
72
|
+
const missing = [];
|
|
73
|
+
for (const param of schema?.required ?? []) {
|
|
74
|
+
const v = synthesize(schema?.properties?.[param], param);
|
|
75
|
+
if (v.ok)
|
|
76
|
+
args[param] = v.value;
|
|
77
|
+
else
|
|
78
|
+
missing.push(param);
|
|
79
|
+
}
|
|
80
|
+
return missing.length ? { ok: false, missing } : { ok: true, args };
|
|
81
|
+
}
|
package/dist/auth.d.ts
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { OAuthClientInformationContext, OAuthClientMetadata, OAuthClientProvider, OAuthDiscoveryState, StoredOAuthClientInformation, StoredOAuthTokens } from '@modelcontextprotocol/client';
|
|
2
|
+
/**
|
|
3
|
+
* The redirect port. Fixed, not random: dynamic registration records the exact
|
|
4
|
+
* redirect URI, so a new port would invalidate the stored client on every login.
|
|
5
|
+
*/
|
|
6
|
+
export declare const DEFAULT_PORT = 33418;
|
|
7
|
+
/** Where logins live: $TOOLMENU_AUTH_DIR, else $XDG_CONFIG_HOME/toolmenu/auth, else ~/.config/toolmenu/auth. */
|
|
8
|
+
export declare function authDir(): string;
|
|
9
|
+
/** Is there a stored login for this server? */
|
|
10
|
+
export declare function hasLogin(serverUrl: string): boolean;
|
|
11
|
+
export interface ProviderOptions {
|
|
12
|
+
/** A browser login may start (auth login). Otherwise a needed login is an error. */
|
|
13
|
+
interactive?: boolean;
|
|
14
|
+
port?: number;
|
|
15
|
+
scope?: string;
|
|
16
|
+
/** A pre-registered client, for servers that don't allow dynamic registration. */
|
|
17
|
+
clientId?: string;
|
|
18
|
+
clientSecret?: string;
|
|
19
|
+
onRedirect?: (url: URL) => void | Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The SDK's OAuthClientProvider, backed by toolmenu's store. Credentials are bound
|
|
23
|
+
* to the authorization server's issuer: stored ones for a different issuer are
|
|
24
|
+
* never handed out.
|
|
25
|
+
*/
|
|
26
|
+
export declare class StoredOAuthProvider implements OAuthClientProvider {
|
|
27
|
+
private readonly serverUrl;
|
|
28
|
+
private readonly options;
|
|
29
|
+
private data;
|
|
30
|
+
private expectedState;
|
|
31
|
+
constructor(serverUrl: string, options?: ProviderOptions);
|
|
32
|
+
private stored;
|
|
33
|
+
private update;
|
|
34
|
+
/** The issuer check: no credentials for an authorization server they weren't issued by. */
|
|
35
|
+
private sameIssuer;
|
|
36
|
+
get redirectUrl(): string;
|
|
37
|
+
get clientMetadata(): OAuthClientMetadata;
|
|
38
|
+
state(): string;
|
|
39
|
+
/** The state the authorization request carried, to check the callback against (the SDK doesn't). */
|
|
40
|
+
get lastState(): string | undefined;
|
|
41
|
+
clientInformation(ctx?: OAuthClientInformationContext): Promise<StoredOAuthClientInformation | undefined>;
|
|
42
|
+
saveClientInformation(clientInformation: StoredOAuthClientInformation, ctx?: OAuthClientInformationContext): Promise<void>;
|
|
43
|
+
tokens(ctx?: OAuthClientInformationContext): Promise<StoredOAuthTokens | undefined>;
|
|
44
|
+
saveTokens(tokens: StoredOAuthTokens, ctx?: OAuthClientInformationContext): Promise<void>;
|
|
45
|
+
redirectToAuthorization(authorizationUrl: URL): Promise<void>;
|
|
46
|
+
saveCodeVerifier(codeVerifier: string): Promise<void>;
|
|
47
|
+
codeVerifier(): Promise<string>;
|
|
48
|
+
saveDiscoveryState(discoveryState: OAuthDiscoveryState): Promise<void>;
|
|
49
|
+
discoveryState(): Promise<OAuthDiscoveryState | undefined>;
|
|
50
|
+
invalidateCredentials(scope: 'all' | 'client' | 'tokens' | 'verifier' | 'discovery'): Promise<void>;
|
|
51
|
+
}
|
|
52
|
+
export declare class LoginNeededError extends Error {
|
|
53
|
+
constructor(serverUrl: string);
|
|
54
|
+
}
|
|
55
|
+
export interface LoginOptions extends Omit<ProviderOptions, 'interactive' | 'onRedirect'> {
|
|
56
|
+
/** How to show the authorization URL. Default: print it and try to open a browser. */
|
|
57
|
+
open?: (url: URL) => void | Promise<void>;
|
|
58
|
+
/** How long to wait for the browser to come back (default 5 min). */
|
|
59
|
+
timeoutMs?: number;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Log in to an OAuth-protected MCP server: register (or reuse a client), open the
|
|
63
|
+
* authorization URL, receive the code on a loopback redirect, check `state`,
|
|
64
|
+
* exchange the code (PKCE, issuer check: the SDK's), then connect once with the new
|
|
65
|
+
* token to confirm it works. Returns the server's name and tool count.
|
|
66
|
+
*/
|
|
67
|
+
export declare function login(serverUrl: string, options?: LoginOptions): Promise<{
|
|
68
|
+
name?: string;
|
|
69
|
+
tools: number;
|
|
70
|
+
}>;
|
|
71
|
+
export declare function logout(serverUrl: string): Promise<boolean>;
|
|
72
|
+
/** Stored logins: the server, its issuer, and when the tokens were saved. No secrets. */
|
|
73
|
+
export declare function listLogins(): Promise<{
|
|
74
|
+
serverUrl: string;
|
|
75
|
+
issuer?: string;
|
|
76
|
+
savedAt?: string;
|
|
77
|
+
}[]>;
|
package/dist/auth.js
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
3
|
+
import { existsSync, readdirSync } from 'node:fs';
|
|
4
|
+
import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
5
|
+
import { createServer } from 'node:http';
|
|
6
|
+
import { homedir } from 'node:os';
|
|
7
|
+
import { join } from 'node:path';
|
|
8
|
+
import { Client, StreamableHTTPClientTransport, UnauthorizedError } from '@modelcontextprotocol/client';
|
|
9
|
+
import { VERSION } from './version.js';
|
|
10
|
+
/**
|
|
11
|
+
* The redirect port. Fixed, not random: dynamic registration records the exact
|
|
12
|
+
* redirect URI, so a new port would invalidate the stored client on every login.
|
|
13
|
+
*/
|
|
14
|
+
export const DEFAULT_PORT = 33418;
|
|
15
|
+
/** Where logins live: $TOOLMENU_AUTH_DIR, else $XDG_CONFIG_HOME/toolmenu/auth, else ~/.config/toolmenu/auth. */
|
|
16
|
+
export function authDir() {
|
|
17
|
+
if (process.env.TOOLMENU_AUTH_DIR)
|
|
18
|
+
return process.env.TOOLMENU_AUTH_DIR;
|
|
19
|
+
return join(process.env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'toolmenu', 'auth');
|
|
20
|
+
}
|
|
21
|
+
/** One file per server endpoint (origin and path), named by its hash. */
|
|
22
|
+
function storePath(serverUrl) {
|
|
23
|
+
const u = new URL(serverUrl);
|
|
24
|
+
return join(authDir(), createHash('sha256').update(u.origin + u.pathname).digest('hex').slice(0, 32) + '.json');
|
|
25
|
+
}
|
|
26
|
+
async function load(serverUrl) {
|
|
27
|
+
try {
|
|
28
|
+
return JSON.parse(await readFile(storePath(serverUrl), 'utf8'));
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return { serverUrl };
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function save(data) {
|
|
35
|
+
await mkdir(authDir(), { recursive: true, mode: 0o700 });
|
|
36
|
+
const path = storePath(data.serverUrl);
|
|
37
|
+
await writeFile(path, JSON.stringify({ ...data, savedAt: new Date().toISOString() }, null, 2) + '\n', { mode: 0o600 });
|
|
38
|
+
await chmod(path, 0o600);
|
|
39
|
+
}
|
|
40
|
+
/** Is there a stored login for this server? */
|
|
41
|
+
export function hasLogin(serverUrl) {
|
|
42
|
+
return existsSync(storePath(serverUrl));
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The SDK's OAuthClientProvider, backed by toolmenu's store. Credentials are bound
|
|
46
|
+
* to the authorization server's issuer: stored ones for a different issuer are
|
|
47
|
+
* never handed out.
|
|
48
|
+
*/
|
|
49
|
+
export class StoredOAuthProvider {
|
|
50
|
+
serverUrl;
|
|
51
|
+
options;
|
|
52
|
+
data;
|
|
53
|
+
expectedState;
|
|
54
|
+
constructor(serverUrl, options = {}) {
|
|
55
|
+
this.serverUrl = serverUrl;
|
|
56
|
+
this.options = options;
|
|
57
|
+
}
|
|
58
|
+
async stored() {
|
|
59
|
+
this.data ??= await load(this.serverUrl);
|
|
60
|
+
return this.data;
|
|
61
|
+
}
|
|
62
|
+
async update(patch) {
|
|
63
|
+
this.data = { ...(await this.stored()), ...patch };
|
|
64
|
+
await save(this.data);
|
|
65
|
+
}
|
|
66
|
+
/** The issuer check: no credentials for an authorization server they weren't issued by. */
|
|
67
|
+
async sameIssuer(ctx) {
|
|
68
|
+
const s = await this.stored();
|
|
69
|
+
return !ctx?.issuer || !s.issuer || s.issuer === ctx.issuer;
|
|
70
|
+
}
|
|
71
|
+
get redirectUrl() {
|
|
72
|
+
return `http://127.0.0.1:${this.options.port ?? DEFAULT_PORT}/callback`;
|
|
73
|
+
}
|
|
74
|
+
get clientMetadata() {
|
|
75
|
+
return {
|
|
76
|
+
client_name: 'toolmenu',
|
|
77
|
+
client_uri: 'https://github.com/niksa90/toolmenu',
|
|
78
|
+
software_id: 'toolmenu',
|
|
79
|
+
software_version: VERSION,
|
|
80
|
+
redirect_uris: [this.redirectUrl],
|
|
81
|
+
grant_types: ['authorization_code', 'refresh_token'],
|
|
82
|
+
response_types: ['code'],
|
|
83
|
+
token_endpoint_auth_method: this.options.clientSecret ? 'client_secret_post' : 'none',
|
|
84
|
+
...(this.options.scope ? { scope: this.options.scope } : {}),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
state() {
|
|
88
|
+
this.expectedState = randomBytes(16).toString('base64url');
|
|
89
|
+
return this.expectedState;
|
|
90
|
+
}
|
|
91
|
+
/** The state the authorization request carried, to check the callback against (the SDK doesn't). */
|
|
92
|
+
get lastState() {
|
|
93
|
+
return this.expectedState;
|
|
94
|
+
}
|
|
95
|
+
async clientInformation(ctx) {
|
|
96
|
+
const s = await this.stored();
|
|
97
|
+
if (this.options.clientId) {
|
|
98
|
+
// Stamped with the issuer, like everything the SDK stores (SEP-2352): the one
|
|
99
|
+
// this server's login was first bound to, or on a first login the one being
|
|
100
|
+
// discovered. The SDK then won't hand this client to another server.
|
|
101
|
+
if (s.issuer && ctx?.issuer && s.issuer !== ctx.issuer)
|
|
102
|
+
return undefined;
|
|
103
|
+
const issuer = s.issuer ?? ctx?.issuer;
|
|
104
|
+
const info = { client_id: this.options.clientId, ...(this.options.clientSecret ? { client_secret: this.options.clientSecret } : {}), ...(issuer ? { issuer } : {}) };
|
|
105
|
+
// Kept like a registered client, so a later snapshot can refresh the token
|
|
106
|
+
// without --client-id.
|
|
107
|
+
if (JSON.stringify(s.clientInformation) !== JSON.stringify(info))
|
|
108
|
+
await this.update({ clientInformation: info, redirectUrl: this.redirectUrl });
|
|
109
|
+
return info;
|
|
110
|
+
}
|
|
111
|
+
if (!(await this.sameIssuer(ctx)))
|
|
112
|
+
return undefined;
|
|
113
|
+
// Registered for another redirect URI (a different --port): register again.
|
|
114
|
+
if (this.options.interactive && s.redirectUrl && s.redirectUrl !== this.redirectUrl)
|
|
115
|
+
return undefined;
|
|
116
|
+
return s.clientInformation;
|
|
117
|
+
}
|
|
118
|
+
async saveClientInformation(clientInformation, ctx) {
|
|
119
|
+
await this.update({ clientInformation, redirectUrl: this.redirectUrl, ...(ctx?.issuer ? { issuer: ctx.issuer } : {}) });
|
|
120
|
+
}
|
|
121
|
+
async tokens(ctx) {
|
|
122
|
+
if (!(await this.sameIssuer(ctx)))
|
|
123
|
+
return undefined;
|
|
124
|
+
return (await this.stored()).tokens;
|
|
125
|
+
}
|
|
126
|
+
async saveTokens(tokens, ctx) {
|
|
127
|
+
await this.update({ tokens, ...(ctx?.issuer ? { issuer: ctx.issuer } : {}) });
|
|
128
|
+
}
|
|
129
|
+
async redirectToAuthorization(authorizationUrl) {
|
|
130
|
+
if (!this.options.interactive) {
|
|
131
|
+
throw new LoginNeededError(this.serverUrl);
|
|
132
|
+
}
|
|
133
|
+
await this.options.onRedirect?.(authorizationUrl);
|
|
134
|
+
}
|
|
135
|
+
async saveCodeVerifier(codeVerifier) {
|
|
136
|
+
await this.update({ codeVerifier });
|
|
137
|
+
}
|
|
138
|
+
async codeVerifier() {
|
|
139
|
+
const v = (await this.stored()).codeVerifier;
|
|
140
|
+
if (!v)
|
|
141
|
+
throw new Error('No PKCE code verifier saved for this login.');
|
|
142
|
+
return v;
|
|
143
|
+
}
|
|
144
|
+
async saveDiscoveryState(discoveryState) {
|
|
145
|
+
await this.update({ discoveryState });
|
|
146
|
+
}
|
|
147
|
+
async discoveryState() {
|
|
148
|
+
return (await this.stored()).discoveryState;
|
|
149
|
+
}
|
|
150
|
+
async invalidateCredentials(scope) {
|
|
151
|
+
const s = await this.stored();
|
|
152
|
+
if (scope === 'all') {
|
|
153
|
+
this.data = { serverUrl: s.serverUrl };
|
|
154
|
+
await rm(storePath(this.serverUrl), { force: true });
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
const patch = {};
|
|
158
|
+
if (scope === 'client')
|
|
159
|
+
patch.clientInformation = undefined;
|
|
160
|
+
if (scope === 'tokens')
|
|
161
|
+
patch.tokens = undefined;
|
|
162
|
+
if (scope === 'verifier')
|
|
163
|
+
patch.codeVerifier = undefined;
|
|
164
|
+
if (scope === 'discovery')
|
|
165
|
+
patch.discoveryState = undefined;
|
|
166
|
+
await this.update(patch);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
export class LoginNeededError extends Error {
|
|
170
|
+
constructor(serverUrl) {
|
|
171
|
+
super(`${serverUrl} needs an OAuth login, and toolmenu can't open a browser here. Run: toolmenu auth login ${serverUrl}`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Log in to an OAuth-protected MCP server: register (or reuse a client), open the
|
|
176
|
+
* authorization URL, receive the code on a loopback redirect, check `state`,
|
|
177
|
+
* exchange the code (PKCE, issuer check: the SDK's), then connect once with the new
|
|
178
|
+
* token to confirm it works. Returns the server's name and tool count.
|
|
179
|
+
*/
|
|
180
|
+
export async function login(serverUrl, options = {}) {
|
|
181
|
+
const port = options.port ?? DEFAULT_PORT;
|
|
182
|
+
const callback = await listenForCallback(port, options.timeoutMs ?? 300_000);
|
|
183
|
+
try {
|
|
184
|
+
const provider = new StoredOAuthProvider(serverUrl, { ...options, port, interactive: true, onRedirect: options.open ?? openInBrowser });
|
|
185
|
+
// Each login discovers the authorization server afresh (a server may have
|
|
186
|
+
// moved); the state is then kept for the callback leg's check.
|
|
187
|
+
await provider.invalidateCredentials('discovery');
|
|
188
|
+
const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { authProvider: provider });
|
|
189
|
+
const client = new Client({ name: 'toolmenu', version: VERSION });
|
|
190
|
+
try {
|
|
191
|
+
await client.connect(transport);
|
|
192
|
+
// Already logged in with a valid (or refreshed) token.
|
|
193
|
+
await client.close();
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (!(error instanceof UnauthorizedError))
|
|
197
|
+
throw error;
|
|
198
|
+
const params = await callback.params;
|
|
199
|
+
if (params.get('error'))
|
|
200
|
+
throw new Error(`The authorization server refused the login: ${params.get('error')}`);
|
|
201
|
+
if (!provider.lastState || params.get('state') !== provider.lastState)
|
|
202
|
+
throw new Error('The login callback carried the wrong state, so it was ignored. Try again.');
|
|
203
|
+
await transport.finishAuth(params);
|
|
204
|
+
await client.close().catch(() => { });
|
|
205
|
+
}
|
|
206
|
+
return await check(serverUrl);
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
// GitHub's hosted server, for one: no dynamic registration, so toolmenu can't
|
|
210
|
+
// register itself. Say what works instead.
|
|
211
|
+
if (error instanceof Error && /does not support dynamic client registration/i.test(error.message) && !options.clientId) {
|
|
212
|
+
error.message =
|
|
213
|
+
`${serverUrl} doesn't let clients register themselves (no dynamic client registration). Two ways in:\n` +
|
|
214
|
+
` - register an OAuth app with the provider, with the callback URL http://127.0.0.1:${port}/callback, then:\n` +
|
|
215
|
+
` toolmenu auth login ${serverUrl} --client-id <id> --client-secret <secret>\n` +
|
|
216
|
+
` - or skip OAuth and pass a token the server accepts: toolmenu snapshot ${serverUrl} --header "Authorization: Bearer <token>"`;
|
|
217
|
+
}
|
|
218
|
+
throw error;
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
callback.close();
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/** Connect with the stored login and list tools: the login works. */
|
|
225
|
+
async function check(serverUrl) {
|
|
226
|
+
const client = new Client({ name: 'toolmenu', version: VERSION });
|
|
227
|
+
await client.connect(new StreamableHTTPClientTransport(new URL(serverUrl), { authProvider: new StoredOAuthProvider(serverUrl) }));
|
|
228
|
+
try {
|
|
229
|
+
const { tools } = await client.listTools();
|
|
230
|
+
return { name: client.getServerVersion()?.name, tools: tools.length };
|
|
231
|
+
}
|
|
232
|
+
finally {
|
|
233
|
+
await client.close().catch(() => { });
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
export async function logout(serverUrl) {
|
|
237
|
+
const had = hasLogin(serverUrl);
|
|
238
|
+
await rm(storePath(serverUrl), { force: true });
|
|
239
|
+
return had;
|
|
240
|
+
}
|
|
241
|
+
/** Stored logins: the server, its issuer, and when the tokens were saved. No secrets. */
|
|
242
|
+
export async function listLogins() {
|
|
243
|
+
if (!existsSync(authDir()))
|
|
244
|
+
return [];
|
|
245
|
+
const out = [];
|
|
246
|
+
for (const f of readdirSync(authDir()).filter((f) => f.endsWith('.json'))) {
|
|
247
|
+
try {
|
|
248
|
+
const s = JSON.parse(await readFile(join(authDir(), f), 'utf8'));
|
|
249
|
+
out.push({ serverUrl: s.serverUrl, issuer: s.issuer, savedAt: s.savedAt });
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
// not ours
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return out;
|
|
256
|
+
}
|
|
257
|
+
/** A one-shot HTTP server on 127.0.0.1 that resolves with the callback's query. */
|
|
258
|
+
function listenForCallback(port, timeoutMs) {
|
|
259
|
+
return new Promise((ready, fail) => {
|
|
260
|
+
let resolveParams;
|
|
261
|
+
let rejectParams;
|
|
262
|
+
const params = new Promise((resolve, reject) => {
|
|
263
|
+
resolveParams = resolve;
|
|
264
|
+
rejectParams = reject;
|
|
265
|
+
});
|
|
266
|
+
params.catch(() => { }); // surfaced where it's awaited
|
|
267
|
+
const server = createServer((req, res) => {
|
|
268
|
+
const url = new URL(req.url ?? '/', `http://127.0.0.1:${port}`);
|
|
269
|
+
if (url.pathname !== '/callback') {
|
|
270
|
+
res.writeHead(404).end();
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
|
|
274
|
+
res.end('<!doctype html><title>toolmenu</title><p>Logged in. You can close this tab and go back to the terminal.</p>');
|
|
275
|
+
resolveParams(url.searchParams);
|
|
276
|
+
});
|
|
277
|
+
const timer = setTimeout(() => rejectParams(new Error(`No login came back within ${Math.round(timeoutMs / 1000)} s.`)), timeoutMs);
|
|
278
|
+
server.on('error', (error) => fail(error.code === 'EADDRINUSE' ? new Error(`Port ${port} is in use: pick another with --port (the login registers that redirect).`) : error));
|
|
279
|
+
server.listen(port, '127.0.0.1', () => ready({
|
|
280
|
+
params,
|
|
281
|
+
close: () => {
|
|
282
|
+
clearTimeout(timer);
|
|
283
|
+
server.closeAllConnections();
|
|
284
|
+
server.close();
|
|
285
|
+
},
|
|
286
|
+
}));
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
/** Print the URL (always) and try to open it (best effort). */
|
|
290
|
+
function openInBrowser(url) {
|
|
291
|
+
const scope = url.searchParams.get('scope');
|
|
292
|
+
// toolmenu only lists tools and calls read-only ones, but it asks for the
|
|
293
|
+
// server's default scopes, as other clients do: some servers show tools by scope.
|
|
294
|
+
if (scope)
|
|
295
|
+
process.stderr.write(`Asking for the server's default scopes: ${scope} (narrower: --scope; toolmenu never calls a write tool on its own)\n`);
|
|
296
|
+
process.stderr.write(`Opening your browser to log in. If it doesn't open, visit:\n ${url.href}\n`);
|
|
297
|
+
const [cmd, args] = process.platform === 'darwin' ? ['open', [url.href]] : process.platform === 'win32' ? ['cmd', ['/c', 'start', '""', url.href]] : ['xdg-open', [url.href]];
|
|
298
|
+
try {
|
|
299
|
+
spawn(cmd, args, { detached: true, stdio: 'ignore' }).on('error', () => { }).unref();
|
|
300
|
+
}
|
|
301
|
+
catch {
|
|
302
|
+
// printed above
|
|
303
|
+
}
|
|
304
|
+
}
|
package/dist/auto.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Scenario } from './session.js';
|
|
2
|
+
import type { Menu } from './types.js';
|
|
3
|
+
export interface AutoOptions {
|
|
4
|
+
/** Also call read-only tools marked openWorldHint: true (web search, fetch, scraping). */
|
|
5
|
+
openWorld?: boolean;
|
|
6
|
+
/** At most this many calls (default 20). */
|
|
7
|
+
maxCalls?: number;
|
|
8
|
+
}
|
|
9
|
+
export type SkipReason = 'not read-only' | 'open world' | 'needs values' | 'over the call budget';
|
|
10
|
+
export interface AutoPlan {
|
|
11
|
+
scenario: Scenario;
|
|
12
|
+
called: string[];
|
|
13
|
+
skipped: {
|
|
14
|
+
tool: string;
|
|
15
|
+
reason: SkipReason;
|
|
16
|
+
missing?: string[];
|
|
17
|
+
}[];
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* A scenario built from the menu, no hand-written steps: every read-only tool
|
|
21
|
+
* whose required arguments the schema can fill, cheapest first (no arguments,
|
|
22
|
+
* then lists and searches, then single lookups, then the rest), then the first
|
|
23
|
+
* call again, which should change nothing. Never a tool that isn't marked
|
|
24
|
+
* readOnlyHint, whatever allow_writes says.
|
|
25
|
+
*/
|
|
26
|
+
export declare function autoScenario(menu: Menu, options?: AutoOptions): AutoPlan;
|
|
27
|
+
/** The plan as a scenario file, to commit and run again with --scenario. */
|
|
28
|
+
export declare function scenarioYaml(plan: AutoPlan, serverName?: string): string;
|
package/dist/auto.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { synthesizeArgs } from './args.js';
|
|
2
|
+
import { COLLECTION_VERBS, LOOKUP_VERBS, verbOf } from './words.js';
|
|
3
|
+
/**
|
|
4
|
+
* A scenario built from the menu, no hand-written steps: every read-only tool
|
|
5
|
+
* whose required arguments the schema can fill, cheapest first (no arguments,
|
|
6
|
+
* then lists and searches, then single lookups, then the rest), then the first
|
|
7
|
+
* call again, which should change nothing. Never a tool that isn't marked
|
|
8
|
+
* readOnlyHint, whatever allow_writes says.
|
|
9
|
+
*/
|
|
10
|
+
export function autoScenario(menu, options = {}) {
|
|
11
|
+
const maxCalls = options.maxCalls ?? 20;
|
|
12
|
+
const skipped = [];
|
|
13
|
+
const candidates = [];
|
|
14
|
+
for (const tool of menu.tools) {
|
|
15
|
+
const a = tool.annotations ?? {};
|
|
16
|
+
if (a.readOnlyHint !== true) {
|
|
17
|
+
skipped.push({ tool: tool.name, reason: 'not read-only' });
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
// readOnlyHint is the server's own claim, and a read-only search still spends
|
|
21
|
+
// credits on a paid API. Servers that mark a tool openWorldHint: true are the
|
|
22
|
+
// web search, fetch and scraping ones (Exa, Firecrawl, Context7, Playwright):
|
|
23
|
+
// opt-in. Unmarked tools are called: on GitHub, Atlassian or Notion that's
|
|
24
|
+
// every read, of the account the credentials belong to (FINDINGS F13).
|
|
25
|
+
if (a.openWorldHint === true && !options.openWorld) {
|
|
26
|
+
skipped.push({ tool: tool.name, reason: 'open world' });
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
const filled = synthesizeArgs(tool.inputSchema);
|
|
30
|
+
if (!filled.ok) {
|
|
31
|
+
skipped.push({ tool: tool.name, reason: 'needs values', missing: filled.missing });
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args) });
|
|
35
|
+
}
|
|
36
|
+
candidates.sort((x, y) => x.rank - y.rank);
|
|
37
|
+
const chosen = candidates.slice(0, maxCalls);
|
|
38
|
+
for (const c of candidates.slice(maxCalls))
|
|
39
|
+
skipped.push({ tool: c.tool.name, reason: 'over the call budget' });
|
|
40
|
+
const steps = [{ kind: 'list' }];
|
|
41
|
+
for (const c of chosen)
|
|
42
|
+
steps.push({ kind: 'call', tool: c.tool.name, args: c.args });
|
|
43
|
+
if (chosen.length)
|
|
44
|
+
steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args });
|
|
45
|
+
return { scenario: { allowWrites: false, steps }, called: chosen.map((c) => c.tool.name), skipped };
|
|
46
|
+
}
|
|
47
|
+
function rank(tool, args) {
|
|
48
|
+
if (Object.keys(args).length === 0)
|
|
49
|
+
return 0;
|
|
50
|
+
const verb = verbOf(tool.name);
|
|
51
|
+
if (verb && COLLECTION_VERBS.has(verb))
|
|
52
|
+
return 1;
|
|
53
|
+
if (verb && LOOKUP_VERBS.has(verb))
|
|
54
|
+
return 2;
|
|
55
|
+
return 3;
|
|
56
|
+
}
|
|
57
|
+
/** The plan as a scenario file, to commit and run again with --scenario. */
|
|
58
|
+
export function scenarioYaml(plan, serverName) {
|
|
59
|
+
const lines = [
|
|
60
|
+
`# Written by \`toolmenu session --auto\`${serverName ? ` for ${serverName}` : ''}: every read-only tool whose`,
|
|
61
|
+
'# required arguments the schema could fill. Edit freely.',
|
|
62
|
+
'allow_writes: false',
|
|
63
|
+
'steps:',
|
|
64
|
+
];
|
|
65
|
+
for (const step of plan.scenario.steps) {
|
|
66
|
+
if (step.kind === 'list')
|
|
67
|
+
lines.push(' - list');
|
|
68
|
+
else if (step.kind === 'call') {
|
|
69
|
+
lines.push(` - call: ${step.tool}`);
|
|
70
|
+
if (Object.keys(step.args).length)
|
|
71
|
+
lines.push(` args: ${JSON.stringify(step.args)}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const needs = plan.skipped.filter((s) => s.reason === 'needs values');
|
|
75
|
+
if (needs.length) {
|
|
76
|
+
lines.push('', ' # Read-only tools that need values the schema doesn\'t give:');
|
|
77
|
+
for (const s of needs.slice(0, 15))
|
|
78
|
+
lines.push(` # - call: ${s.tool}`, ` # args: { ${s.missing.map((m) => `${m}: TODO`).join(', ')} }`);
|
|
79
|
+
}
|
|
80
|
+
return lines.join('\n') + '\n';
|
|
81
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -13,7 +13,9 @@ import { buildMenu } from './menu.js';
|
|
|
13
13
|
import { existsSync } from 'node:fs';
|
|
14
14
|
import { loadRoutes } from './routes.js';
|
|
15
15
|
import { snapshot } from './snapshot.js';
|
|
16
|
-
import { MAIN_SEED, seeded } from './probe.js';
|
|
16
|
+
import { MAIN_SEED, probeMenu, seeded } from './probe.js';
|
|
17
|
+
import { autoScenario, scenarioYaml } from './auto.js';
|
|
18
|
+
import { authDir, listLogins, login, logout } from './auth.js';
|
|
17
19
|
import { SEVERITY_RANK } from './types.js';
|
|
18
20
|
import { VERSION } from './version.js';
|
|
19
21
|
const HELP = `toolmenu ${VERSION}
|
|
@@ -24,14 +26,18 @@ Usage:
|
|
|
24
26
|
toolmenu snapshot [options] <url> Streamable HTTP server
|
|
25
27
|
toolmenu diff [options] <old.json> <new.json> compare two snapshots
|
|
26
28
|
toolmenu history [options] <npm-package> snapshot and diff published versions
|
|
29
|
+
toolmenu auth login <url> | logout <url> | list OAuth logins for HTTP servers
|
|
27
30
|
toolmenu session --scenario <file> [options] -- <command> | <url>
|
|
28
31
|
run a scripted session, watch the menu
|
|
32
|
+
toolmenu session --auto [options] -- <command> | <url>
|
|
33
|
+
the same, with steps built from the menu
|
|
29
34
|
|
|
30
35
|
Commands:
|
|
31
36
|
snapshot establish the menu: list tools twice, write menu.json, run the menu rules
|
|
32
37
|
diff compare releases: breaking changes, token change, semver bump
|
|
33
38
|
session observe the menu changing while a scripted session runs
|
|
34
39
|
history research release history: install, snapshot and diff published npm versions
|
|
40
|
+
auth log in to an OAuth-protected server once; snapshot and session then use it
|
|
35
41
|
|
|
36
42
|
Options:
|
|
37
43
|
--out <path> where to write the menu (default: menu.json)
|
|
@@ -44,6 +50,7 @@ Options:
|
|
|
44
50
|
--header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
|
|
45
51
|
--env <K=V> environment variable for a stdio server, repeatable. The server
|
|
46
52
|
only gets a minimal environment (PATH, HOME, ...) plus these
|
|
53
|
+
--no-auth don't use a stored OAuth login for this server
|
|
47
54
|
--timeout <ms> per-request timeout (default: 30000)
|
|
48
55
|
--processes <n> server processes (stdio) or connections (HTTP) to compare,
|
|
49
56
|
the main one included (default: 2; 1 turns the check off)
|
|
@@ -61,6 +68,15 @@ session options:
|
|
|
61
68
|
--plan print the steps without connecting or running anything
|
|
62
69
|
--init write a starter scenario from the server's menu (to --scenario,
|
|
63
70
|
default scenario.yml; never overwrites)
|
|
71
|
+
--auto build the steps from the menu: every read-only tool whose
|
|
72
|
+
required arguments the schema can fill (const, default,
|
|
73
|
+
examples, enum, type), then the first call again
|
|
74
|
+
--open-world with --auto, also call read-only tools marked openWorldHint:
|
|
75
|
+
true (web search, fetch, scraping): they may cost API credits
|
|
76
|
+
--max-calls <n> with --auto, at most n calls (default: 20)
|
|
77
|
+
--save-scenario <p> with --auto, write the steps it ran as a scenario file
|
|
78
|
+
--union-out <path> write every tool the session saw as a menu file, to commit as
|
|
79
|
+
the baseline for diff (tools behind unlocks included)
|
|
64
80
|
|
|
65
81
|
history options (best effort; installs and runs third-party code, so use a container):
|
|
66
82
|
--versions <n|all> number of published versions to inspect (default: 10)
|
|
@@ -74,6 +90,12 @@ history options (best effort; installs and runs third-party code, so use a conta
|
|
|
74
90
|
--keep-installs keep each version's install directory
|
|
75
91
|
--out <dir> where menus and history.json go (default: toolmenu-history/<package>)
|
|
76
92
|
|
|
93
|
+
auth options:
|
|
94
|
+
--port <n> loopback port for the login redirect (default: 33418)
|
|
95
|
+
--scope <scopes> scopes to ask for (default: the server's)
|
|
96
|
+
--client-id <id> a pre-registered client, for servers without dynamic registration
|
|
97
|
+
--client-secret <s> its secret, if it has one
|
|
98
|
+
|
|
77
99
|
Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
|
|
78
100
|
`;
|
|
79
101
|
class UsageError extends Error {
|
|
@@ -96,6 +118,11 @@ export async function main(argv) {
|
|
|
96
118
|
header: { type: 'string', multiple: true },
|
|
97
119
|
env: { type: 'string', multiple: true },
|
|
98
120
|
timeout: { type: 'string' },
|
|
121
|
+
'no-auth': { type: 'boolean' },
|
|
122
|
+
port: { type: 'string' },
|
|
123
|
+
scope: { type: 'string' },
|
|
124
|
+
'client-id': { type: 'string' },
|
|
125
|
+
'client-secret': { type: 'string' },
|
|
99
126
|
processes: { type: 'string' },
|
|
100
127
|
versions: { type: 'string' },
|
|
101
128
|
release: { type: 'string' },
|
|
@@ -111,6 +138,11 @@ export async function main(argv) {
|
|
|
111
138
|
scenario: { type: 'string' },
|
|
112
139
|
plan: { type: 'boolean' },
|
|
113
140
|
init: { type: 'boolean' },
|
|
141
|
+
'union-out': { type: 'string' },
|
|
142
|
+
auto: { type: 'boolean' },
|
|
143
|
+
'open-world': { type: 'boolean' },
|
|
144
|
+
'max-calls': { type: 'string' },
|
|
145
|
+
'save-scenario': { type: 'string' },
|
|
114
146
|
help: { type: 'boolean', short: 'h' },
|
|
115
147
|
version: { type: 'boolean', short: 'v' },
|
|
116
148
|
},
|
|
@@ -124,7 +156,7 @@ export async function main(argv) {
|
|
|
124
156
|
process.stdout.write(HELP);
|
|
125
157
|
return sub || values.help ? 0 : 2;
|
|
126
158
|
}
|
|
127
|
-
if (!['snapshot', 'diff', 'history', 'session'].includes(sub))
|
|
159
|
+
if (!['snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
|
|
128
160
|
throw new UsageError(`Unknown command "${sub}". Try --help.`);
|
|
129
161
|
const format = (values.json ? 'json' : values.format ?? 'text');
|
|
130
162
|
if (!['text', 'json', 'github', 'markdown'].includes(format))
|
|
@@ -135,6 +167,8 @@ export async function main(argv) {
|
|
|
135
167
|
const timeoutMs = values.timeout ? Number(values.timeout) : 30_000;
|
|
136
168
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
|
|
137
169
|
throw new UsageError(`--timeout must be a positive number of milliseconds`);
|
|
170
|
+
if (sub === 'auth')
|
|
171
|
+
return authCommand(rest, values);
|
|
138
172
|
const config = await loadConfig(values.config);
|
|
139
173
|
const processes = values.processes !== undefined ? Number(values.processes) : config.processes ?? 2;
|
|
140
174
|
if (!Number.isInteger(processes) || processes < 1)
|
|
@@ -193,7 +227,7 @@ export async function main(argv) {
|
|
|
193
227
|
const path = values.scenario ?? 'scenario.yml';
|
|
194
228
|
if (existsSync(path))
|
|
195
229
|
throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
|
|
196
|
-
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
230
|
+
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
197
231
|
const conn = await connect(seeded(initTarget, MAIN_SEED), { timeoutMs });
|
|
198
232
|
try {
|
|
199
233
|
const list = await listTools(conn, { timeoutMs });
|
|
@@ -206,6 +240,33 @@ export async function main(argv) {
|
|
|
206
240
|
}
|
|
207
241
|
return 0;
|
|
208
242
|
}
|
|
243
|
+
if (sub === 'session' && values.auto) {
|
|
244
|
+
if (values.scenario)
|
|
245
|
+
throw new UsageError('--auto builds the steps itself; drop --scenario (or drop --auto).');
|
|
246
|
+
const maxCalls = values['max-calls'] ? Number(values['max-calls']) : 20;
|
|
247
|
+
if (!Number.isInteger(maxCalls) || maxCalls < 1)
|
|
248
|
+
throw new UsageError('--max-calls must be a whole number, 1 or more');
|
|
249
|
+
const savePath = values['save-scenario'];
|
|
250
|
+
if (savePath && existsSync(savePath))
|
|
251
|
+
throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
|
|
252
|
+
const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
253
|
+
const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs);
|
|
254
|
+
const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
|
|
255
|
+
if (savePath)
|
|
256
|
+
await writeFile(savePath, scenarioYaml(plan, menu.server.name));
|
|
257
|
+
if (values.plan) {
|
|
258
|
+
process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
|
|
259
|
+
return 0;
|
|
260
|
+
}
|
|
261
|
+
const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto' });
|
|
262
|
+
result.auto = { called: plan.called, skipped: plan.skipped };
|
|
263
|
+
if (values['union-out'])
|
|
264
|
+
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
265
|
+
const output = formatSession(result, format);
|
|
266
|
+
if (output)
|
|
267
|
+
process.stdout.write(output + '\n');
|
|
268
|
+
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
269
|
+
}
|
|
209
270
|
if (sub === 'session') {
|
|
210
271
|
if (!values.scenario)
|
|
211
272
|
throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
|
|
@@ -220,14 +281,16 @@ export async function main(argv) {
|
|
|
220
281
|
process.stdout.write(formatPlan(scenario, values.scenario) + '\n');
|
|
221
282
|
return 0;
|
|
222
283
|
}
|
|
223
|
-
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
284
|
+
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
224
285
|
const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
|
|
286
|
+
if (values['union-out'])
|
|
287
|
+
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
225
288
|
const output = formatSession(result, format);
|
|
226
289
|
if (output)
|
|
227
290
|
process.stdout.write(output + '\n');
|
|
228
291
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
229
292
|
}
|
|
230
|
-
const target = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
293
|
+
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
231
294
|
const routesPath = values.routes ?? config.routes;
|
|
232
295
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
233
296
|
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
@@ -239,7 +302,7 @@ export async function main(argv) {
|
|
|
239
302
|
process.stdout.write(output + '\n');
|
|
240
303
|
return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
241
304
|
}
|
|
242
|
-
function parseTarget(positionals, command, headers, env) {
|
|
305
|
+
function parseTarget(positionals, command, headers, env, noAuth = false) {
|
|
243
306
|
if (command) {
|
|
244
307
|
if (command.length === 0)
|
|
245
308
|
throw new UsageError('Nothing after "--": give the command that starts the server.');
|
|
@@ -254,7 +317,33 @@ function parseTarget(positionals, command, headers, env) {
|
|
|
254
317
|
throw new UsageError(`Unexpected "${extra[0]}". For a stdio server, put the command after "--".`);
|
|
255
318
|
if (!/^https?:\/\//.test(url))
|
|
256
319
|
throw new UsageError(`"${url}" isn't an http(s) URL. For a stdio server, put the command after "--".`);
|
|
257
|
-
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header') };
|
|
320
|
+
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header'), ...(noAuth ? { noAuth } : {}) };
|
|
321
|
+
}
|
|
322
|
+
async function authCommand(args, values) {
|
|
323
|
+
const [action, url, ...extra] = args;
|
|
324
|
+
if (action === 'list') {
|
|
325
|
+
const logins = await listLogins();
|
|
326
|
+
process.stdout.write(logins.length ? logins.map((l) => `${l.serverUrl} (issuer ${l.issuer ?? '?'}, saved ${l.savedAt?.slice(0, 10) ?? '?'})`).join('\n') + '\n' : `No logins. Stored in ${authDir()}.\n`);
|
|
327
|
+
return 0;
|
|
328
|
+
}
|
|
329
|
+
if (action !== 'login' && action !== 'logout')
|
|
330
|
+
throw new UsageError('auth takes login <url>, logout <url> or list.');
|
|
331
|
+
if (!url || extra.length)
|
|
332
|
+
throw new UsageError(`auth ${action} needs one server URL: toolmenu auth ${action} https://example.com/mcp`);
|
|
333
|
+
if (!/^https?:\/\//.test(url))
|
|
334
|
+
throw new UsageError(`"${url}" isn't an http(s) URL.`);
|
|
335
|
+
if (action === 'logout') {
|
|
336
|
+
process.stdout.write((await logout(url)) ? `Logged out of ${url}.\n` : `No login stored for ${url}.\n`);
|
|
337
|
+
return 0;
|
|
338
|
+
}
|
|
339
|
+
const port = values.port ? Number(values.port) : undefined;
|
|
340
|
+
if (port !== undefined && !(Number.isInteger(port) && port > 0 && port < 65536))
|
|
341
|
+
throw new UsageError('--port must be a port number');
|
|
342
|
+
if (values['client-secret'] && !values['client-id'])
|
|
343
|
+
throw new UsageError('--client-secret goes with --client-id');
|
|
344
|
+
const result = await login(url, { port, scope: values.scope, clientId: values['client-id'], clientSecret: values['client-secret'] });
|
|
345
|
+
process.stdout.write(`Logged in to ${result.name ?? url}: ${result.tools} tools. snapshot and session use this login from now on (--no-auth to skip it).\n`);
|
|
346
|
+
return 0;
|
|
258
347
|
}
|
|
259
348
|
function parsePairs(items, separator, flag) {
|
|
260
349
|
const pairs = {};
|
package/dist/connect.d.ts
CHANGED
package/dist/connect.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
|
|
2
2
|
import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
|
|
3
|
+
import { hasLogin, StoredOAuthProvider } from './auth.js';
|
|
3
4
|
import { eraOf } from './menu.js';
|
|
4
5
|
import { VERSION } from './version.js';
|
|
5
6
|
/** Every response the server sent, matched to the request method, as it came off the wire. */
|
|
@@ -48,6 +49,7 @@ export async function connect(target, options = {}) {
|
|
|
48
49
|
const client = new Client({ name: 'toolmenu', version: VERSION }, { versionNegotiation: { mode: 'auto' } });
|
|
49
50
|
let transport;
|
|
50
51
|
let stderrText = '';
|
|
52
|
+
let oauth = false;
|
|
51
53
|
if (target.kind === 'stdio') {
|
|
52
54
|
const stdio = new StdioClientTransport({
|
|
53
55
|
command: target.command,
|
|
@@ -62,8 +64,13 @@ export async function connect(target, options = {}) {
|
|
|
62
64
|
transport = stdio;
|
|
63
65
|
}
|
|
64
66
|
else {
|
|
67
|
+
// A stored OAuth login (toolmenu auth login) is used unless the request brings
|
|
68
|
+
// its own Authorization header, or --no-auth.
|
|
69
|
+
const ownAuth = Object.keys(target.headers ?? {}).some((h) => h.toLowerCase() === 'authorization');
|
|
70
|
+
oauth = !target.noAuth && !ownAuth && hasLogin(target.url);
|
|
65
71
|
transport = new StreamableHTTPClientTransport(new URL(target.url), {
|
|
66
72
|
requestInit: { headers: target.headers ?? {} },
|
|
73
|
+
...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
|
|
67
74
|
});
|
|
68
75
|
}
|
|
69
76
|
try {
|
|
@@ -75,6 +82,9 @@ export async function connect(target, options = {}) {
|
|
|
75
82
|
const stderr = stderrText.trim();
|
|
76
83
|
if (stderr && error instanceof Error)
|
|
77
84
|
error.message += `\nserver stderr:\n${stderr}`;
|
|
85
|
+
if (target.kind === 'http' && !oauth && error instanceof Error && /\b401\b|unauthori[sz]ed/i.test(error.message)) {
|
|
86
|
+
error.message += `\nIf the server uses OAuth, log in once with: toolmenu auth login ${target.url}`;
|
|
87
|
+
}
|
|
78
88
|
await client.close().catch(() => { });
|
|
79
89
|
throw error;
|
|
80
90
|
}
|
|
@@ -90,7 +100,7 @@ export async function connect(target, options = {}) {
|
|
|
90
100
|
server: { name: server.name, version: server.version },
|
|
91
101
|
capabilities: (client.getServerCapabilities() ?? {}),
|
|
92
102
|
usedAuth: target.kind === 'http' &&
|
|
93
|
-
Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h)),
|
|
103
|
+
(oauth || Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h))),
|
|
94
104
|
stderr: () => stderrText,
|
|
95
105
|
close: () => client.close(),
|
|
96
106
|
};
|
package/dist/report.js
CHANGED
|
@@ -197,8 +197,11 @@ export function formatHistory(h, format, outDir) {
|
|
|
197
197
|
return lines.join('\n');
|
|
198
198
|
}
|
|
199
199
|
export function formatSession(s, format) {
|
|
200
|
-
if (format === 'json')
|
|
201
|
-
|
|
200
|
+
if (format === 'json') {
|
|
201
|
+
// The union menu itself goes to --union-out; the report says how big it is.
|
|
202
|
+
const { union, ...rest } = s;
|
|
203
|
+
return JSON.stringify({ command: 'session', ...rest, union: { tools: union.tools.length, totalTokens: union.totalTokens }, counts: counts(s.findings) }, null, 2);
|
|
204
|
+
}
|
|
202
205
|
if (format === 'github') {
|
|
203
206
|
return githubLines(s.findings.map((f) => ({ ...f, message: f.step ? `step ${f.step} (${s.steps[f.step - 1]?.label}): ${f.message}` : f.message }))).join('\n');
|
|
204
207
|
}
|
|
@@ -208,6 +211,7 @@ export function formatSession(s, format) {
|
|
|
208
211
|
'',
|
|
209
212
|
`\`${s.server.name ?? 'server'}\` ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport} · ${s.baseline.tools} → ${s.final.tools} tools · ~${s.baseline.tokens.toLocaleString('en-US')} → ~${s.final.tokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
210
213
|
'',
|
|
214
|
+
...(autoLine(s) ? [autoLine(s), ''] : []),
|
|
211
215
|
'| Step | Menu | list_changed | Scope |',
|
|
212
216
|
'|---|---|---|---|',
|
|
213
217
|
...s.steps.map((st) => `| ${st.index}. ${mdCell(st.label)} | ${st.status !== 'ok' ? st.status : st.changed ? `changed (${st.tools} tools)` : 'no change'} | ${st.changed ? (st.listChanged ? 'received' : '**missing**') : ''} | ${st.scope ?? ''} |`),
|
|
@@ -221,6 +225,7 @@ export function formatSession(s, format) {
|
|
|
221
225
|
`toolmenu session ${s.scenario}`,
|
|
222
226
|
` ${s.server.name ?? 'server'} ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport}`,
|
|
223
227
|
` baseline: ${plural(s.baseline.tools, 'tool')} · ${tok(s.baseline.tokens)} tokens (estimate) · listening for list_changed: ${s.listening ? 'yes' : 'no'}`,
|
|
228
|
+
...(autoLine(s) ? [` ${autoLine(s)}`] : []),
|
|
224
229
|
` fresh-${s.transport === 'stdio' ? 'process' : 'connection'} check: ${s.connectionCheck === undefined ? 'not checked' : s.connectionCheck === 'same' ? 'the same menu' : 'a DIFFERENT menu'}`,
|
|
225
230
|
'',
|
|
226
231
|
];
|
|
@@ -264,6 +269,21 @@ export function formatPlan(scenario, name) {
|
|
|
264
269
|
lines.push('', 'After every step the menu is listed and compared with the one before. Nothing was run.');
|
|
265
270
|
return lines.join('\n');
|
|
266
271
|
}
|
|
272
|
+
/** One line on what --auto called and what it left out, and how to reach more. */
|
|
273
|
+
function autoLine(s) {
|
|
274
|
+
if (!s.auto)
|
|
275
|
+
return undefined;
|
|
276
|
+
const by = (r) => s.auto.skipped.filter((x) => x.reason === r);
|
|
277
|
+
const needs = by('needs values');
|
|
278
|
+
const params = [...new Set(needs.flatMap((x) => x.missing ?? []))];
|
|
279
|
+
const parts = [
|
|
280
|
+
by('not read-only').length && `${by('not read-only').length} not read-only`,
|
|
281
|
+
by('open world').length && `${by('open world').length} marked openWorldHint: true (--open-world to call them)`,
|
|
282
|
+
needs.length && `${needs.length} need values the schema doesn't give (${params.slice(0, 6).join(', ')}${params.length > 6 ? ', …' : ''}: write a scenario, or --save-scenario and fill them in)`,
|
|
283
|
+
by('over the call budget').length && `${by('over the call budget').length} over --max-calls`,
|
|
284
|
+
].filter(Boolean);
|
|
285
|
+
return `auto: called ${s.auto.called.length} tool${s.auto.called.length === 1 ? '' : 's'}${parts.length ? ` · skipped ${parts.join(', ')}` : ''}`;
|
|
286
|
+
}
|
|
267
287
|
function mdCell(text) {
|
|
268
288
|
return text.replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
|
269
289
|
}
|
package/dist/session.d.ts
CHANGED
|
@@ -54,6 +54,16 @@ export interface SessionResult {
|
|
|
54
54
|
connectionCheck?: 'same' | 'different';
|
|
55
55
|
steps: StepRecord[];
|
|
56
56
|
findings: Finding[];
|
|
57
|
+
/**
|
|
58
|
+
* Every tool the session saw, in the order first seen, each as last seen: the
|
|
59
|
+
* menu to diff between releases when tools only appear after an unlock.
|
|
60
|
+
*/
|
|
61
|
+
union: Menu;
|
|
62
|
+
/** With --auto: which tools were called, and which weren't and why. */
|
|
63
|
+
auto?: {
|
|
64
|
+
called: string[];
|
|
65
|
+
skipped: import('./auto.js').AutoPlan['skipped'];
|
|
66
|
+
};
|
|
57
67
|
}
|
|
58
68
|
export interface SessionOptions {
|
|
59
69
|
timeoutMs?: number;
|
package/dist/session.js
CHANGED
|
@@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises';
|
|
|
2
2
|
import { parse } from 'yaml';
|
|
3
3
|
import { cacheBreak, compareMenus } from './compare.js';
|
|
4
4
|
import { connect, listTools } from './connect.js';
|
|
5
|
-
import { buildMenu } from './menu.js';
|
|
5
|
+
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
6
|
import { isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
7
|
import { varianceFinding } from './rules/determinism.js';
|
|
8
8
|
import { classifyFailure, FAILURE_LABELS, SETUP_FAILURES } from './failures.js';
|
|
@@ -86,6 +86,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
86
86
|
}
|
|
87
87
|
const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
88
88
|
let current = await menuOf(conn);
|
|
89
|
+
// Map keeps first-insertion order; set() on a known name updates it in place.
|
|
90
|
+
const seen = new Map();
|
|
91
|
+
const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
|
|
92
|
+
see(current);
|
|
89
93
|
const baseline = current;
|
|
90
94
|
// Fresh processes or connections, same credentials, must see the same menu.
|
|
91
95
|
let connectionCheck;
|
|
@@ -210,6 +214,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
210
214
|
raw.push(...scopeFindings(record.scope, modern, index));
|
|
211
215
|
}
|
|
212
216
|
current = next;
|
|
217
|
+
see(next);
|
|
213
218
|
}
|
|
214
219
|
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
215
220
|
record.tools = current.tools.length;
|
|
@@ -227,6 +232,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
227
232
|
connectionCheck,
|
|
228
233
|
steps,
|
|
229
234
|
findings: settle(mergeSideEffects(raw), options),
|
|
235
|
+
union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
|
|
230
236
|
};
|
|
231
237
|
}
|
|
232
238
|
finally {
|