sunpeak 0.19.12 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/commands/inspect.mjs +321 -6
- package/bin/commands/test-init.mjs +100 -39
- package/bin/commands/test.mjs +6 -0
- package/bin/lib/inspect/inspect-config.mjs +16 -1
- package/bin/lib/inspect/inspect-server.d.mts +32 -0
- package/bin/lib/inspect/inspect-server.mjs +11 -0
- package/bin/lib/resolve-bin.mjs +39 -0
- package/bin/lib/test/base-config.mjs +3 -2
- package/bin/lib/test/matchers.mjs +2 -2
- package/bin/lib/test/test-config.mjs +18 -7
- package/bin/lib/test/test-fixtures.d.mts +52 -92
- package/bin/lib/test/test-fixtures.mjs +174 -147
- package/dist/chatgpt/index.cjs +1 -1
- package/dist/chatgpt/index.js +1 -1
- package/dist/claude/index.cjs +1 -1
- package/dist/claude/index.js +1 -1
- package/dist/host/chatgpt/index.cjs +1 -1
- package/dist/host/chatgpt/index.js +1 -1
- package/dist/index.cjs +4 -4
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/inspector/index.cjs +1 -1
- package/dist/inspector/index.js +1 -1
- package/dist/{inspector-D5DckQuU.js → inspector-BBDa5yCm.js} +57 -23
- package/dist/inspector-BBDa5yCm.js.map +1 -0
- package/dist/{inspector-jY9O18z9.cjs → inspector-DAA1Wiyh.cjs} +58 -24
- package/dist/inspector-DAA1Wiyh.cjs.map +1 -0
- package/dist/lib/discovery-cli.cjs +1 -1
- package/dist/mcp/index.cjs +22 -25
- package/dist/mcp/index.cjs.map +1 -1
- package/dist/mcp/index.js +19 -22
- package/dist/mcp/index.js.map +1 -1
- package/dist/{use-app-Bfargfa3.js → use-app-Cr0auUa1.js} +2 -2
- package/dist/{use-app-Bfargfa3.js.map → use-app-Cr0auUa1.js.map} +1 -1
- package/dist/{use-app-CbsBEmwv.cjs → use-app-DPkj5Jp_.cjs} +2 -2
- package/dist/{use-app-CbsBEmwv.cjs.map → use-app-DPkj5Jp_.cjs.map} +1 -1
- package/package.json +17 -11
- package/template/dist/albums/albums.html +4 -4
- package/template/dist/albums/albums.json +1 -1
- package/template/dist/carousel/carousel.html +4 -4
- package/template/dist/carousel/carousel.json +1 -1
- package/template/dist/map/map.html +6 -6
- package/template/dist/map/map.json +1 -1
- package/template/dist/review/review.html +4 -4
- package/template/dist/review/review.json +1 -1
- package/template/node_modules/.bin/vite +2 -2
- package/template/node_modules/.bin/vitest +2 -2
- package/template/node_modules/.vite/deps/_metadata.json +4 -4
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps.js +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_app-bridge.js +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_app-bridge.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_react.js +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/@testing-library_react.js +4 -4
- package/template/node_modules/.vite-mcp/deps/@testing-library_react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/_metadata.json +33 -33
- package/template/node_modules/.vite-mcp/deps/{client-CU1wWud4.js → client-B_5CX--u.js} +7 -7
- package/template/node_modules/.vite-mcp/deps/{client-CU1wWud4.js.map → client-B_5CX--u.js.map} +1 -1
- package/template/node_modules/.vite-mcp/deps/embla-carousel-react.js +1 -1
- package/template/node_modules/.vite-mcp/deps/embla-carousel-react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react-dom.js +3 -3
- package/template/node_modules/.vite-mcp/deps/react-dom.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react-dom_client.js +1 -1
- package/template/node_modules/.vite-mcp/deps/react.js +3 -3
- package/template/node_modules/.vite-mcp/deps/react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react_jsx-dev-runtime.js +2 -2
- package/template/node_modules/.vite-mcp/deps/react_jsx-dev-runtime.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react_jsx-runtime.js +2 -2
- package/template/node_modules/.vite-mcp/deps/react_jsx-runtime.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/vitest.js +1024 -622
- package/template/node_modules/.vite-mcp/deps/vitest.js.map +1 -1
- package/template/package.json +6 -6
- package/template/tests/e2e/albums.spec.ts +24 -52
- package/template/tests/e2e/carousel.spec.ts +36 -58
- package/template/tests/e2e/map.spec.ts +35 -56
- package/template/tests/e2e/review.spec.ts +56 -85
- package/template/tests/e2e/visual.spec.ts +14 -12
- package/dist/inspector-D5DckQuU.js.map +0 -1
- package/dist/inspector-jY9O18z9.cjs.map +0 -1
package/README.md
CHANGED
|
@@ -53,8 +53,8 @@ Automatically test any MCP server against replicated ChatGPT and Claude runtimes
|
|
|
53
53
|
```ts
|
|
54
54
|
import { test, expect } from 'sunpeak/test';
|
|
55
55
|
|
|
56
|
-
test('review tool renders title', async ({
|
|
57
|
-
const result = await
|
|
56
|
+
test('review tool renders title', async ({ inspector }) => {
|
|
57
|
+
const result = await inspector.renderTool('review-diff');
|
|
58
58
|
const app = result.app();
|
|
59
59
|
await expect(app.locator('h1:has-text("Refactor")')).toBeVisible();
|
|
60
60
|
});
|
package/bin/commands/inspect.mjs
CHANGED
|
@@ -18,6 +18,7 @@ import * as path from 'path';
|
|
|
18
18
|
const { existsSync, readdirSync, readFileSync } = fs;
|
|
19
19
|
const { join, resolve, dirname } = path;
|
|
20
20
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
21
|
+
import { createServer as createHttpServer } from 'http';
|
|
21
22
|
import { getPort } from '../lib/get-port.mjs';
|
|
22
23
|
import { startSandboxServer } from '../lib/sandbox-server.mjs';
|
|
23
24
|
import { getDevOverlayScript } from '../lib/dev-overlay.mjs';
|
|
@@ -35,6 +36,8 @@ function parseArgs(args) {
|
|
|
35
36
|
simulations: undefined,
|
|
36
37
|
port: undefined,
|
|
37
38
|
name: undefined,
|
|
39
|
+
env: undefined,
|
|
40
|
+
cwd: undefined,
|
|
38
41
|
};
|
|
39
42
|
|
|
40
43
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -47,6 +50,16 @@ function parseArgs(args) {
|
|
|
47
50
|
opts.port = Number(args[++i]);
|
|
48
51
|
} else if (arg === '--name' && i + 1 < args.length) {
|
|
49
52
|
opts.name = args[++i];
|
|
53
|
+
} else if (arg === '--env' && i + 1 < args.length) {
|
|
54
|
+
// Repeatable: --env KEY=VALUE --env KEY2=VALUE2
|
|
55
|
+
const pair = args[++i];
|
|
56
|
+
const eqIdx = pair.indexOf('=');
|
|
57
|
+
if (eqIdx > 0) {
|
|
58
|
+
opts.env = opts.env || {};
|
|
59
|
+
opts.env[pair.slice(0, eqIdx)] = pair.slice(eqIdx + 1);
|
|
60
|
+
}
|
|
61
|
+
} else if (arg === '--cwd' && i + 1 < args.length) {
|
|
62
|
+
opts.cwd = args[++i];
|
|
50
63
|
} else if (arg === '--help' || arg === '-h') {
|
|
51
64
|
printHelp();
|
|
52
65
|
process.exit(0);
|
|
@@ -68,11 +81,14 @@ Options:
|
|
|
68
81
|
--simulations <dir> Simulation JSON directory (opt-in, no default)
|
|
69
82
|
--port, -p <number> Dev server port (default: 3000)
|
|
70
83
|
--name <string> App name in inspector chrome
|
|
84
|
+
--env <KEY=VALUE> Environment variable for stdio servers (repeatable)
|
|
85
|
+
--cwd <path> Working directory for stdio servers
|
|
71
86
|
--help, -h Show this help
|
|
72
87
|
|
|
73
88
|
Examples:
|
|
74
89
|
sunpeak inspect --server http://localhost:8000/mcp
|
|
75
90
|
sunpeak inspect --server "python my_server.py"
|
|
91
|
+
sunpeak inspect --server "python server.py" --env API_KEY=sk-123 --cwd ./backend
|
|
76
92
|
sunpeak inspect --server http://localhost:8000/mcp --simulations tests/simulations
|
|
77
93
|
`);
|
|
78
94
|
}
|
|
@@ -160,11 +176,219 @@ function createInMemoryOAuthProvider(redirectUrl, opts = {}) {
|
|
|
160
176
|
};
|
|
161
177
|
}
|
|
162
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Negotiate OAuth with an MCP server and return an authenticated provider.
|
|
181
|
+
*
|
|
182
|
+
* Handles two cases:
|
|
183
|
+
* 1. Anonymous/auto-approved OAuth: the authorization endpoint redirects
|
|
184
|
+
* immediately back with a code (no user interaction needed).
|
|
185
|
+
* 2. Interactive OAuth: opens the authorization URL in the user's browser
|
|
186
|
+
* and waits for the callback.
|
|
187
|
+
*
|
|
188
|
+
* @param {string} serverUrl - The MCP server URL
|
|
189
|
+
* @returns {Promise<import('@modelcontextprotocol/sdk/client/auth.js').OAuthClientProvider>}
|
|
190
|
+
*/
|
|
191
|
+
async function negotiateOAuth(serverUrl) {
|
|
192
|
+
const { auth } = await import('@modelcontextprotocol/sdk/client/auth.js');
|
|
193
|
+
|
|
194
|
+
// Start a temporary callback server for receiving the OAuth code.
|
|
195
|
+
const callbackPort = await getPort(24681);
|
|
196
|
+
const callbackUrl = `http://localhost:${callbackPort}/oauth/callback`;
|
|
197
|
+
|
|
198
|
+
const oauthState = createInMemoryOAuthProvider(callbackUrl);
|
|
199
|
+
const { provider } = oauthState;
|
|
200
|
+
|
|
201
|
+
// First call to auth() — discovers metadata, registers client, and either
|
|
202
|
+
// returns AUTHORIZED (client_credentials) or REDIRECT (authorization_code).
|
|
203
|
+
const result = await auth(provider, { serverUrl: new URL(serverUrl) });
|
|
204
|
+
|
|
205
|
+
if (result === 'AUTHORIZED') {
|
|
206
|
+
return provider;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// result === 'REDIRECT': we need to follow the authorization URL.
|
|
210
|
+
const authUrl = oauthState.getAuthUrl();
|
|
211
|
+
if (!authUrl) {
|
|
212
|
+
throw new Error('OAuth flow returned REDIRECT but no authorization URL was captured');
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// Try the anonymous/auto-approved path first: follow the authorization URL
|
|
216
|
+
// without a browser and see if it immediately redirects with a code.
|
|
217
|
+
const code = await tryAnonymousOAuth(authUrl.toString(), callbackUrl);
|
|
218
|
+
if (code) {
|
|
219
|
+
// Complete the flow with the authorization code.
|
|
220
|
+
const tokenResult = await auth(provider, {
|
|
221
|
+
serverUrl: new URL(serverUrl),
|
|
222
|
+
authorizationCode: code,
|
|
223
|
+
});
|
|
224
|
+
if (tokenResult === 'AUTHORIZED') {
|
|
225
|
+
return provider;
|
|
226
|
+
}
|
|
227
|
+
throw new Error('OAuth token exchange failed after anonymous authorization');
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// Anonymous path didn't work — this server requires interactive login.
|
|
231
|
+
// Start a callback server and open the auth URL in the user's browser.
|
|
232
|
+
const interactiveCode = await waitForInteractiveOAuth(
|
|
233
|
+
authUrl.toString(),
|
|
234
|
+
callbackUrl,
|
|
235
|
+
callbackPort
|
|
236
|
+
);
|
|
237
|
+
|
|
238
|
+
const tokenResult = await auth(provider, {
|
|
239
|
+
serverUrl: new URL(serverUrl),
|
|
240
|
+
authorizationCode: interactiveCode,
|
|
241
|
+
});
|
|
242
|
+
if (tokenResult === 'AUTHORIZED') {
|
|
243
|
+
return provider;
|
|
244
|
+
}
|
|
245
|
+
throw new Error('OAuth token exchange failed after interactive authorization');
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Try to complete OAuth without user interaction by following redirects.
|
|
250
|
+
* Returns the authorization code if the server auto-approves, or null if
|
|
251
|
+
* the server requires interactive login (returns an HTML page).
|
|
252
|
+
*
|
|
253
|
+
* @param {string} authUrl - The authorization URL
|
|
254
|
+
* @param {string} callbackUrl - The expected callback URL prefix
|
|
255
|
+
* @returns {Promise<string | null>}
|
|
256
|
+
*/
|
|
257
|
+
async function tryAnonymousOAuth(authUrl, callbackUrl) {
|
|
258
|
+
// Follow redirects manually to detect when the server redirects back
|
|
259
|
+
// to our callback URL with a code parameter.
|
|
260
|
+
let url = authUrl;
|
|
261
|
+
const maxRedirects = 10;
|
|
262
|
+
for (let i = 0; i < maxRedirects; i++) {
|
|
263
|
+
const response = await fetch(url, { redirect: 'manual' });
|
|
264
|
+
const location = response.headers.get('location');
|
|
265
|
+
|
|
266
|
+
if (!location) {
|
|
267
|
+
// No redirect — server returned a page (login form). Not auto-approved.
|
|
268
|
+
// Drain the response body to free the socket.
|
|
269
|
+
await response.text().catch(() => {});
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Resolve relative redirects.
|
|
274
|
+
const resolved = new URL(location, url).toString();
|
|
275
|
+
|
|
276
|
+
// Check if the redirect goes to our callback URL.
|
|
277
|
+
if (resolved.startsWith(callbackUrl)) {
|
|
278
|
+
const params = new URL(resolved).searchParams;
|
|
279
|
+
const code = params.get('code');
|
|
280
|
+
if (code) return code;
|
|
281
|
+
const error = params.get('error');
|
|
282
|
+
if (error) {
|
|
283
|
+
throw new Error(`OAuth authorization failed: ${error} — ${params.get('error_description') || ''}`);
|
|
284
|
+
}
|
|
285
|
+
return null;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
url = resolved;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Wait for the user to complete an interactive OAuth flow in their browser.
|
|
296
|
+
* Starts a temporary HTTP server to receive the callback, opens the auth URL,
|
|
297
|
+
* and resolves with the authorization code.
|
|
298
|
+
*
|
|
299
|
+
* @param {string} authUrl - The authorization URL to open in the browser
|
|
300
|
+
* @param {string} callbackUrl - Our callback URL
|
|
301
|
+
* @param {number} callbackPort - Port for the callback server
|
|
302
|
+
* @returns {Promise<string>}
|
|
303
|
+
*/
|
|
304
|
+
async function waitForInteractiveOAuth(authUrl, callbackUrl, callbackPort) {
|
|
305
|
+
return new Promise((resolve, reject) => {
|
|
306
|
+
let settled = false;
|
|
307
|
+
const settle = (fn, value) => {
|
|
308
|
+
if (settled) return;
|
|
309
|
+
settled = true;
|
|
310
|
+
clearTimeout(timer);
|
|
311
|
+
server.close();
|
|
312
|
+
fn(value);
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
const server = createHttpServer((req, res) => {
|
|
316
|
+
const reqUrl = new URL(req.url, callbackUrl);
|
|
317
|
+
if (!reqUrl.pathname.startsWith('/oauth/callback')) {
|
|
318
|
+
res.writeHead(404);
|
|
319
|
+
res.end('Not found');
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
const code = reqUrl.searchParams.get('code');
|
|
324
|
+
const error = reqUrl.searchParams.get('error');
|
|
325
|
+
|
|
326
|
+
// Serve a simple page that tells the user they can close the tab.
|
|
327
|
+
const escHtml = (s) => s.replace(/[<>&"']/g, (c) =>
|
|
328
|
+
({ '<': '<', '>': '>', '&': '&', '"': '"', "'": ''' })[c]
|
|
329
|
+
);
|
|
330
|
+
const message = code
|
|
331
|
+
? 'Authorization complete. You can close this tab.'
|
|
332
|
+
: `Authorization failed: ${escHtml(error || 'unknown error')}`;
|
|
333
|
+
res.writeHead(200, { 'Content-Type': 'text/html' });
|
|
334
|
+
res.end(`<!DOCTYPE html><html><body><p>${message}</p></body></html>`);
|
|
335
|
+
|
|
336
|
+
if (code) {
|
|
337
|
+
settle(resolve, code);
|
|
338
|
+
} else {
|
|
339
|
+
settle(reject, new Error(`OAuth authorization failed: ${error || 'unknown error'}`));
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
server.on('error', (err) => {
|
|
344
|
+
settle(reject, new Error(`OAuth callback server failed: ${err.message}`));
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
server.listen(callbackPort, async () => {
|
|
348
|
+
console.log('Opening browser for OAuth authorization...');
|
|
349
|
+
// Use execFile with array args to avoid shell injection from the auth URL.
|
|
350
|
+
const { execFile } = await import('child_process');
|
|
351
|
+
const cmd = process.platform === 'darwin' ? 'open' :
|
|
352
|
+
process.platform === 'win32' ? 'start' : 'xdg-open';
|
|
353
|
+
execFile(cmd, [authUrl], (err) => {
|
|
354
|
+
if (err) console.error(`Failed to open browser: ${err.message}`);
|
|
355
|
+
});
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
// Timeout after 2 minutes.
|
|
359
|
+
const timer = setTimeout(() => {
|
|
360
|
+
settle(reject, new Error('OAuth authorization timed out (2 minutes)'));
|
|
361
|
+
}, 120_000);
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* Detect if an error from createMcpConnection is an auth error (401/Unauthorized).
|
|
367
|
+
* @param {Error} err
|
|
368
|
+
* @returns {boolean}
|
|
369
|
+
*/
|
|
370
|
+
function isAuthError(err) {
|
|
371
|
+
// The MCP SDK throws UnauthorizedError for auth failures.
|
|
372
|
+
if (err.constructor?.name === 'UnauthorizedError') return true;
|
|
373
|
+
|
|
374
|
+
// StreamableHTTPError includes a status code in its message.
|
|
375
|
+
// Check for the specific "401" HTTP status pattern, not substring matches.
|
|
376
|
+
const msg = err.message || '';
|
|
377
|
+
if (msg.includes('invalid_token')) return true;
|
|
378
|
+
|
|
379
|
+
// Connection errors (ECONNREFUSED, ETIMEDOUT, etc.) are never auth errors.
|
|
380
|
+
if (msg.includes('ECONNREFUSED') || msg.includes('ETIMEDOUT') || msg.includes('ENOTFOUND')) {
|
|
381
|
+
return false;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
|
|
163
387
|
/**
|
|
164
388
|
* Create an MCP client connection.
|
|
165
389
|
* @param {string} serverArg - URL or command string
|
|
166
|
-
* @param {{ type?: 'none' | 'bearer' | 'oauth', bearerToken?: string, authProvider?: import('@modelcontextprotocol/sdk/client/auth.js').OAuthClientProvider }} [authConfig]
|
|
167
|
-
* @returns {Promise<{ client: import('@modelcontextprotocol/sdk/client/index.js').Client, transport: import('@modelcontextprotocol/sdk/types.js').Transport }>}
|
|
390
|
+
* @param {{ type?: 'none' | 'bearer' | 'oauth', bearerToken?: string, authProvider?: import('@modelcontextprotocol/sdk/client/auth.js').OAuthClientProvider, env?: Record<string, string>, cwd?: string }} [authConfig]
|
|
391
|
+
* @returns {Promise<{ client: import('@modelcontextprotocol/sdk/client/index.js').Client, transport: import('@modelcontextprotocol/sdk/types.js').Transport, stderrOutput?: string[] }>}
|
|
168
392
|
*/
|
|
169
393
|
async function createMcpConnection(serverArg, authConfig) {
|
|
170
394
|
const { Client } = await import('@modelcontextprotocol/sdk/client/index.js');
|
|
@@ -197,9 +421,47 @@ async function createMcpConnection(serverArg, authConfig) {
|
|
|
197
421
|
const { StdioClientTransport } = await import(
|
|
198
422
|
'@modelcontextprotocol/sdk/client/stdio.js'
|
|
199
423
|
);
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
424
|
+
|
|
425
|
+
const transportOpts = {
|
|
426
|
+
command,
|
|
427
|
+
args: cmdArgs,
|
|
428
|
+
stderr: 'pipe',
|
|
429
|
+
...(authConfig?.env ? { env: { ...process.env, ...authConfig.env } } : {}),
|
|
430
|
+
...(authConfig?.cwd ? { cwd: authConfig.cwd } : {}),
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
const transport = new StdioClientTransport(transportOpts);
|
|
434
|
+
|
|
435
|
+
// Buffer stderr lines so we can surface them on connection failure,
|
|
436
|
+
// while still printing them in real time (preserving the SDK's default
|
|
437
|
+
// 'inherit' behavior for interactive use).
|
|
438
|
+
const stderrOutput = [];
|
|
439
|
+
const MAX_STDERR_LINES = 50;
|
|
440
|
+
if (transport.stderr) {
|
|
441
|
+
transport.stderr.on('data', (chunk) => {
|
|
442
|
+
process.stderr.write(chunk);
|
|
443
|
+
const lines = chunk.toString().split('\n');
|
|
444
|
+
for (const line of lines) {
|
|
445
|
+
if (line) {
|
|
446
|
+
stderrOutput.push(line);
|
|
447
|
+
if (stderrOutput.length > MAX_STDERR_LINES) {
|
|
448
|
+
stderrOutput.shift();
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
try {
|
|
456
|
+
await client.connect(transport);
|
|
457
|
+
} catch (err) {
|
|
458
|
+
// Attach captured stderr so callers can surface it for diagnostics.
|
|
459
|
+
err._stderrOutput = stderrOutput;
|
|
460
|
+
// Clean up the spawned process so it doesn't linger.
|
|
461
|
+
try { await transport.close(); } catch { /* best-effort */ }
|
|
462
|
+
throw err;
|
|
463
|
+
}
|
|
464
|
+
return { client, transport, stderrOutput };
|
|
203
465
|
}
|
|
204
466
|
}
|
|
205
467
|
|
|
@@ -405,6 +667,20 @@ function sunpeakInspectEndpointsPlugin(getClient, setClient, pluginOpts = {}) {
|
|
|
405
667
|
}
|
|
406
668
|
});
|
|
407
669
|
|
|
670
|
+
// List resources from connected server
|
|
671
|
+
server.middlewares.use('/__sunpeak/list-resources', async (_req, res) => {
|
|
672
|
+
try {
|
|
673
|
+
const client = getClient();
|
|
674
|
+
const result = await client.listResources();
|
|
675
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
676
|
+
res.end(JSON.stringify(result));
|
|
677
|
+
} catch (err) {
|
|
678
|
+
// Server may not support resources — return empty list
|
|
679
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
680
|
+
res.end(JSON.stringify({ resources: [] }));
|
|
681
|
+
}
|
|
682
|
+
});
|
|
683
|
+
|
|
408
684
|
// Call tool on connected server
|
|
409
685
|
server.middlewares.use('/__sunpeak/call-tool', async (req, res) => {
|
|
410
686
|
if (req.method !== 'POST') {
|
|
@@ -911,6 +1187,8 @@ function readRequestBody(req) {
|
|
|
911
1187
|
* @param {Record<string, string>} [opts.resolveAlias] - Vite resolve aliases (e.g., to map sunpeak imports to source)
|
|
912
1188
|
* @param {object[]} [opts.vitePlugins] - Additional Vite plugins (e.g., Tailwind for source CSS)
|
|
913
1189
|
* @param {object} [opts.viteCssConfig] - Vite css config override (e.g., lightningcss customAtRules)
|
|
1190
|
+
* @param {Record<string, string>} [opts.env] - Extra environment variables for stdio server processes
|
|
1191
|
+
* @param {string} [opts.cwd] - Working directory for stdio server processes
|
|
914
1192
|
*/
|
|
915
1193
|
export async function inspectServer(opts) {
|
|
916
1194
|
const {
|
|
@@ -928,6 +1206,8 @@ export async function inspectServer(opts) {
|
|
|
928
1206
|
resolveAlias,
|
|
929
1207
|
vitePlugins: extraVitePlugins = [],
|
|
930
1208
|
viteCssConfig,
|
|
1209
|
+
env: serverEnv,
|
|
1210
|
+
cwd: serverCwd,
|
|
931
1211
|
} = opts;
|
|
932
1212
|
|
|
933
1213
|
// Load favicon from sunpeak package for the inspector UI.
|
|
@@ -948,14 +1228,47 @@ export async function inspectServer(opts) {
|
|
|
948
1228
|
|
|
949
1229
|
// Connect to the MCP server (with retry for local servers that may still be starting)
|
|
950
1230
|
let mcpConnection;
|
|
1231
|
+
let lastStderrOutput = [];
|
|
951
1232
|
const maxRetries = 5;
|
|
1233
|
+
const connectionOpts = {};
|
|
1234
|
+
if (serverEnv) connectionOpts.env = serverEnv;
|
|
1235
|
+
if (serverCwd) connectionOpts.cwd = serverCwd;
|
|
952
1236
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
953
1237
|
try {
|
|
954
|
-
mcpConnection = await createMcpConnection(serverArg);
|
|
1238
|
+
mcpConnection = await createMcpConnection(serverArg, connectionOpts);
|
|
955
1239
|
break;
|
|
956
1240
|
} catch (err) {
|
|
1241
|
+
// Capture stderr from the failed connection attempt for diagnostics.
|
|
1242
|
+
if (err._stderrOutput?.length) {
|
|
1243
|
+
lastStderrOutput = err._stderrOutput;
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
// If the server requires OAuth, negotiate it and retry once.
|
|
1247
|
+
if (isAuthError(err) && serverArg.startsWith('http')) {
|
|
1248
|
+
console.log('Server requires authentication. Negotiating OAuth...');
|
|
1249
|
+
try {
|
|
1250
|
+
const authProvider = await negotiateOAuth(serverArg);
|
|
1251
|
+
console.log('OAuth authorized. Reconnecting...');
|
|
1252
|
+
mcpConnection = await createMcpConnection(serverArg, {
|
|
1253
|
+
...connectionOpts,
|
|
1254
|
+
type: 'oauth',
|
|
1255
|
+
authProvider,
|
|
1256
|
+
});
|
|
1257
|
+
break;
|
|
1258
|
+
} catch (oauthErr) {
|
|
1259
|
+
console.error(`OAuth negotiation failed: ${oauthErr.message}`);
|
|
1260
|
+
process.exit(1);
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
|
|
957
1264
|
if (attempt === maxRetries) {
|
|
958
1265
|
console.error(`Failed to connect to MCP server: ${err.message}`);
|
|
1266
|
+
if (lastStderrOutput.length) {
|
|
1267
|
+
console.error('\nServer stderr output:');
|
|
1268
|
+
for (const line of lastStderrOutput) {
|
|
1269
|
+
console.error(` ${line}`);
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
959
1272
|
process.exit(1);
|
|
960
1273
|
}
|
|
961
1274
|
console.log(`Connection attempt ${attempt}/${maxRetries} failed, retrying...`);
|
|
@@ -1195,5 +1508,7 @@ export async function inspect(args) {
|
|
|
1195
1508
|
simulationsDir,
|
|
1196
1509
|
port: opts.port,
|
|
1197
1510
|
name: opts.name,
|
|
1511
|
+
env: opts.env,
|
|
1512
|
+
cwd: opts.cwd,
|
|
1198
1513
|
});
|
|
1199
1514
|
}
|
|
@@ -1,10 +1,23 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
|
|
2
2
|
import { execSync } from 'child_process';
|
|
3
3
|
import { join, dirname } from 'path';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
4
5
|
import * as p from '@clack/prompts';
|
|
5
6
|
import { EVAL_PROVIDERS, generateModelLines } from '../lib/eval/eval-providers.mjs';
|
|
6
7
|
import { detectPackageManager } from '../utils.mjs';
|
|
7
8
|
|
|
9
|
+
/** Read the current sunpeak package version for pinning in scaffolded configs. */
|
|
10
|
+
function getSunpeakVersion() {
|
|
11
|
+
try {
|
|
12
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
const pkgPath = join(__dirname, '..', '..', 'package.json');
|
|
14
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
|
|
15
|
+
return pkg.version ? `^${pkg.version}` : 'latest';
|
|
16
|
+
} catch {
|
|
17
|
+
return 'latest';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
8
21
|
/**
|
|
9
22
|
* Default dependencies (real implementations).
|
|
10
23
|
* Override in tests via the `deps` parameter.
|
|
@@ -49,7 +62,7 @@ export const defaultDeps = {
|
|
|
49
62
|
*
|
|
50
63
|
* Scaffolds all 5 test types:
|
|
51
64
|
* 1. E2E tests — Playwright-based inspector tests (mcp fixture)
|
|
52
|
-
* 2. Visual regression — Screenshot comparison via
|
|
65
|
+
* 2. Visual regression — Screenshot comparison via result.screenshot()
|
|
53
66
|
* 3. Live tests — Test against real ChatGPT/Claude hosts
|
|
54
67
|
* 4. Evals — Multi-model tool calling reliability tests
|
|
55
68
|
* 5. Unit tests — Direct tool handler tests (JS/TS projects only)
|
|
@@ -212,11 +225,32 @@ async function getServerConfig(cliServer, d) {
|
|
|
212
225
|
|
|
213
226
|
function generateServerConfigBlock(server, relativeTo = '.') {
|
|
214
227
|
if (server.type === 'later') {
|
|
215
|
-
return ` // TODO: Configure your MCP server connection
|
|
228
|
+
return ` // TODO: Configure your MCP server connection before running tests.
|
|
229
|
+
// Uncomment one of the options below:
|
|
230
|
+
//
|
|
231
|
+
// HTTP server (Python FastAPI, Go, etc.):
|
|
232
|
+
// server: { url: 'http://localhost:8000/mcp' },
|
|
233
|
+
//
|
|
234
|
+
// Python (uv):
|
|
235
|
+
// server: { command: 'uv', args: ['run', 'python', 'server.py'] },
|
|
236
|
+
//
|
|
237
|
+
// Python (venv):
|
|
238
|
+
// server: { command: '.venv/bin/python', args: ['server.py'] },
|
|
239
|
+
//
|
|
240
|
+
// Go:
|
|
241
|
+
// server: { command: 'go', args: ['run', './cmd/server'] },
|
|
242
|
+
//
|
|
243
|
+
// Node.js:
|
|
244
|
+
// server: { command: 'node', args: ['server.js'] },
|
|
245
|
+
//
|
|
246
|
+
// Optional server options:
|
|
216
247
|
// server: {
|
|
217
|
-
// command: 'python',
|
|
218
|
-
//
|
|
219
|
-
//
|
|
248
|
+
// command: 'python', args: ['server.py'],
|
|
249
|
+
// env: { API_KEY: 'test-key' }, // Extra environment variables
|
|
250
|
+
// cwd: './backend', // Working directory
|
|
251
|
+
// },
|
|
252
|
+
//
|
|
253
|
+
// timeout: 120_000, // Server startup timeout in ms (default: 60s)`;
|
|
220
254
|
}
|
|
221
255
|
if (server.type === 'url') {
|
|
222
256
|
return ` server: {
|
|
@@ -369,31 +403,31 @@ function scaffoldVisualTest(filePath, d) {
|
|
|
369
403
|
* Uncomment the tests below and replace 'your-tool' with your tool name.
|
|
370
404
|
*/
|
|
371
405
|
|
|
372
|
-
// test('tool renders correctly in light mode', async ({
|
|
373
|
-
// const result = await
|
|
406
|
+
// test('tool renders correctly in light mode', async ({ inspector }) => {
|
|
407
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' }, { theme: 'light' });
|
|
374
408
|
// expect(result).not.toBeError();
|
|
375
409
|
//
|
|
376
410
|
// // Wait for UI to render, then screenshot:
|
|
377
411
|
// // const app = result.app();
|
|
378
412
|
// // await expect(app.getByText('Expected text')).toBeVisible();
|
|
379
|
-
// // await
|
|
413
|
+
// // await result.screenshot('tool-light');
|
|
380
414
|
// });
|
|
381
415
|
|
|
382
|
-
// test('tool renders correctly in dark mode', async ({
|
|
383
|
-
// const result = await
|
|
416
|
+
// test('tool renders correctly in dark mode', async ({ inspector }) => {
|
|
417
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' }, { theme: 'dark' });
|
|
384
418
|
// expect(result).not.toBeError();
|
|
385
419
|
//
|
|
386
420
|
// // const app = result.app();
|
|
387
421
|
// // await expect(app.getByText('Expected text')).toBeVisible();
|
|
388
|
-
// // await
|
|
422
|
+
// // await result.screenshot('tool-dark');
|
|
389
423
|
// });
|
|
390
424
|
|
|
391
425
|
// Full-page screenshot (captures the inspector chrome too):
|
|
392
|
-
// test('full page renders correctly', async ({
|
|
393
|
-
// const result = await
|
|
426
|
+
// test('full page renders correctly', async ({ inspector }) => {
|
|
427
|
+
// const result = await inspector.renderTool('your-tool', {}, { theme: 'light' });
|
|
394
428
|
// const app = result.app();
|
|
395
429
|
// await expect(app.getByText('Expected text')).toBeVisible();
|
|
396
|
-
// await
|
|
430
|
+
// await result.screenshot('tool-page', { target: 'page', maxDiffPixelRatio: 0.02 });
|
|
397
431
|
// });
|
|
398
432
|
`
|
|
399
433
|
);
|
|
@@ -557,7 +591,7 @@ async function initExternalProject(cliServer, d) {
|
|
|
557
591
|
type: 'module',
|
|
558
592
|
devDependencies: {
|
|
559
593
|
'@types/node': 'latest',
|
|
560
|
-
sunpeak:
|
|
594
|
+
sunpeak: getSunpeakVersion(),
|
|
561
595
|
'@playwright/test': 'latest',
|
|
562
596
|
},
|
|
563
597
|
scripts: {
|
|
@@ -599,24 +633,28 @@ ${serverBlock}
|
|
|
599
633
|
) + '\n'
|
|
600
634
|
);
|
|
601
635
|
|
|
602
|
-
// 1. E2E test — smoke test, verifies the server
|
|
636
|
+
// 1. E2E test — smoke test, verifies the server exposes tools
|
|
603
637
|
d.writeFileSync(
|
|
604
638
|
join(testDir, 'smoke.test.ts'),
|
|
605
639
|
`import { test, expect } from 'sunpeak/test';
|
|
606
640
|
|
|
607
|
-
test('server
|
|
608
|
-
|
|
609
|
-
|
|
641
|
+
test('server exposes tools', async ({ mcp }) => {
|
|
642
|
+
const tools = await mcp.listTools();
|
|
643
|
+
expect(tools.length).toBeGreaterThan(0);
|
|
610
644
|
});
|
|
611
645
|
|
|
612
|
-
//
|
|
613
|
-
// test('my tool
|
|
646
|
+
// Protocol-level test (no UI rendering):
|
|
647
|
+
// test('my tool returns data', async ({ mcp }) => {
|
|
614
648
|
// const result = await mcp.callTool('your-tool', { key: 'value' });
|
|
649
|
+
// expect(result.isError).toBeFalsy();
|
|
650
|
+
// });
|
|
651
|
+
|
|
652
|
+
// UI rendering test:
|
|
653
|
+
// test('my tool renders correctly', async ({ inspector }) => {
|
|
654
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' });
|
|
615
655
|
// expect(result).not.toBeError();
|
|
616
|
-
//
|
|
617
|
-
//
|
|
618
|
-
// // const app = result.app();
|
|
619
|
-
// // await expect(app.getByText('Hello')).toBeVisible();
|
|
656
|
+
// const app = result.app();
|
|
657
|
+
// await expect(app.getByText('Hello')).toBeVisible();
|
|
620
658
|
// });
|
|
621
659
|
`
|
|
622
660
|
);
|
|
@@ -631,12 +669,27 @@ test('server is reachable and inspector loads', async ({ mcp }) => {
|
|
|
631
669
|
scaffoldEvals(join(testDir, 'evals'), { server, d });
|
|
632
670
|
|
|
633
671
|
d.log.success('Created tests/sunpeak/ with all test types.');
|
|
634
|
-
|
|
672
|
+
if (server.type === 'later') {
|
|
673
|
+
d.log.warn('Server not configured. Edit tests/sunpeak/playwright.config.ts before running tests.');
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
// Auto-install dependencies so users can run tests immediately
|
|
635
677
|
const pm = d.detectPackageManager();
|
|
636
|
-
d.log.
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
678
|
+
d.log.step('Installing dependencies...');
|
|
679
|
+
try {
|
|
680
|
+
d.execSync(`${pm} install`, { cwd: testDir, stdio: 'inherit' });
|
|
681
|
+
} catch {
|
|
682
|
+
d.log.warn(`Dependency install failed. Run manually: cd tests/sunpeak && ${pm} install`);
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
d.log.step('Installing Playwright browser...');
|
|
686
|
+
try {
|
|
687
|
+
d.execSync(`${pm} exec playwright install chromium`, { cwd: testDir, stdio: 'inherit' });
|
|
688
|
+
} catch {
|
|
689
|
+
d.log.warn(`Browser install failed. Run manually: cd tests/sunpeak && ${pm} exec playwright install chromium`);
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
d.log.step('Ready! Run tests with:');
|
|
640
693
|
d.log.message(' sunpeak test # E2E tests');
|
|
641
694
|
d.log.message(' sunpeak test --visual # Visual regression (generates baselines on first run)');
|
|
642
695
|
d.log.message(' sunpeak test --live # Live tests against real hosts (requires login)');
|
|
@@ -677,18 +730,23 @@ ${serverBlock}
|
|
|
677
730
|
testPath,
|
|
678
731
|
`import { test, expect } from 'sunpeak/test';
|
|
679
732
|
|
|
680
|
-
test('server
|
|
681
|
-
await
|
|
733
|
+
test('server exposes tools', async ({ mcp }) => {
|
|
734
|
+
const tools = await mcp.listTools();
|
|
735
|
+
expect(tools.length).toBeGreaterThan(0);
|
|
682
736
|
});
|
|
683
737
|
|
|
684
|
-
//
|
|
685
|
-
// test('my tool
|
|
738
|
+
// Protocol-level test (no UI rendering):
|
|
739
|
+
// test('my tool returns data', async ({ mcp }) => {
|
|
686
740
|
// const result = await mcp.callTool('your-tool', { key: 'value' });
|
|
741
|
+
// expect(result.isError).toBeFalsy();
|
|
742
|
+
// });
|
|
743
|
+
|
|
744
|
+
// UI rendering test:
|
|
745
|
+
// test('my tool renders correctly', async ({ inspector }) => {
|
|
746
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' });
|
|
687
747
|
// expect(result).not.toBeError();
|
|
688
|
-
//
|
|
689
|
-
//
|
|
690
|
-
// // const app = result.app();
|
|
691
|
-
// // await expect(app.getByText('Hello')).toBeVisible();
|
|
748
|
+
// const app = result.app();
|
|
749
|
+
// await expect(app.getByText('Hello')).toBeVisible();
|
|
692
750
|
// });
|
|
693
751
|
`
|
|
694
752
|
);
|
|
@@ -707,6 +765,9 @@ test('server is reachable and inspector loads', async ({ mcp }) => {
|
|
|
707
765
|
// 5. Unit test
|
|
708
766
|
scaffoldUnitTest(join(cwd, 'tests', 'unit', 'example.test.ts'), d);
|
|
709
767
|
|
|
768
|
+
if (server.type === 'later') {
|
|
769
|
+
d.log.warn('Server not configured. Edit playwright.config.ts before running tests.');
|
|
770
|
+
}
|
|
710
771
|
const pkgMgr = d.detectPackageManager();
|
|
711
772
|
d.log.step('Next steps:');
|
|
712
773
|
d.log.message(` ${pkgMgr} add -D sunpeak @playwright/test vitest`);
|
|
@@ -772,6 +833,6 @@ export default defineConfig();
|
|
|
772
833
|
d.log.message(' Replace: import { test, expect } from "@playwright/test"');
|
|
773
834
|
d.log.message(' With: import { test, expect } from "sunpeak/test"');
|
|
774
835
|
d.log.message('');
|
|
775
|
-
d.log.message(' Use the `mcp`
|
|
836
|
+
d.log.message(' Use the `mcp` and `inspector` fixtures instead of raw page navigation.');
|
|
776
837
|
d.log.message(' See sunpeak docs for migration examples.');
|
|
777
838
|
}
|