sunpeak 0.19.12 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/commands/inspect.mjs +361 -12
- package/bin/commands/test-init.mjs +190 -118
- package/bin/commands/test.mjs +12 -1
- package/bin/lib/eval/eval-runner.mjs +7 -1
- package/bin/lib/inspect/inspect-config.mjs +17 -2
- package/bin/lib/inspect/inspect-server.d.mts +32 -0
- package/bin/lib/inspect/inspect-server.mjs +11 -0
- package/bin/lib/live/live-config.d.mts +10 -0
- package/bin/lib/live/live-config.mjs +34 -2
- package/bin/lib/resolve-bin.mjs +39 -0
- package/bin/lib/test/base-config.mjs +6 -3
- package/bin/lib/test/matchers.mjs +2 -2
- package/bin/lib/test/test-config.mjs +19 -8
- package/bin/lib/test/test-fixtures.d.mts +52 -92
- package/bin/lib/test/test-fixtures.mjs +174 -147
- package/dist/chatgpt/index.cjs +1 -1
- package/dist/chatgpt/index.js +1 -1
- package/dist/claude/index.cjs +1 -1
- package/dist/claude/index.js +1 -1
- package/dist/host/chatgpt/index.cjs +1 -1
- package/dist/host/chatgpt/index.js +1 -1
- package/dist/index.cjs +4 -4
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/inspector/index.cjs +1 -1
- package/dist/inspector/index.js +1 -1
- package/dist/{inspector-D5DckQuU.js → inspector-BBDa5yCm.js} +57 -23
- package/dist/inspector-BBDa5yCm.js.map +1 -0
- package/dist/{inspector-jY9O18z9.cjs → inspector-DAA1Wiyh.cjs} +58 -24
- package/dist/inspector-DAA1Wiyh.cjs.map +1 -0
- package/dist/lib/discovery-cli.cjs +1 -1
- package/dist/mcp/index.cjs +22 -25
- package/dist/mcp/index.cjs.map +1 -1
- package/dist/mcp/index.js +19 -22
- package/dist/mcp/index.js.map +1 -1
- package/dist/{use-app-Bfargfa3.js → use-app-Cr0auUa1.js} +2 -2
- package/dist/{use-app-Bfargfa3.js.map → use-app-Cr0auUa1.js.map} +1 -1
- package/dist/{use-app-CbsBEmwv.cjs → use-app-DPkj5Jp_.cjs} +2 -2
- package/dist/{use-app-CbsBEmwv.cjs.map → use-app-DPkj5Jp_.cjs.map} +1 -1
- package/package.json +17 -11
- package/template/dist/albums/albums.html +4 -4
- package/template/dist/albums/albums.json +1 -1
- package/template/dist/carousel/carousel.html +4 -4
- package/template/dist/carousel/carousel.json +1 -1
- package/template/dist/map/map.html +6 -6
- package/template/dist/map/map.json +1 -1
- package/template/dist/review/review.html +4 -4
- package/template/dist/review/review.json +1 -1
- package/template/node_modules/.bin/vite +2 -2
- package/template/node_modules/.bin/vitest +2 -2
- package/template/node_modules/.vite/deps/_metadata.json +4 -4
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps.js +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_app-bridge.js +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_app-bridge.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_react.js +1 -1
- package/template/node_modules/.vite-mcp/deps/@modelcontextprotocol_ext-apps_react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/@testing-library_react.js +4 -4
- package/template/node_modules/.vite-mcp/deps/@testing-library_react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/_metadata.json +33 -33
- package/template/node_modules/.vite-mcp/deps/{client-CU1wWud4.js → client-B_5CX--u.js} +7 -7
- package/template/node_modules/.vite-mcp/deps/{client-CU1wWud4.js.map → client-B_5CX--u.js.map} +1 -1
- package/template/node_modules/.vite-mcp/deps/embla-carousel-react.js +1 -1
- package/template/node_modules/.vite-mcp/deps/embla-carousel-react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react-dom.js +3 -3
- package/template/node_modules/.vite-mcp/deps/react-dom.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react-dom_client.js +1 -1
- package/template/node_modules/.vite-mcp/deps/react.js +3 -3
- package/template/node_modules/.vite-mcp/deps/react.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react_jsx-dev-runtime.js +2 -2
- package/template/node_modules/.vite-mcp/deps/react_jsx-dev-runtime.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/react_jsx-runtime.js +2 -2
- package/template/node_modules/.vite-mcp/deps/react_jsx-runtime.js.map +1 -1
- package/template/node_modules/.vite-mcp/deps/vitest.js +1024 -622
- package/template/node_modules/.vite-mcp/deps/vitest.js.map +1 -1
- package/template/package.json +6 -6
- package/template/tests/e2e/albums.spec.ts +24 -52
- package/template/tests/e2e/carousel.spec.ts +36 -58
- package/template/tests/e2e/map.spec.ts +35 -56
- package/template/tests/e2e/review.spec.ts +56 -85
- package/template/tests/e2e/visual.spec.ts +14 -12
- package/dist/inspector-D5DckQuU.js.map +0 -1
- package/dist/inspector-jY9O18z9.cjs.map +0 -1
|
@@ -1,10 +1,23 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
|
|
2
2
|
import { execSync } from 'child_process';
|
|
3
3
|
import { join, dirname } from 'path';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
4
5
|
import * as p from '@clack/prompts';
|
|
5
6
|
import { EVAL_PROVIDERS, generateModelLines } from '../lib/eval/eval-providers.mjs';
|
|
6
7
|
import { detectPackageManager } from '../utils.mjs';
|
|
7
8
|
|
|
9
|
+
/** Read the current sunpeak package version for pinning in scaffolded configs. */
|
|
10
|
+
function getSunpeakVersion() {
|
|
11
|
+
try {
|
|
12
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
const pkgPath = join(__dirname, '..', '..', 'package.json');
|
|
14
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
|
|
15
|
+
return pkg.version ? `^${pkg.version}` : 'latest';
|
|
16
|
+
} catch {
|
|
17
|
+
return 'latest';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
8
21
|
/**
|
|
9
22
|
* Default dependencies (real implementations).
|
|
10
23
|
* Override in tests via the `deps` parameter.
|
|
@@ -27,6 +40,7 @@ export const defaultDeps = {
|
|
|
27
40
|
mkdirSync,
|
|
28
41
|
execSync,
|
|
29
42
|
cwd: () => process.cwd(),
|
|
43
|
+
isTTY: () => !!process.stdin.isTTY,
|
|
30
44
|
intro: p.intro,
|
|
31
45
|
outro: p.outro,
|
|
32
46
|
confirm: p.confirm,
|
|
@@ -49,7 +63,7 @@ export const defaultDeps = {
|
|
|
49
63
|
*
|
|
50
64
|
* Scaffolds all 5 test types:
|
|
51
65
|
* 1. E2E tests — Playwright-based inspector tests (mcp fixture)
|
|
52
|
-
* 2. Visual regression — Screenshot comparison via
|
|
66
|
+
* 2. Visual regression — Screenshot comparison via result.screenshot()
|
|
53
67
|
* 3. Live tests — Test against real ChatGPT/Claude hosts
|
|
54
68
|
* 4. Evals — Multi-model tool calling reliability tests
|
|
55
69
|
* 5. Unit tests — Direct tool handler tests (JS/TS projects only)
|
|
@@ -67,6 +81,7 @@ export async function testInit(args = [], deps = defaultDeps) {
|
|
|
67
81
|
: undefined;
|
|
68
82
|
|
|
69
83
|
const projectType = detectProjectType(d);
|
|
84
|
+
const interactive = d.isTTY();
|
|
70
85
|
|
|
71
86
|
if (projectType === 'sunpeak') {
|
|
72
87
|
await initSunpeakProject(d);
|
|
@@ -76,74 +91,76 @@ export async function testInit(args = [], deps = defaultDeps) {
|
|
|
76
91
|
await initExternalProject(cliServer, d);
|
|
77
92
|
}
|
|
78
93
|
|
|
79
|
-
// Offer to configure eval providers
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
94
|
+
// Offer to configure eval providers (skip without a TTY — prompts can't work)
|
|
95
|
+
if (interactive) {
|
|
96
|
+
const providers = await d.selectProviders();
|
|
97
|
+
if (!d.isCancel(providers) && providers.length > 0) {
|
|
98
|
+
const pm = d.detectPackageManager();
|
|
99
|
+
const pkgsToInstall = ['ai', ...providers.map((p) => p.pkg)];
|
|
100
|
+
const installCmd = `${pm} add -D ${pkgsToInstall.join(' ')}`;
|
|
101
|
+
try {
|
|
102
|
+
d.execSync(installCmd, { cwd: d.cwd(), stdio: 'inherit' });
|
|
103
|
+
} catch {
|
|
104
|
+
d.log.info(`Provider install failed. Install manually: ${installCmd}`);
|
|
105
|
+
}
|
|
90
106
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
+
// Uncomment selected models in eval.config.ts
|
|
108
|
+
const evalDir = d.existsSync(join(d.cwd(), 'tests', 'evals'))
|
|
109
|
+
? join(d.cwd(), 'tests', 'evals')
|
|
110
|
+
: d.existsSync(join(d.cwd(), 'tests', 'sunpeak', 'evals'))
|
|
111
|
+
? join(d.cwd(), 'tests', 'sunpeak', 'evals')
|
|
112
|
+
: null;
|
|
113
|
+
if (evalDir) {
|
|
114
|
+
const configPath = join(evalDir, 'eval.config.ts');
|
|
115
|
+
if (d.existsSync(configPath)) {
|
|
116
|
+
let config = d.readFileSync(configPath, 'utf-8');
|
|
117
|
+
for (const prov of providers) {
|
|
118
|
+
for (const model of prov.models) {
|
|
119
|
+
config = config.replace(
|
|
120
|
+
new RegExp(`^(\\s*)// ('${model.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}',?.*)$`, 'm'),
|
|
121
|
+
'$1$2'
|
|
122
|
+
);
|
|
123
|
+
}
|
|
107
124
|
}
|
|
125
|
+
d.writeFileSync(configPath, config);
|
|
108
126
|
}
|
|
109
|
-
d.writeFileSync(configPath, config);
|
|
110
|
-
}
|
|
111
127
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
128
|
+
// Prompt for API keys and write .env
|
|
129
|
+
const envLines = [];
|
|
130
|
+
const seen = new Set();
|
|
131
|
+
for (const prov of providers) {
|
|
132
|
+
if (seen.has(prov.envVar)) continue;
|
|
133
|
+
seen.add(prov.envVar);
|
|
134
|
+
const key = await d.password({
|
|
135
|
+
message: `${prov.envVar} (enter to skip)`,
|
|
136
|
+
mask: '*',
|
|
137
|
+
});
|
|
138
|
+
if (!d.isCancel(key) && key) {
|
|
139
|
+
envLines.push(`${prov.envVar}=${key}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (envLines.length > 0 && evalDir) {
|
|
143
|
+
const relEnvPath = evalDir.startsWith(d.cwd()) ? evalDir.slice(d.cwd().length + 1) : evalDir;
|
|
144
|
+
d.writeFileSync(join(evalDir, '.env'), envLines.join('\n') + '\n');
|
|
145
|
+
d.log.info(`API keys saved to ${relEnvPath}/.env (gitignored)`);
|
|
124
146
|
}
|
|
125
|
-
}
|
|
126
|
-
if (envLines.length > 0 && evalDir) {
|
|
127
|
-
const relEnvPath = evalDir.startsWith(d.cwd()) ? evalDir.slice(d.cwd().length + 1) : evalDir;
|
|
128
|
-
d.writeFileSync(join(evalDir, '.env'), envLines.join('\n') + '\n');
|
|
129
|
-
d.log.info(`API keys saved to ${relEnvPath}/.env (gitignored)`);
|
|
130
147
|
}
|
|
131
148
|
}
|
|
132
|
-
}
|
|
133
149
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
150
|
+
// Offer to install the testing skill
|
|
151
|
+
const installSkill = await d.confirm({
|
|
152
|
+
message: 'Install the test-mcp-server skill? (helps your coding agent write tests)',
|
|
153
|
+
initialValue: true,
|
|
154
|
+
});
|
|
155
|
+
if (!d.isCancel(installSkill) && installSkill) {
|
|
156
|
+
try {
|
|
157
|
+
d.execSync('pnpm dlx skills add Sunpeak-AI/sunpeak@test-mcp-server', {
|
|
158
|
+
cwd: d.cwd(),
|
|
159
|
+
stdio: 'inherit',
|
|
160
|
+
});
|
|
161
|
+
} catch {
|
|
162
|
+
d.log.info('Skill install skipped. Install later: pnpm dlx skills add Sunpeak-AI/sunpeak@test-mcp-server');
|
|
163
|
+
}
|
|
147
164
|
}
|
|
148
165
|
}
|
|
149
166
|
|
|
@@ -178,6 +195,11 @@ async function getServerConfig(cliServer, d) {
|
|
|
178
195
|
return { type: 'command', value: cliServer };
|
|
179
196
|
}
|
|
180
197
|
|
|
198
|
+
// Without a TTY, interactive prompts can't work — default to "configure later".
|
|
199
|
+
if (!d.isTTY()) {
|
|
200
|
+
return { type: 'later' };
|
|
201
|
+
}
|
|
202
|
+
|
|
181
203
|
const serverType = await d.select({
|
|
182
204
|
message: 'How does your MCP server start?',
|
|
183
205
|
options: [
|
|
@@ -212,11 +234,32 @@ async function getServerConfig(cliServer, d) {
|
|
|
212
234
|
|
|
213
235
|
function generateServerConfigBlock(server, relativeTo = '.') {
|
|
214
236
|
if (server.type === 'later') {
|
|
215
|
-
return ` // TODO: Configure your MCP server connection
|
|
237
|
+
return ` // TODO: Configure your MCP server connection before running tests.
|
|
238
|
+
// Uncomment one of the options below:
|
|
239
|
+
//
|
|
240
|
+
// HTTP server (Python FastAPI, Go, etc.):
|
|
241
|
+
// server: { url: 'http://localhost:8000/mcp' },
|
|
242
|
+
//
|
|
243
|
+
// Python (uv):
|
|
244
|
+
// server: { command: 'uv', args: ['run', 'python', 'server.py'] },
|
|
245
|
+
//
|
|
246
|
+
// Python (venv):
|
|
247
|
+
// server: { command: '.venv/bin/python', args: ['server.py'] },
|
|
248
|
+
//
|
|
249
|
+
// Go:
|
|
250
|
+
// server: { command: 'go', args: ['run', './cmd/server'] },
|
|
251
|
+
//
|
|
252
|
+
// Node.js:
|
|
253
|
+
// server: { command: 'node', args: ['server.js'] },
|
|
254
|
+
//
|
|
255
|
+
// Optional server options:
|
|
216
256
|
// server: {
|
|
217
|
-
// command: 'python',
|
|
218
|
-
//
|
|
219
|
-
//
|
|
257
|
+
// command: 'python', args: ['server.py'],
|
|
258
|
+
// env: { API_KEY: 'test-key' }, // Extra environment variables
|
|
259
|
+
// cwd: './backend', // Working directory
|
|
260
|
+
// },
|
|
261
|
+
//
|
|
262
|
+
// timeout: 120_000, // Server startup timeout in ms (default: 60s)`;
|
|
220
263
|
}
|
|
221
264
|
if (server.type === 'url') {
|
|
222
265
|
return ` server: {
|
|
@@ -369,31 +412,31 @@ function scaffoldVisualTest(filePath, d) {
|
|
|
369
412
|
* Uncomment the tests below and replace 'your-tool' with your tool name.
|
|
370
413
|
*/
|
|
371
414
|
|
|
372
|
-
// test('tool renders correctly in light mode', async ({
|
|
373
|
-
// const result = await
|
|
415
|
+
// test('tool renders correctly in light mode', async ({ inspector }) => {
|
|
416
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' }, { theme: 'light' });
|
|
374
417
|
// expect(result).not.toBeError();
|
|
375
418
|
//
|
|
376
419
|
// // Wait for UI to render, then screenshot:
|
|
377
420
|
// // const app = result.app();
|
|
378
421
|
// // await expect(app.getByText('Expected text')).toBeVisible();
|
|
379
|
-
// // await
|
|
422
|
+
// // await result.screenshot('tool-light');
|
|
380
423
|
// });
|
|
381
424
|
|
|
382
|
-
// test('tool renders correctly in dark mode', async ({
|
|
383
|
-
// const result = await
|
|
425
|
+
// test('tool renders correctly in dark mode', async ({ inspector }) => {
|
|
426
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' }, { theme: 'dark' });
|
|
384
427
|
// expect(result).not.toBeError();
|
|
385
428
|
//
|
|
386
429
|
// // const app = result.app();
|
|
387
430
|
// // await expect(app.getByText('Expected text')).toBeVisible();
|
|
388
|
-
// // await
|
|
431
|
+
// // await result.screenshot('tool-dark');
|
|
389
432
|
// });
|
|
390
433
|
|
|
391
434
|
// Full-page screenshot (captures the inspector chrome too):
|
|
392
|
-
// test('full page renders correctly', async ({
|
|
393
|
-
// const result = await
|
|
435
|
+
// test('full page renders correctly', async ({ inspector }) => {
|
|
436
|
+
// const result = await inspector.renderTool('your-tool', {}, { theme: 'light' });
|
|
394
437
|
// const app = result.app();
|
|
395
438
|
// await expect(app.getByText('Expected text')).toBeVisible();
|
|
396
|
-
// await
|
|
439
|
+
// await result.screenshot('tool-page', { target: 'page', maxDiffPixelRatio: 0.02 });
|
|
397
440
|
// });
|
|
398
441
|
`
|
|
399
442
|
);
|
|
@@ -403,9 +446,9 @@ function scaffoldVisualTest(filePath, d) {
|
|
|
403
446
|
/**
|
|
404
447
|
* Scaffold live test boilerplate (test against real ChatGPT/Claude).
|
|
405
448
|
* @param {string} liveDir - Directory to create live test files in
|
|
406
|
-
* @param {{ isSunpeak?: boolean, d: object }} options
|
|
449
|
+
* @param {{ isSunpeak?: boolean, server?: object, d: object }} options
|
|
407
450
|
*/
|
|
408
|
-
function scaffoldLiveTests(liveDir, { isSunpeak, d } = {}) {
|
|
451
|
+
function scaffoldLiveTests(liveDir, { isSunpeak, server, d } = {}) {
|
|
409
452
|
if (d.existsSync(join(liveDir, 'playwright.config.ts'))) {
|
|
410
453
|
d.log.info('Live test config already exists. Skipping live test scaffold.');
|
|
411
454
|
return;
|
|
@@ -425,9 +468,24 @@ function scaffoldLiveTests(liveDir, { isSunpeak, d } = {}) {
|
|
|
425
468
|
* 3. Run: sunpeak test --live
|
|
426
469
|
*
|
|
427
470
|
* On first run, a browser window opens for you to log in to the host.
|
|
428
|
-
* The session is saved for subsequent runs (typically lasts a few hours)
|
|
471
|
+
* The session is saved for subsequent runs (typically lasts a few hours).
|
|
472
|
+
*/`;
|
|
473
|
+
|
|
474
|
+
// Build the server option for non-sunpeak projects
|
|
475
|
+
let serverOption = '';
|
|
476
|
+
if (!isSunpeak && server?.type === 'url') {
|
|
477
|
+
serverOption = `\n server: { url: '${server.value}' },`;
|
|
478
|
+
} else if (!isSunpeak && server?.type === 'command') {
|
|
479
|
+
const parts = server.value.split(/\s+/);
|
|
480
|
+
const cmd = parts[0];
|
|
481
|
+
const args = parts.slice(1);
|
|
482
|
+
serverOption = args.length > 0
|
|
483
|
+
? `\n server: { command: '${cmd}', args: [${args.map(a => `'${a}'`).join(', ')}] },`
|
|
484
|
+
: `\n server: { command: '${cmd}' },`;
|
|
485
|
+
}
|
|
429
486
|
|
|
430
|
-
const
|
|
487
|
+
const configContent = `${liveConfigPreamble}
|
|
488
|
+
export default defineLiveConfig({${serverOption}
|
|
431
489
|
// hosts: ['chatgpt'], // Which hosts to test against
|
|
432
490
|
// colorScheme: 'light', // Default color scheme
|
|
433
491
|
// viewport: { width: 1280, height: 720 },
|
|
@@ -435,19 +493,6 @@ function scaffoldLiveTests(liveDir, { isSunpeak, d } = {}) {
|
|
|
435
493
|
});
|
|
436
494
|
`;
|
|
437
495
|
|
|
438
|
-
const configContent = isSunpeak
|
|
439
|
-
? `${liveConfigPreamble}
|
|
440
|
-
*/
|
|
441
|
-
${liveConfigExport}`
|
|
442
|
-
: `${liveConfigPreamble}
|
|
443
|
-
*
|
|
444
|
-
* NOTE: defineLiveConfig() starts a local sunpeak dev server as its backend.
|
|
445
|
-
* If your MCP server is not a sunpeak project, you may need to customize the
|
|
446
|
-
* webServer option in the Playwright config below to start your own server,
|
|
447
|
-
* or remove webServer entirely if your server is already running.
|
|
448
|
-
*/
|
|
449
|
-
${liveConfigExport}`;
|
|
450
|
-
|
|
451
496
|
d.writeFileSync(join(liveDir, 'playwright.config.ts'), configContent);
|
|
452
497
|
|
|
453
498
|
// Live test example
|
|
@@ -557,7 +602,7 @@ async function initExternalProject(cliServer, d) {
|
|
|
557
602
|
type: 'module',
|
|
558
603
|
devDependencies: {
|
|
559
604
|
'@types/node': 'latest',
|
|
560
|
-
sunpeak:
|
|
605
|
+
sunpeak: getSunpeakVersion(),
|
|
561
606
|
'@playwright/test': 'latest',
|
|
562
607
|
},
|
|
563
608
|
scripts: {
|
|
@@ -599,24 +644,28 @@ ${serverBlock}
|
|
|
599
644
|
) + '\n'
|
|
600
645
|
);
|
|
601
646
|
|
|
602
|
-
// 1. E2E test — smoke test, verifies the server
|
|
647
|
+
// 1. E2E test — smoke test, verifies the server exposes tools
|
|
603
648
|
d.writeFileSync(
|
|
604
649
|
join(testDir, 'smoke.test.ts'),
|
|
605
650
|
`import { test, expect } from 'sunpeak/test';
|
|
606
651
|
|
|
607
|
-
test('server
|
|
608
|
-
|
|
609
|
-
|
|
652
|
+
test('server exposes tools', async ({ mcp }) => {
|
|
653
|
+
const tools = await mcp.listTools();
|
|
654
|
+
expect(tools.length).toBeGreaterThan(0);
|
|
610
655
|
});
|
|
611
656
|
|
|
612
|
-
//
|
|
613
|
-
// test('my tool
|
|
657
|
+
// Protocol-level test (no UI rendering):
|
|
658
|
+
// test('my tool returns data', async ({ mcp }) => {
|
|
614
659
|
// const result = await mcp.callTool('your-tool', { key: 'value' });
|
|
660
|
+
// expect(result.isError).toBeFalsy();
|
|
661
|
+
// });
|
|
662
|
+
|
|
663
|
+
// UI rendering test:
|
|
664
|
+
// test('my tool renders correctly', async ({ inspector }) => {
|
|
665
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' });
|
|
615
666
|
// expect(result).not.toBeError();
|
|
616
|
-
//
|
|
617
|
-
//
|
|
618
|
-
// // const app = result.app();
|
|
619
|
-
// // await expect(app.getByText('Hello')).toBeVisible();
|
|
667
|
+
// const app = result.app();
|
|
668
|
+
// await expect(app.getByText('Hello')).toBeVisible();
|
|
620
669
|
// });
|
|
621
670
|
`
|
|
622
671
|
);
|
|
@@ -625,18 +674,33 @@ test('server is reachable and inspector loads', async ({ mcp }) => {
|
|
|
625
674
|
scaffoldVisualTest(join(testDir, 'visual.test.ts'), d);
|
|
626
675
|
|
|
627
676
|
// 3. Live tests
|
|
628
|
-
scaffoldLiveTests(join(testDir, 'live'), { isSunpeak: false, d });
|
|
677
|
+
scaffoldLiveTests(join(testDir, 'live'), { isSunpeak: false, server, d });
|
|
629
678
|
|
|
630
679
|
// 4. Eval boilerplate
|
|
631
680
|
scaffoldEvals(join(testDir, 'evals'), { server, d });
|
|
632
681
|
|
|
633
682
|
d.log.success('Created tests/sunpeak/ with all test types.');
|
|
634
|
-
|
|
683
|
+
if (server.type === 'later') {
|
|
684
|
+
d.log.warn('Server not configured. Edit tests/sunpeak/playwright.config.ts before running tests.');
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// Auto-install dependencies so users can run tests immediately
|
|
635
688
|
const pm = d.detectPackageManager();
|
|
636
|
-
d.log.
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
689
|
+
d.log.step('Installing dependencies...');
|
|
690
|
+
try {
|
|
691
|
+
d.execSync(`${pm} install`, { cwd: testDir, stdio: 'inherit' });
|
|
692
|
+
} catch {
|
|
693
|
+
d.log.warn(`Dependency install failed. Run manually: cd tests/sunpeak && ${pm} install`);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
d.log.step('Installing Playwright browser...');
|
|
697
|
+
try {
|
|
698
|
+
d.execSync(`${pm} exec playwright install chromium`, { cwd: testDir, stdio: 'inherit' });
|
|
699
|
+
} catch {
|
|
700
|
+
d.log.warn(`Browser install failed. Run manually: cd tests/sunpeak && ${pm} exec playwright install chromium`);
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
d.log.step('Ready! Run tests with:');
|
|
640
704
|
d.log.message(' sunpeak test # E2E tests');
|
|
641
705
|
d.log.message(' sunpeak test --visual # Visual regression (generates baselines on first run)');
|
|
642
706
|
d.log.message(' sunpeak test --live # Live tests against real hosts (requires login)');
|
|
@@ -677,18 +741,23 @@ ${serverBlock}
|
|
|
677
741
|
testPath,
|
|
678
742
|
`import { test, expect } from 'sunpeak/test';
|
|
679
743
|
|
|
680
|
-
test('server
|
|
681
|
-
await
|
|
744
|
+
test('server exposes tools', async ({ mcp }) => {
|
|
745
|
+
const tools = await mcp.listTools();
|
|
746
|
+
expect(tools.length).toBeGreaterThan(0);
|
|
682
747
|
});
|
|
683
748
|
|
|
684
|
-
//
|
|
685
|
-
// test('my tool
|
|
749
|
+
// Protocol-level test (no UI rendering):
|
|
750
|
+
// test('my tool returns data', async ({ mcp }) => {
|
|
686
751
|
// const result = await mcp.callTool('your-tool', { key: 'value' });
|
|
752
|
+
// expect(result.isError).toBeFalsy();
|
|
753
|
+
// });
|
|
754
|
+
|
|
755
|
+
// UI rendering test:
|
|
756
|
+
// test('my tool renders correctly', async ({ inspector }) => {
|
|
757
|
+
// const result = await inspector.renderTool('your-tool', { key: 'value' });
|
|
687
758
|
// expect(result).not.toBeError();
|
|
688
|
-
//
|
|
689
|
-
//
|
|
690
|
-
// // const app = result.app();
|
|
691
|
-
// // await expect(app.getByText('Hello')).toBeVisible();
|
|
759
|
+
// const app = result.app();
|
|
760
|
+
// await expect(app.getByText('Hello')).toBeVisible();
|
|
692
761
|
// });
|
|
693
762
|
`
|
|
694
763
|
);
|
|
@@ -699,7 +768,7 @@ test('server is reachable and inspector loads', async ({ mcp }) => {
|
|
|
699
768
|
scaffoldVisualTest(join(e2eDir, 'visual.test.ts'), d);
|
|
700
769
|
|
|
701
770
|
// 3. Live tests
|
|
702
|
-
scaffoldLiveTests(join(cwd, 'tests', 'live'), { isSunpeak: false, d });
|
|
771
|
+
scaffoldLiveTests(join(cwd, 'tests', 'live'), { isSunpeak: false, server, d });
|
|
703
772
|
|
|
704
773
|
// 4. Eval boilerplate
|
|
705
774
|
scaffoldEvals(join(cwd, 'tests', 'evals'), { server, d });
|
|
@@ -707,6 +776,9 @@ test('server is reachable and inspector loads', async ({ mcp }) => {
|
|
|
707
776
|
// 5. Unit test
|
|
708
777
|
scaffoldUnitTest(join(cwd, 'tests', 'unit', 'example.test.ts'), d);
|
|
709
778
|
|
|
779
|
+
if (server.type === 'later') {
|
|
780
|
+
d.log.warn('Server not configured. Edit playwright.config.ts before running tests.');
|
|
781
|
+
}
|
|
710
782
|
const pkgMgr = d.detectPackageManager();
|
|
711
783
|
d.log.step('Next steps:');
|
|
712
784
|
d.log.message(` ${pkgMgr} add -D sunpeak @playwright/test vitest`);
|
|
@@ -772,6 +844,6 @@ export default defineConfig();
|
|
|
772
844
|
d.log.message(' Replace: import { test, expect } from "@playwright/test"');
|
|
773
845
|
d.log.message(' With: import { test, expect } from "sunpeak/test"');
|
|
774
846
|
d.log.message('');
|
|
775
|
-
d.log.message(' Use the `mcp`
|
|
847
|
+
d.log.message(' Use the `mcp` and `inspector` fixtures instead of raw page navigation.');
|
|
776
848
|
d.log.message(' See sunpeak docs for migration examples.');
|
|
777
849
|
}
|
package/bin/commands/test.mjs
CHANGED
|
@@ -70,6 +70,9 @@ export async function runTest(args) {
|
|
|
70
70
|
'playwright.config.js',
|
|
71
71
|
'sunpeak.config.ts',
|
|
72
72
|
'sunpeak.config.js',
|
|
73
|
+
// Fallback for non-JS projects: tests/sunpeak/ self-contained directory
|
|
74
|
+
'tests/sunpeak/playwright.config.ts',
|
|
75
|
+
'tests/sunpeak/playwright.config.js',
|
|
73
76
|
],
|
|
74
77
|
visual: isVisual,
|
|
75
78
|
updateSnapshots: isVisual && isUpdate,
|
|
@@ -82,9 +85,15 @@ export async function runTest(args) {
|
|
|
82
85
|
configCandidates: [
|
|
83
86
|
'tests/live/playwright.config.ts',
|
|
84
87
|
'tests/live/playwright.config.js',
|
|
88
|
+
// Non-JS projects: tests/sunpeak/ self-contained directory (run from project root)
|
|
89
|
+
'tests/sunpeak/live/playwright.config.ts',
|
|
90
|
+
'tests/sunpeak/live/playwright.config.js',
|
|
91
|
+
// Non-JS projects: run from within tests/sunpeak/ directly
|
|
92
|
+
'live/playwright.config.ts',
|
|
93
|
+
'live/playwright.config.js',
|
|
85
94
|
],
|
|
86
95
|
configRequired: true,
|
|
87
|
-
configErrorMessage: 'No live test config found at tests/live/playwright.config.ts',
|
|
96
|
+
configErrorMessage: 'No live test config found. Expected at tests/live/playwright.config.ts or live/playwright.config.ts',
|
|
88
97
|
});
|
|
89
98
|
results.push({ suite: 'live', code });
|
|
90
99
|
}
|
|
@@ -531,6 +540,8 @@ function findEvalDir() {
|
|
|
531
540
|
const candidates = [
|
|
532
541
|
'tests/evals',
|
|
533
542
|
'tests/sunpeak/evals',
|
|
543
|
+
// When running from within tests/sunpeak/ directly (non-JS projects)
|
|
544
|
+
'evals',
|
|
534
545
|
];
|
|
535
546
|
|
|
536
547
|
for (const candidate of candidates) {
|
|
@@ -77,7 +77,13 @@ export async function createMcpConnection(serverArg) {
|
|
|
77
77
|
const { StreamableHTTPClientTransport } = await import(
|
|
78
78
|
'@modelcontextprotocol/sdk/client/streamableHttp.js'
|
|
79
79
|
);
|
|
80
|
-
|
|
80
|
+
// Follow redirects (e.g. /mcp → /mcp/) before creating the transport.
|
|
81
|
+
let finalUrl = serverArg;
|
|
82
|
+
try {
|
|
83
|
+
const resp = await fetch(serverArg, { method: 'HEAD', redirect: 'follow' });
|
|
84
|
+
if (resp.url && resp.url !== serverArg) finalUrl = resp.url;
|
|
85
|
+
} catch { /* use original URL */ }
|
|
86
|
+
const transport = new StreamableHTTPClientTransport(new URL(finalUrl));
|
|
81
87
|
await client.connect(transport);
|
|
82
88
|
return { client, transport };
|
|
83
89
|
} else {
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* and external servers.
|
|
16
16
|
*/
|
|
17
17
|
import { createBaseConfig, resolvePorts } from '../test/base-config.mjs';
|
|
18
|
+
import { resolveSunpeakBin } from '../resolve-bin.mjs';
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* Create a complete Playwright config for testing an external MCP server.
|
|
@@ -26,6 +27,9 @@ import { createBaseConfig, resolvePorts } from '../test/base-config.mjs';
|
|
|
26
27
|
* @param {string[]} [options.hosts=['chatgpt', 'claude']] - Host shells to test
|
|
27
28
|
* @param {string} [options.name] - App name in inspector chrome
|
|
28
29
|
* @param {Object} [options.use] - Additional Playwright `use` options
|
|
30
|
+
* @param {number} [options.timeout] - Server startup timeout in ms (default: 60000)
|
|
31
|
+
* @param {Record<string, string>} [options.env] - Environment variables for stdio servers
|
|
32
|
+
* @param {string} [options.cwd] - Working directory for stdio servers
|
|
29
33
|
* @returns {import('@playwright/test').PlaywrightTestConfig}
|
|
30
34
|
*/
|
|
31
35
|
export function defineInspectConfig(options) {
|
|
@@ -37,6 +41,9 @@ export function defineInspectConfig(options) {
|
|
|
37
41
|
name,
|
|
38
42
|
use: userUse,
|
|
39
43
|
visual,
|
|
44
|
+
timeout,
|
|
45
|
+
env,
|
|
46
|
+
cwd,
|
|
40
47
|
} = options;
|
|
41
48
|
|
|
42
49
|
if (!server) {
|
|
@@ -49,8 +56,15 @@ export function defineInspectConfig(options) {
|
|
|
49
56
|
const serverArg = server.includes(' ') ? `"${server}"` : server;
|
|
50
57
|
const command = [
|
|
51
58
|
`SUNPEAK_SANDBOX_PORT=${sandboxPort}`,
|
|
52
|
-
|
|
59
|
+
`${resolveSunpeakBin()} inspect`,
|
|
53
60
|
`--server ${serverArg}`,
|
|
61
|
+
...(env
|
|
62
|
+
? Object.entries(env).map(([k, v]) => {
|
|
63
|
+
const pair = `${k}=${v}`;
|
|
64
|
+
return pair.includes(' ') ? `--env "${pair}"` : `--env ${pair}`;
|
|
65
|
+
})
|
|
66
|
+
: []),
|
|
67
|
+
...(cwd ? [cwd.includes(' ') ? `--cwd "${cwd}"` : `--cwd ${cwd}`] : []),
|
|
54
68
|
...(simulationsDir ? [`--simulations ${simulationsDir}`] : []),
|
|
55
69
|
`--port ${port}`,
|
|
56
70
|
...(name ? [`--name "${name}"`] : []),
|
|
@@ -62,9 +76,10 @@ export function defineInspectConfig(options) {
|
|
|
62
76
|
port,
|
|
63
77
|
use: userUse,
|
|
64
78
|
visual,
|
|
79
|
+
timeout,
|
|
65
80
|
webServer: {
|
|
66
81
|
command,
|
|
67
|
-
healthUrl: `http://
|
|
82
|
+
healthUrl: `http://127.0.0.1:${port}/health`,
|
|
68
83
|
},
|
|
69
84
|
});
|
|
70
85
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Start the sunpeak inspector server programmatically.
|
|
3
|
+
*
|
|
4
|
+
* Connects to the provided MCP server, discovers tools and resources,
|
|
5
|
+
* and serves the inspector UI on the specified port.
|
|
6
|
+
*/
|
|
7
|
+
export function inspectServer(opts: {
|
|
8
|
+
/** MCP server URL or stdio command. */
|
|
9
|
+
server: string;
|
|
10
|
+
/** Path to simulation fixtures directory. */
|
|
11
|
+
simulationsDir?: string | null;
|
|
12
|
+
/** Dev server port (default: 3000). */
|
|
13
|
+
port?: number;
|
|
14
|
+
/** App name in inspector chrome. */
|
|
15
|
+
name?: string;
|
|
16
|
+
/** Existing sandbox server URL (skips creating one). */
|
|
17
|
+
sandboxUrl?: string;
|
|
18
|
+
/** If true, show framework-only controls (Prod Resources). */
|
|
19
|
+
frameworkMode?: boolean;
|
|
20
|
+
/** Initial prod resources state. */
|
|
21
|
+
defaultProdResources?: boolean;
|
|
22
|
+
/** Project directory for serving /dist/ files (prod resources). */
|
|
23
|
+
projectRoot?: string | null;
|
|
24
|
+
/** Whether to open browser (default: !CI). */
|
|
25
|
+
open?: boolean;
|
|
26
|
+
/** Additional cleanup callback on exit. */
|
|
27
|
+
onCleanup?: () => Promise<void>;
|
|
28
|
+
/** Extra environment variables for stdio server processes. */
|
|
29
|
+
env?: Record<string, string>;
|
|
30
|
+
/** Working directory for stdio server processes. */
|
|
31
|
+
cwd?: string;
|
|
32
|
+
}): Promise<void>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Programmatic entry point for the sunpeak inspector server.
|
|
3
|
+
*
|
|
4
|
+
* Allows frameworks to start the inspector from their own CLI without
|
|
5
|
+
* shelling out to the `sunpeak inspect` command.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* import { inspectServer } from 'sunpeak/inspect';
|
|
9
|
+
* await inspectServer({ server: 'http://localhost:8000/mcp', port: 3000 });
|
|
10
|
+
*/
|
|
11
|
+
export { inspectServer } from '../../commands/inspect.mjs';
|
|
@@ -28,6 +28,16 @@ export interface LiveConfigOptions {
|
|
|
28
28
|
|
|
29
29
|
/** Additional Playwright `use` options, merged with defaults. */
|
|
30
30
|
use?: Record<string, unknown>;
|
|
31
|
+
|
|
32
|
+
/** External MCP server config. Omit for sunpeak framework projects. */
|
|
33
|
+
server?: {
|
|
34
|
+
/** Server URL (e.g., 'http://localhost:8000/mcp') */
|
|
35
|
+
url?: string;
|
|
36
|
+
/** Server start command */
|
|
37
|
+
command?: string;
|
|
38
|
+
/** Command arguments */
|
|
39
|
+
args?: string[];
|
|
40
|
+
};
|
|
31
41
|
}
|
|
32
42
|
|
|
33
43
|
export interface HostConfigOptions {
|