sunpeak 0.18.13 → 0.19.1

package/bin/lib/test/base-config.mjs

@@ -0,0 +1,60 @@
+/**
+ * Shared Playwright config builder used by both defineConfig() (sunpeak projects)
+ * and defineInspectConfig() (external MCP servers).
+ *
+ * Produces a config with per-host Playwright projects, sensible defaults for
+ * MCP App testing, and a webServer entry to launch the inspector backend.
+ */
+import { getPortSync } from '../get-port.mjs';
+
+/**
+ * @param {Object} options
+ * @param {string[]} options.hosts - Host shells to create projects for
+ * @param {string} options.testDir - Test directory
+ * @param {Object} options.webServer - { command, healthUrl }
+ * @param {number} options.port - Inspector port
+ * @param {Object} [options.use] - Additional Playwright `use` options
+ * @param {string} [options.globalSetup] - Global setup file path
+ * @returns {import('@playwright/test').PlaywrightTestConfig}
+ */
+export function createBaseConfig({ hosts, testDir, webServer, port, use, globalSetup }) {
+  return {
+    ...(globalSetup ? { globalSetup } : {}),
+    testDir,
+    fullyParallel: true,
+    forbidOnly: !!process.env.CI,
+    retries: process.env.CI ? 2 : 1,
+    // Limit workers to avoid overwhelming the double-iframe sandbox proxy.
+    workers: process.env.CI ? 1 : 2,
+    reporter: 'list',
+    use: {
+      baseURL: `http://localhost:${port}`,
+      trace: 'on-first-retry',
+      ...use,
+    },
+    projects: hosts.map((host) => ({ name: host })),
+    webServer: {
+      command: webServer.command,
+      url: webServer.healthUrl,
+      reuseExistingServer: !process.env.CI,
+      timeout: 60_000,
+    },
+  };
+}
+
+/**
+ * Resolve ports for the inspector and sandbox proxy.
+ * Respects env vars for CI where validate.mjs assigns unique ports.
+ */
+export function resolvePorts() {
+  const port = parsePort(process.env.SUNPEAK_TEST_PORT) ?? getPortSync(6776);
+  const sandboxPort = parsePort(process.env.SUNPEAK_SANDBOX_PORT) ?? getPortSync(24680);
+  return { port, sandboxPort };
+}
+
+/** Parse a port string, returning the number or null if invalid/absent. */
+function parsePort(value) {
+  if (value == null) return null;
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : null;
+}

package/bin/lib/test/matchers.mjs

@@ -0,0 +1,99 @@
+/**
+ * MCP-native custom matchers for Playwright's expect.
+ *
+ * These matchers operate on ToolResult objects returned by mcp.callTool()
+ * and provide MCP-concept-native assertions.
+ */
+
+/**
+ * Register MCP matchers on a Playwright expect instance.
+ * @param {import('@playwright/test').Expect} expect
+ */
+export function registerMatchers(expect) {
+  expect.extend({
+    /**
+     * Assert that a tool result is an error.
+     * Usage: expect(result).toBeError()
+     */
+    toBeError(received) {
+      const pass = received?.isError === true;
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected tool result not to be an error, but it was`
+            : `Expected tool result to be an error, but isError was ${received?.isError}`,
+      };
+    },
+
+    /**
+     * Assert that any content item's text contains the given string.
+     * Usage: expect(result).toHaveTextContent('temperature')
+     */
+    toHaveTextContent(received, expected) {
+      const content = received?.content || [];
+      const texts = content
+        .filter((c) => c.type === 'text' && typeof c.text === 'string')
+        .map((c) => c.text);
+      const pass = texts.some((t) => t.includes(expected));
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected tool result not to contain text "${expected}", but found it`
+            : `Expected tool result to contain text "${expected}" in content items.\nFound texts: ${JSON.stringify(texts)}`,
+      };
+    },
+
+    /**
+     * Assert that structuredContent matches the expected shape (deep partial match).
+     * Usage: expect(result).toHaveStructuredContent({ type: 'weather' })
+     */
+    toHaveStructuredContent(received, expected) {
+      const sc = received?.structuredContent;
+      const pass = sc !== undefined && deepPartialMatch(sc, expected);
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected structuredContent not to match, but it did`
+            : `Expected structuredContent to match ${JSON.stringify(expected)}, got ${JSON.stringify(sc)}`,
+      };
+    },
+
+    /**
+     * Assert that content array contains an item of the given type.
+     * Usage: expect(result).toHaveContentType('image')
+     */
+    toHaveContentType(received, expectedType) {
+      const content = received?.content || [];
+      const types = content.map((c) => c.type);
+      const pass = types.includes(expectedType);
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected content not to include type "${expectedType}", but it did`
+            : `Expected content to include type "${expectedType}". Found types: ${JSON.stringify(types)}`,
+      };
+    },
+  });
+}
+
+/**
+ * Deep partial match: every key in `expected` must exist in `actual` and match.
+ * Extra keys in `actual` are allowed.
+ */
+function deepPartialMatch(actual, expected) {
+  if (expected === actual) return true;
+  if (expected === null || actual === null) return expected === actual;
+  if (typeof expected !== 'object' || typeof actual !== 'object') return expected === actual;
+  if (Array.isArray(expected)) {
+    if (!Array.isArray(actual)) return false;
+    if (expected.length !== actual.length) return false;
+    return expected.every((item, i) => deepPartialMatch(actual[i], item));
+  }
+  return Object.keys(expected).every(
+    (key) => key in actual && deepPartialMatch(actual[key], expected[key])
+  );
+}

package/bin/lib/test/test-config.d.mts

@@ -0,0 +1,44 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+
+/**
+ * MCP server connection configuration.
+ */
+export interface ServerConfig {
+  /** Server start command (e.g., 'python'). */
+  command?: string;
+  /** Command arguments (e.g., ['server.py']). */
+  args?: string[];
+  /** HTTP server URL (alternative to command/args). */
+  url?: string;
+  /** Environment variables for the server process. */
+  env?: Record<string, string>;
+}
+
+/**
+ * Configuration options for sunpeak test config.
+ */
+export interface TestConfigOptions {
+  /**
+   * MCP server connection. Omit for sunpeak framework projects (auto-detected).
+   * Required for external MCP servers.
+   */
+  server?: ServerConfig;
+  /** Host shells to test against (default: ['chatgpt', 'claude']). */
+  hosts?: string[];
+  /** Test directory (default: 'tests/e2e' for sunpeak, '.' for external). */
+  testDir?: string;
+  /** Simulations directory for mock data. */
+  simulationsDir?: string;
+  /** Global setup file path. */
+  globalSetup?: string;
+  /** Additional Playwright `use` options. */
+  use?: Record<string, unknown>;
+}
+
+/**
+ * Create a Playwright config for testing MCP servers.
+ *
+ * Auto-detects sunpeak projects and starts `sunpeak dev` as the backend.
+ * For external servers, starts `sunpeak inspect` with the provided server config.
+ */
+export declare function defineConfig(options?: TestConfigOptions): PlaywrightTestConfig;

package/bin/lib/test/test-config.mjs

@@ -0,0 +1,123 @@
+/**
+ * Playwright config factory for MCP server testing.
+ *
+ * Auto-detects project type:
+ * - sunpeak framework projects: starts `sunpeak dev` as the backend
+ * - External MCP servers: starts `sunpeak inspect` as the backend
+ *
+ * Usage (sunpeak project):
+ *   import { defineConfig } from 'sunpeak/test/config';
+ *   export default defineConfig();
+ *
+ * Usage (external server):
+ *   import { defineConfig } from 'sunpeak/test/config';
+ *   export default defineConfig({
+ *     server: { command: 'python', args: ['server.py'] },
+ *   });
+ */
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { createBaseConfig, resolvePorts } from './base-config.mjs';
+
+/**
+ * @param {Object} [options]
+ * @param {Object} [options.server] - MCP server connection (omit for sunpeak projects)
+ * @param {string} [options.server.command] - Server start command
+ * @param {string[]} [options.server.args] - Command arguments
+ * @param {string} [options.server.url] - HTTP server URL (alternative to command)
+ * @param {Record<string, string>} [options.server.env] - Environment variables
+ * @param {string[]} [options.hosts] - Host shells to test (default: ['chatgpt', 'claude'])
+ * @param {string} [options.testDir] - Test directory
+ * @param {string} [options.simulationsDir] - Simulations directory for mock data
+ * @param {string} [options.globalSetup] - Global setup file path
+ * @param {Object} [options.use] - Additional Playwright `use` options
+ * @returns {import('@playwright/test').PlaywrightTestConfig}
+ */
+export function defineConfig(options = {}) {
+  const {
+    server,
+    hosts = ['chatgpt', 'claude'],
+    testDir,
+    simulationsDir,
+    globalSetup,
+    use: userUse,
+  } = options;
+
+  const { port, sandboxPort } = resolvePorts();
+  const isSunpeakProject = !server && detectSunpeakProject();
+
+  const resolvedTestDir = testDir || (isSunpeakProject ? 'tests/e2e' : '.');
+
+  let command;
+  if (server) {
+    // External MCP server mode
+    command = buildInspectCommand({ server, port, sandboxPort, simulationsDir });
+  } else if (isSunpeakProject) {
+    // sunpeak framework project mode
+    command = `PORT=${port} SUNPEAK_SANDBOX_PORT=${sandboxPort} pnpm dev`;
+  } else {
+    throw new Error(
+      'defineConfig: either provide a `server` option or run from a sunpeak project directory.'
+    );
+  }
+
+  return createBaseConfig({
+    hosts,
+    testDir: resolvedTestDir,
+    port,
+    use: userUse,
+    globalSetup,
+    webServer: {
+      command,
+      healthUrl: `http://localhost:${port}/health`,
+    },
+  });
+}
+
+/**
+ * Detect if the current directory is a sunpeak framework project.
+ */
+function detectSunpeakProject() {
+  const pkgPath = join(process.cwd(), 'package.json');
+  if (!existsSync(pkgPath)) return false;
+  try {
+    const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+    const deps = { ...pkg.dependencies, ...pkg.devDependencies };
+    return 'sunpeak' in deps;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Build the `sunpeak inspect` command for external MCP servers.
+ */
+function buildInspectCommand({ server, port, sandboxPort, simulationsDir }) {
+  const parts = [`SUNPEAK_SANDBOX_PORT=${sandboxPort}`];
+
+  if (server.env) {
+    for (const [key, value] of Object.entries(server.env)) {
+      parts.push(`${key}=${value}`);
+    }
+  }
+
+  parts.push('npx sunpeak inspect');
+
+  if (server.url) {
+    parts.push(`--server ${server.url}`);
+  } else if (server.command) {
+    const cmd = server.args
+      ? `${server.command} ${server.args.join(' ')}`
+      : server.command;
+    // Quote the command if it contains spaces
+    parts.push(`--server "${cmd}"`);
+  }
+
+  if (simulationsDir) {
+    parts.push(`--simulations ${simulationsDir}`);
+  }
+
+  parts.push(`--port ${port}`);
+
+  return parts.join(' ');
+}

package/bin/lib/test/test-fixtures.d.mts

@@ -0,0 +1,96 @@
+import type { Page, FrameLocator, TestType, Expect } from '@playwright/test';
+
+/**
+ * Result from calling an MCP tool via the inspector.
+ */
+export interface ToolResult {
+  /** Raw MCP content items from the tool response. */
+  content: Array<{ type: string; text?: string; [key: string]: unknown }>;
+  /** Structured content from the tool response. */
+  structuredContent?: unknown;
+  /** Whether the tool returned an error. */
+  isError: boolean;
+  /**
+   * Get a FrameLocator for the rendered resource UI.
+   * Handles the double-iframe traversal automatically.
+   */
+  app(): FrameLocator;
+}
+
+/**
+ * Options for callTool().
+ */
+export interface CallToolOptions {
+  /** Color theme for the inspector. */
+  theme?: 'light' | 'dark';
+  /** Display mode for the resource. */
+  displayMode?: 'inline' | 'pip' | 'fullscreen';
+  /** Use production resource builds instead of HMR. */
+  prodResources?: boolean;
+  /** Additional inspector URL parameters. */
+  [key: string]: unknown;
+}
+
+/**
+ * MCP test fixture for testing MCP servers via the inspector.
+ */
+export interface McpFixture {
+  /** The underlying Playwright Page. */
+  page: Page;
+  /** Current host ID (from Playwright project name). */
+  host: string;
+
+  /**
+   * Call a tool and get the rendered result.
+   * Navigates the inspector, waits for the resource to render,
+   * and returns a ToolResult for assertions.
+   */
+  callTool(
+    name: string,
+    input?: Record<string, unknown>,
+    options?: CallToolOptions
+  ): Promise<ToolResult>;
+
+  /**
+   * Navigate to a tool with no mock data ("Press Run" state).
+   */
+  openTool(name: string, options?: { theme?: 'light' | 'dark' }): Promise<void>;
+
+  /**
+   * Click the Run button and return the rendered result.
+   */
+  runTool(): Promise<ToolResult>;
+
+  /** Change the theme via the sidebar toggle. */
+  setTheme(theme: 'light' | 'dark'): Promise<void>;
+
+  /** Change the display mode via the sidebar buttons. */
+  setDisplayMode(mode: 'inline' | 'pip' | 'fullscreen'): Promise<void>;
+}
+
+/**
+ * Extended Playwright test with `mcp` fixture.
+ */
+export declare const test: TestType<{ mcp: McpFixture }, {}>;
+
+/**
+ * Extended Playwright expect with MCP-native matchers.
+ */
+export declare const expect: Expect<{
+  /**
+   * Assert that a tool result is an error.
+   */
+  toBeError(): void;
+  /**
+   * Assert that any content item's text contains the given string.
+   */
+  toHaveTextContent(text: string): void;
+  /**
+   * Assert that structuredContent matches the expected shape.
+   */
+  toHaveStructuredContent(shape: unknown): void;
+  /**
+   * Assert that content array contains an item of the given type.
+   */
+  toHaveContentType(type: string): void;
+}>;

package/bin/lib/test/test-fixtures.mjs

@@ -0,0 +1,189 @@
+/**
+ * MCP-first Playwright fixtures for testing MCP servers.
+ *
+ * Provides an `mcp` fixture that abstracts the inspector, double-iframe
+ * traversal, URL construction, and host selection. Tests read like MCP
+ * operations, not browser automation.
+ *
+ * Usage:
+ *   import { test, expect } from 'sunpeak/test';
+ *
+ *   test('weather tool', async ({ mcp }) => {
+ *     const result = await mcp.callTool('get-weather', { city: 'SF' });
+ *     expect(result).not.toBeError();
+ *     expect(result).toHaveTextContent('temperature');
+ *
+ *     const app = result.app();
+ *     await expect(app.getByText('San Francisco')).toBeVisible();
+ *   });
+ */
+import { resolvePlaywrightESM } from '../live/utils.mjs';
+import { registerMatchers } from './matchers.mjs';
+
+const projectRoot = process.env.SUNPEAK_PROJECT_ROOT || process.cwd();
+const { test: base, expect } = await resolvePlaywrightESM(projectRoot);
+
+// Register MCP-native matchers
+registerMatchers(expect);
+
+/**
+ * Build an inspector URL path with query parameters.
+ * Inlined to avoid importing from dist (which pulls in React).
+ */
+function buildInspectorUrl(params) {
+  const sp = new URLSearchParams();
+  for (const [key, value] of Object.entries(params)) {
+    if (value !== undefined && value !== null) {
+      sp.set(key, String(value));
+    }
+  }
+  // Always disable dev overlay in tests
+  sp.set('devOverlay', 'false');
+  const qs = sp.toString();
+  return qs ? `/?${qs}` : '/';
+}
+
+/**
+ * Resolve the host ID from the Playwright project name.
+ */
+function resolveHostId(projectName) {
+  if (!projectName) return 'chatgpt';
+  if (projectName.startsWith('chatgpt')) return 'chatgpt';
+  if (projectName.startsWith('claude')) return 'claude';
+  return projectName;
+}
+
+/**
+ * Create a ToolResult wrapper around the inspector's rendered state.
+ */
+function createToolResult(page, resultData) {
+  return {
+    content: resultData?.content || [],
+    structuredContent: resultData?.structuredContent,
+    isError: resultData?.isError || false,
+
+    /**
+     * Get a FrameLocator for the rendered resource UI.
+     * Handles the double-iframe traversal (outer sandbox proxy + inner app).
+     * Returns the locator regardless — Playwright will throw with a clear
+     * error if no iframe exists when you interact with it.
+     */
+    app() {
+      return page.frameLocator('iframe').frameLocator('iframe');
+    },
+  };
+}
+
+const test = base.extend({
+  mcp: async ({ page }, use, testInfo) => {
+    const host = resolveHostId(testInfo.project.name);
+
+    const fixture = {
+      page,
+      host,
+
+      /**
+       * Call a tool and get the rendered result.
+       *
+       * For sunpeak projects: navigates to the matching simulation (simulation
+       * fixture data including toolInput is served by sunpeak dev).
+       * For external servers: navigates to the matching simulation created by
+       * inspectServer from discovered tools.
+       *
+       * Note: The `input` parameter is accepted for API consistency and future
+       * use but is not currently passed to the inspector. Simulation fixture
+       * data provides the tool input for rendering.
+       *
+       * @param {string} name - Tool/simulation name (e.g., 'show-albums')
+       * @param {Record<string, unknown>} [_input] - Reserved for future use
+       * @param {Object} [options] - Display options
+       * @returns {Promise<ToolResult>}
+       */
+      async callTool(name, _input, options = {}) {
+        const { theme, displayMode, ...rest } = options;
+
+        const params = {
+          simulation: name,
+          host,
+          ...(theme && { theme }),
+          ...(displayMode && { displayMode }),
+          ...rest,
+        };
+
+        await page.goto(buildInspectorUrl(params));
+
+        // Wait for the resource iframe to have content
+        try {
+          const frame = page.frameLocator('iframe').frameLocator('iframe');
+          await frame.locator('body').waitFor({ state: 'attached', timeout: 15_000 });
+        } catch {
+          // Tool may not have a resource (no UI)
+        }
+
+        return createToolResult(page, {
+          content: [],
+          structuredContent: undefined,
+          isError: false,
+        });
+      },
+
+      /**
+       * Navigate to a tool with no mock data ("Press Run" state).
+       * Use for testing the empty/loading state before a tool is executed.
+       */
+      async openTool(name, options = {}) {
+        const { theme, ...rest } = options;
+        const params = {
+          tool: name,
+          host,
+          ...(theme && { theme }),
+          ...rest,
+        };
+        await page.goto(buildInspectorUrl(params));
+        await page.locator('#root').waitFor({ state: 'attached' });
+      },
+
+      /**
+       * Click the Run button and wait for the resource to render.
+       * Use after openTool() in Prod Tools mode.
+       */
+      async runTool() {
+        await page.locator('button:has-text("Run")').click();
+        await page.locator('iframe').waitFor({ state: 'attached', timeout: 30_000 });
+        return createToolResult(page, {
+          content: [],
+          structuredContent: undefined,
+          isError: false,
+        });
+      },
+
+      /**
+       * Change the theme via the sidebar toggle.
+       */
+      async setTheme(theme) {
+        const label = theme === 'light' ? 'Light' : 'Dark';
+        const button = page.locator(`button:has-text("${label}")`);
+        if (await button.isVisible().catch(() => false)) {
+          await button.click();
+          // Wait for theme to propagate to the iframe
+          await page.waitForTimeout(300);
+        }
+      },
+
+      /**
+       * Change the display mode via the sidebar buttons.
+       */
+      async setDisplayMode(mode) {
+        const labels = { inline: 'Inline', pip: 'PiP', fullscreen: 'Full' };
+        const label = labels[mode] || mode;
+        await page.locator(`button:has-text("${label}")`).click();
+        // Wait for display mode transition
+        await page.waitForTimeout(500);
+      },
+    };
+
+    await use(fixture);
+  },
+});
+
+export { test, expect };

package/bin/sunpeak.js

@@ -37,7 +37,7 @@ function getVersion() {
   }
 
   // Commands that don't require a package.json
-  const standaloneCommands = ['new', 'upgrade', 'inspect', 'help', undefined];
+  const standaloneCommands = ['new', 'upgrade', 'inspect', 'test', 'help', undefined];
 
   if (command && !standaloneCommands.includes(command)) {
     checkPackageJson();
@@ -79,6 +79,13 @@ function getVersion() {
       }
       break;
 
+    case 'test':
+      {
+        const { runTest } = await import(join(COMMANDS_DIR, 'test.mjs'));
+        await runTest(args);
+      }
+      break;
+
     case 'upgrade':
       {
         const { upgrade } = await import(join(COMMANDS_DIR, 'upgrade.mjs'));
@@ -100,16 +107,22 @@ function getVersion() {
 Install:
   pnpm add -g sunpeak
 
-Usage:
+Testing (works with any MCP server):
+  sunpeak inspect          Inspect any MCP server in the inspector
+    --server, -s <url|cmd> MCP server URL or stdio command (required)
+    --simulations <dir>    Simulation JSON directory
+  sunpeak test             Run e2e tests against the inspector
+    init                   Scaffold test infrastructure into a project
+    --unit                 Run unit tests (vitest)
+    --live                 Run live tests against real hosts
+
+App framework (for sunpeak projects):
   sunpeak new [name] [resources]  Create a new project
   sunpeak dev              Start dev server + inspector + MCP endpoint
     --no-begging           Suppress GitHub star message
   sunpeak build            Build resources + tools for production
   sunpeak start            Start production MCP server
     --port, -p             Server port (default: 8000, or PORT env)
-  sunpeak inspect          Inspect any MCP server in the inspector
-    --server, -s <url|cmd> MCP server URL or stdio command (required)
-    --simulations <dir>    Simulation JSON directory
   sunpeak upgrade          Upgrade sunpeak to latest version
   sunpeak --version        Show version number