sunpeak 0.18.14 → 0.19.2

package/bin/lib/inspect/inspect-config.mjs

@@ -2,16 +2,19 @@
  * Playwright config factory for inspect mode (BYOS — Bring Your Own Server).
  *
  * Generates a complete Playwright config that starts `sunpeak inspect` as the
- * webServer and runs e2e tests against the inspector. Follows the same pattern
- * as `defineLiveConfig` for live tests.
+ * webServer and runs e2e tests against the inspector.
  *
  * Usage in playwright.config.ts:
  *   import { defineInspectConfig } from 'sunpeak/test/inspect/config';
  *   export default defineInspectConfig({
  *     server: 'http://localhost:8000/mcp',
  *   });
+ *
+ * Note: For new projects, prefer `defineConfig` from 'sunpeak/test/config'
+ * which auto-detects the project type and handles both sunpeak projects
+ * and external servers.
  */
-import { getPortSync } from '../get-port.mjs';
+import { createBaseConfig, resolvePorts } from '../test/base-config.mjs';
 
 /**
  * Create a complete Playwright config for testing an external MCP server.
@@ -19,7 +22,7 @@ import { getPortSync } from '../get-port.mjs';
  * @param {Object} options
  * @param {string} options.server - MCP server URL or stdio command (required)
  * @param {string} [options.testDir='tests/e2e'] - Test directory
- * @param {string} [options.simulationsDir='tests/simulations'] - Simulation JSON directory
+ * @param {string} [options.simulationsDir] - Simulation JSON directory
  * @param {string[]} [options.hosts=['chatgpt', 'claude']] - Host shells to test
  * @param {string} [options.name] - App name in inspector chrome
  * @param {Object} [options.use] - Additional Playwright `use` options
@@ -33,18 +36,19 @@ export function defineInspectConfig(options) {
     hosts = ['chatgpt', 'claude'],
     name,
     use: userUse,
+    visual,
   } = options;
 
   if (!server) {
     throw new Error('defineInspectConfig: `server` option is required');
   }
 
-  const port = Number(process.env.SUNPEAK_TEST_PORT) || getPortSync(6776);
-  const sandboxPort = Number(process.env.SUNPEAK_SANDBOX_PORT) || getPortSync(24680);
+  const { port, sandboxPort } = resolvePorts();
 
   // Build the sunpeak inspect command
   const serverArg = server.includes(' ') ? `"${server}"` : server;
   const command = [
+    `SUNPEAK_SANDBOX_PORT=${sandboxPort}`,
     'npx sunpeak inspect',
     `--server ${serverArg}`,
     ...(simulationsDir ? [`--simulations ${simulationsDir}`] : []),
@@ -52,25 +56,15 @@ export function defineInspectConfig(options) {
     ...(name ? [`--name "${name}"`] : []),
   ].join(' ');
 
-  return {
+  return createBaseConfig({
+    hosts,
     testDir,
-    fullyParallel: true,
-    forbidOnly: !!process.env.CI,
-    retries: process.env.CI ? 2 : 1,
-    // Limit workers to avoid overwhelming the double-iframe sandbox proxy.
-    workers: process.env.CI ? 1 : 2,
-    reporter: 'list',
-    use: {
-      baseURL: `http://localhost:${port}`,
-      trace: 'on-first-retry',
-      ...userUse,
-    },
-    projects: hosts.map((host) => ({ name: host })),
+    port,
+    use: userUse,
+    visual,
     webServer: {
-      command: `SUNPEAK_SANDBOX_PORT=${sandboxPort} ${command}`,
-      url: `http://localhost:${port}/health`,
-      reuseExistingServer: !process.env.CI,
-      timeout: 60_000,
+      command,
+      healthUrl: `http://localhost:${port}/health`,
     },
-  };
+  });
 }

package/bin/lib/test/base-config.mjs

@@ -0,0 +1,75 @@
+/**
+ * Shared Playwright config builder used by both defineConfig() (sunpeak projects)
+ * and defineInspectConfig() (external MCP servers).
+ *
+ * Produces a config with per-host Playwright projects, sensible defaults for
+ * MCP App testing, and a webServer entry to launch the inspector backend.
+ */
+import { getPortSync } from '../get-port.mjs';
+
+/**
+ * @param {Object} options
+ * @param {string[]} options.hosts - Host shells to create projects for
+ * @param {string} options.testDir - Test directory
+ * @param {Object} options.webServer - { command, healthUrl }
+ * @param {number} options.port - Inspector port
+ * @param {Object} [options.use] - Additional Playwright `use` options
+ * @param {string} [options.globalSetup] - Global setup file path
+ * @returns {import('@playwright/test').PlaywrightTestConfig}
+ */
+export function createBaseConfig({ hosts, testDir, webServer, port, use, globalSetup, visual }) {
+  // Separate snapshot path from other visual options passed to expect.toHaveScreenshot
+  const { snapshotPathTemplate, ...toHaveScreenshotDefaults } = visual ?? {};
+
+  return {
+    ...(globalSetup ? { globalSetup } : {}),
+    testDir,
+    fullyParallel: true,
+    forbidOnly: !!process.env.CI,
+    retries: process.env.CI ? 2 : 1,
+    // Limit workers to avoid overwhelming the double-iframe sandbox proxy.
+    workers: process.env.CI ? 1 : 2,
+    reporter: 'list',
+    // Only override snapshot path when visual config is provided, to avoid
+    // changing Playwright's default for projects that don't use visual testing.
+    ...(visual
+      ? {
+          snapshotPathTemplate:
+            snapshotPathTemplate ??
+            '{testDir}/__screenshots__/{projectName}/{testFilePath}/{arg}{ext}',
+        }
+      : {}),
+    ...(Object.keys(toHaveScreenshotDefaults).length > 0
+      ? { expect: { toHaveScreenshot: toHaveScreenshotDefaults } }
+      : {}),
+    use: {
+      baseURL: `http://localhost:${port}`,
+      trace: 'on-first-retry',
+      ...use,
+    },
+    projects: hosts.map((host) => ({ name: host })),
+    webServer: {
+      command: webServer.command,
+      url: webServer.healthUrl,
+      reuseExistingServer: !process.env.CI,
+      timeout: 60_000,
+    },
+  };
+}
+
+/**
+ * Resolve ports for the inspector and sandbox proxy.
+ * Respects env vars for CI where validate.mjs assigns unique ports.
+ */
+export function resolvePorts() {
+  const port = parsePort(process.env.SUNPEAK_TEST_PORT) ?? getPortSync(6776);
+  const sandboxPort = parsePort(process.env.SUNPEAK_SANDBOX_PORT) ?? getPortSync(24680);
+  return { port, sandboxPort };
+}
+
+/** Parse a port string, returning the number or null if invalid/absent. */
+function parsePort(value) {
+  if (value == null) return null;
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : null;
+}

package/bin/lib/test/matchers.mjs

@@ -0,0 +1,99 @@
+/**
+ * MCP-native custom matchers for Playwright's expect.
+ *
+ * These matchers operate on ToolResult objects returned by mcp.callTool()
+ * and provide MCP-concept-native assertions.
+ */
+
+/**
+ * Register MCP matchers on a Playwright expect instance.
+ * @param {import('@playwright/test').Expect} expect
+ */
+export function registerMatchers(expect) {
+  expect.extend({
+    /**
+     * Assert that a tool result is an error.
+     * Usage: expect(result).toBeError()
+     */
+    toBeError(received) {
+      const pass = received?.isError === true;
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected tool result not to be an error, but it was`
+            : `Expected tool result to be an error, but isError was ${received?.isError}`,
+      };
+    },
+
+    /**
+     * Assert that any content item's text contains the given string.
+     * Usage: expect(result).toHaveTextContent('temperature')
+     */
+    toHaveTextContent(received, expected) {
+      const content = received?.content || [];
+      const texts = content
+        .filter((c) => c.type === 'text' && typeof c.text === 'string')
+        .map((c) => c.text);
+      const pass = texts.some((t) => t.includes(expected));
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected tool result not to contain text "${expected}", but found it`
+            : `Expected tool result to contain text "${expected}" in content items.\nFound texts: ${JSON.stringify(texts)}`,
+      };
+    },
+
+    /**
+     * Assert that structuredContent matches the expected shape (deep partial match).
+     * Usage: expect(result).toHaveStructuredContent({ type: 'weather' })
+     */
+    toHaveStructuredContent(received, expected) {
+      const sc = received?.structuredContent;
+      const pass = sc !== undefined && deepPartialMatch(sc, expected);
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected structuredContent not to match, but it did`
+            : `Expected structuredContent to match ${JSON.stringify(expected)}, got ${JSON.stringify(sc)}`,
+      };
+    },
+
+    /**
+     * Assert that content array contains an item of the given type.
+     * Usage: expect(result).toHaveContentType('image')
+     */
+    toHaveContentType(received, expectedType) {
+      const content = received?.content || [];
+      const types = content.map((c) => c.type);
+      const pass = types.includes(expectedType);
+      return {
+        pass,
+        message: () =>
+          pass
+            ? `Expected content not to include type "${expectedType}", but it did`
+            : `Expected content to include type "${expectedType}". Found types: ${JSON.stringify(types)}`,
+      };
+    },
+  });
+}
+
+/**
+ * Deep partial match: every key in `expected` must exist in `actual` and match.
+ * Extra keys in `actual` are allowed.
+ */
+function deepPartialMatch(actual, expected) {
+  if (expected === actual) return true;
+  if (expected === null || actual === null) return expected === actual;
+  if (typeof expected !== 'object' || typeof actual !== 'object') return expected === actual;
+  if (Array.isArray(expected)) {
+    if (!Array.isArray(actual)) return false;
+    if (expected.length !== actual.length) return false;
+    return expected.every((item, i) => deepPartialMatch(actual[i], item));
+  }
+  return Object.keys(expected).every(
+    (key) => key in actual && deepPartialMatch(actual[key], expected[key])
+  );
+}

package/bin/lib/test/test-config.d.mts

@@ -0,0 +1,66 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+
+/**
+ * MCP server connection configuration.
+ */
+export interface ServerConfig {
+  /** Server start command (e.g., 'python'). */
+  command?: string;
+  /** Command arguments (e.g., ['server.py']). */
+  args?: string[];
+  /** HTTP server URL (alternative to command/args). */
+  url?: string;
+  /** Environment variables for the server process. */
+  env?: Record<string, string>;
+}
+
+/**
+ * Visual regression testing configuration.
+ *
+ * All fields except `snapshotPathTemplate` are forwarded to Playwright's
+ * `expect.toHaveScreenshot` config. See Playwright docs for the full set
+ * of options (threshold, maxDiffPixelRatio, maxDiffPixels, animations, etc.).
+ */
+export interface VisualConfig {
+  /** Snapshot directory path template. Default: '{testDir}/__screenshots__/{projectName}/{testFilePath}/{arg}{ext}'. */
+  snapshotPathTemplate?: string;
+  /** Pixel comparison threshold (0-1). */
+  threshold?: number;
+  /** Maximum allowed ratio of differing pixels (0-1). */
+  maxDiffPixelRatio?: number;
+  /** Absolute count of allowed different pixels. */
+  maxDiffPixels?: number;
+  /** Any other Playwright toHaveScreenshot options applied as project-wide defaults. */
+  [key: string]: unknown;
+}
+
+/**
+ * Configuration options for sunpeak test config.
+ */
+export interface TestConfigOptions {
+  /**
+   * MCP server connection. Omit for sunpeak framework projects (auto-detected).
+   * Required for external MCP servers.
+   */
+  server?: ServerConfig;
+  /** Host shells to test against (default: ['chatgpt', 'claude']). */
+  hosts?: string[];
+  /** Test directory (default: 'tests/e2e' for sunpeak, '.' for external). */
+  testDir?: string;
+  /** Simulations directory for mock data. */
+  simulationsDir?: string;
+  /** Global setup file path. */
+  globalSetup?: string;
+  /** Additional Playwright `use` options. */
+  use?: Record<string, unknown>;
+  /** Visual regression testing configuration. */
+  visual?: VisualConfig;
+}
+
+/**
+ * Create a Playwright config for testing MCP servers.
+ *
+ * Auto-detects sunpeak projects and starts `sunpeak dev` as the backend.
+ * For external servers, starts `sunpeak inspect` with the provided server config.
+ */
+export declare function defineConfig(options?: TestConfigOptions): PlaywrightTestConfig;

package/bin/lib/test/test-config.mjs

@@ -0,0 +1,125 @@
+/**
+ * Playwright config factory for MCP server testing.
+ *
+ * Auto-detects project type:
+ * - sunpeak framework projects: starts `sunpeak dev` as the backend
+ * - External MCP servers: starts `sunpeak inspect` as the backend
+ *
+ * Usage (sunpeak project):
+ *   import { defineConfig } from 'sunpeak/test/config';
+ *   export default defineConfig();
+ *
+ * Usage (external server):
+ *   import { defineConfig } from 'sunpeak/test/config';
+ *   export default defineConfig({
+ *     server: { command: 'python', args: ['server.py'] },
+ *   });
+ */
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { createBaseConfig, resolvePorts } from './base-config.mjs';
+
+/**
+ * @param {Object} [options]
+ * @param {Object} [options.server] - MCP server connection (omit for sunpeak projects)
+ * @param {string} [options.server.command] - Server start command
+ * @param {string[]} [options.server.args] - Command arguments
+ * @param {string} [options.server.url] - HTTP server URL (alternative to command)
+ * @param {Record<string, string>} [options.server.env] - Environment variables
+ * @param {string[]} [options.hosts] - Host shells to test (default: ['chatgpt', 'claude'])
+ * @param {string} [options.testDir] - Test directory
+ * @param {string} [options.simulationsDir] - Simulations directory for mock data
+ * @param {string} [options.globalSetup] - Global setup file path
+ * @param {Object} [options.use] - Additional Playwright `use` options
+ * @returns {import('@playwright/test').PlaywrightTestConfig}
+ */
+export function defineConfig(options = {}) {
+  const {
+    server,
+    hosts = ['chatgpt', 'claude'],
+    testDir,
+    simulationsDir,
+    globalSetup,
+    use: userUse,
+    visual,
+  } = options;
+
+  const { port, sandboxPort } = resolvePorts();
+  const isSunpeakProject = !server && detectSunpeakProject();
+
+  const resolvedTestDir = testDir || (isSunpeakProject ? 'tests/e2e' : '.');
+
+  let command;
+  if (server) {
+    // External MCP server mode
+    command = buildInspectCommand({ server, port, sandboxPort, simulationsDir });
+  } else if (isSunpeakProject) {
+    // sunpeak framework project mode
+    command = `PORT=${port} SUNPEAK_SANDBOX_PORT=${sandboxPort} pnpm dev`;
+  } else {
+    throw new Error(
+      'defineConfig: either provide a `server` option or run from a sunpeak project directory.'
+    );
+  }
+
+  return createBaseConfig({
+    hosts,
+    testDir: resolvedTestDir,
+    port,
+    use: userUse,
+    globalSetup,
+    visual,
+    webServer: {
+      command,
+      healthUrl: `http://localhost:${port}/health`,
+    },
+  });
+}
+
+/**
+ * Detect if the current directory is a sunpeak framework project.
+ */
+function detectSunpeakProject() {
+  const pkgPath = join(process.cwd(), 'package.json');
+  if (!existsSync(pkgPath)) return false;
+  try {
+    const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+    const deps = { ...pkg.dependencies, ...pkg.devDependencies };
+    return 'sunpeak' in deps;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Build the `sunpeak inspect` command for external MCP servers.
+ */
+function buildInspectCommand({ server, port, sandboxPort, simulationsDir }) {
+  const parts = [`SUNPEAK_SANDBOX_PORT=${sandboxPort}`];
+
+  if (server.env) {
+    for (const [key, value] of Object.entries(server.env)) {
+      parts.push(`${key}=${value}`);
+    }
+  }
+
+  parts.push('npx sunpeak inspect');
+
+  if (server.url) {
+    parts.push(`--server ${server.url}`);
+  } else if (server.command) {
+    const cmd = server.args
+      ? `${server.command} ${server.args.join(' ')}`
+      : server.command;
+    // Quote the command if it contains spaces
+    parts.push(`--server "${cmd}"`);
+  }
+
+  if (simulationsDir) {
+    parts.push(`--simulations ${simulationsDir}`);
+  }
+
+  parts.push(`--port ${port}`);
+
+  return parts.join(' ');
+}

package/bin/lib/test/test-fixtures.d.mts

@@ -0,0 +1,129 @@
+import type {
+  Page,
+  FrameLocator,
+  Locator,
+  TestType,
+  Expect,
+  PageAssertionsToHaveScreenshotOptions,
+} from '@playwright/test';
+
+/**
+ * Result from calling an MCP tool via the inspector.
+ */
+export interface ToolResult {
+  /** Raw MCP content items from the tool response. */
+  content: Array<{ type: string; text?: string; [key: string]: unknown }>;
+  /** Structured content from the tool response. */
+  structuredContent?: unknown;
+  /** Whether the tool returned an error. */
+  isError: boolean;
+  /**
+   * Get a FrameLocator for the rendered resource UI.
+   * Handles the double-iframe traversal automatically.
+   */
+  app(): FrameLocator;
+}
+
+/**
+ * Options for callTool().
+ */
+export interface CallToolOptions {
+  /** Color theme for the inspector. */
+  theme?: 'light' | 'dark';
+  /** Display mode for the resource. */
+  displayMode?: 'inline' | 'pip' | 'fullscreen';
+  /** Use production resource builds instead of HMR. */
+  prodResources?: boolean;
+  /** Additional inspector URL parameters. */
+  [key: string]: unknown;
+}
+
+/**
+ * Options for screenshot().
+ *
+ * Extends Playwright's toHaveScreenshot() options with sunpeak-specific
+ * `target` and `element` fields. All standard Playwright options (threshold,
+ * maxDiffPixelRatio, maxDiffPixels, mask, maskColor, animations, caret,
+ * fullPage, clip, scale, stylePath, omitBackground, timeout, etc.)
+ * are passed through directly.
+ */
+export interface ScreenshotOptions extends PageAssertionsToHaveScreenshotOptions {
+  /** What to screenshot: 'app' (inner iframe content) or 'page' (full inspector). Default: 'app'. */
+  target?: 'app' | 'page';
+  /** Specific locator to screenshot instead of the default target. */
+  element?: Locator;
+}
+
+/**
+ * MCP test fixture for testing MCP servers via the inspector.
+ */
+export interface McpFixture {
+  /** The underlying Playwright Page. */
+  page: Page;
+  /** Current host ID (from Playwright project name). */
+  host: string;
+
+  /**
+   * Call a tool and get the rendered result.
+   * Navigates the inspector, waits for the resource to render,
+   * and returns a ToolResult for assertions.
+   */
+  callTool(
+    name: string,
+    input?: Record<string, unknown>,
+    options?: CallToolOptions
+  ): Promise<ToolResult>;
+
+  /**
+   * Navigate to a tool with no mock data ("Press Run" state).
+   */
+  openTool(name: string, options?: { theme?: 'light' | 'dark' }): Promise<void>;
+
+  /**
+   * Click the Run button and return the rendered result.
+   */
+  runTool(): Promise<ToolResult>;
+
+  /** Change the theme via the sidebar toggle. */
+  setTheme(theme: 'light' | 'dark'): Promise<void>;
+
+  /** Change the display mode via the sidebar buttons. */
+  setDisplayMode(mode: 'inline' | 'pip' | 'fullscreen'): Promise<void>;
+
+  /**
+   * Take a screenshot and compare against a baseline.
+   * Only performs the comparison when visual testing is enabled
+   * (`sunpeak test --visual`). Silently skips otherwise.
+   *
+   * @param name - Snapshot name (auto-generated from test title if omitted)
+   * @param options - Screenshot and comparison options
+   */
+  screenshot(name?: string, options?: ScreenshotOptions): Promise<void>;
+}
+
+/**
+ * Extended Playwright test with `mcp` fixture.
+ */
+export declare const test: TestType<{ mcp: McpFixture }, {}>;
+
+/**
+ * Extended Playwright expect with MCP-native matchers.
+ */
+export declare const expect: Expect<{
+  /**
+   * Assert that a tool result is an error.
+   */
+  toBeError(): void;
+  /**
+   * Assert that any content item's text contains the given string.
+   */
+  toHaveTextContent(text: string): void;
+  /**
+   * Assert that structuredContent matches the expected shape.
+   */
+  toHaveStructuredContent(shape: unknown): void;
+  /**
+   * Assert that content array contains an item of the given type.
+   */
+  toHaveContentType(type: string): void;
+}>;