sunpeak 0.19.2 → 0.19.10

package/bin/commands/test.mjs

@@ -1,5 +1,5 @@
 import { spawn } from 'child_process';
-import { existsSync } from 'fs';
+import { existsSync, readFileSync, writeFileSync, unlinkSync } from 'fs';
 import { join } from 'path';
 
 /**
@@ -10,13 +10,15 @@ import { join } from 'path';
  * sunpeak test --unit          Run unit tests (vitest)
  * sunpeak test --e2e           Run e2e tests (Playwright)
  * sunpeak test --live          Run live tests against real hosts
+ * sunpeak test --eval          Run evals against LLM models
  * sunpeak test --visual        Run e2e tests with visual regression comparison
  * sunpeak test --visual --update  Update visual regression baselines
  * sunpeak test [pattern]       Pass through to the relevant runner
  *
- * Flags are additive: --unit --e2e --live runs all three.
+ * Flags are additive: --unit --e2e --live --eval runs all four.
  * --visual implies --e2e and enables screenshot comparison.
  * --update implies --visual.
+ * --eval and --live are never included in the default run (they cost money).
  */
 export async function runTest(args) {
   // Handle `sunpeak test init` subcommand
@@ -29,16 +31,17 @@ export async function runTest(args) {
   const isUnit = args.includes('--unit');
   const isE2e = args.includes('--e2e');
   const isLive = args.includes('--live');
+  const isEval = args.includes('--eval');
   let isVisual = args.includes('--visual');
   const isUpdate = args.includes('--update');
   const filteredArgs = args.filter(
-    (a) => !['--unit', '--e2e', '--live', '--visual', '--update'].includes(a)
+    (a) => !['--unit', '--e2e', '--live', '--eval', '--visual', '--update'].includes(a)
   );
 
   // --update implies --visual (no point updating without enabling visual)
   if (isUpdate) isVisual = true;
 
-  const hasAnyScope = isUnit || isE2e || isLive || isVisual;
+  const hasAnyScope = isUnit || isE2e || isLive || isEval || isVisual;
 
   // When extra args are present (file patterns, etc.) and no scope flags given,
   // default to e2e only — passing Playwright file patterns to vitest would fail.
@@ -47,9 +50,11 @@ export async function runTest(args) {
   // Determine which suites to run.
   // No scope flags → unit + e2e (unless extra args narrow to e2e).
   // --visual implies e2e.
+  // --eval and --live are never in the default run (they cost money).
   const runUnit = hasAnyScope ? isUnit : !hasExtraArgs;
   const runE2e = hasAnyScope ? (isE2e || isVisual) : true;
   const runLive = isLive;
+  const runEval = isEval;
 
   const results = [];
 
@@ -84,6 +89,11 @@ export async function runTest(args) {
     results.push({ suite: 'live', code });
   }
 
+  if (runEval) {
+    const code = await runEvals(filteredArgs);
+    results.push({ suite: 'eval', code });
+  }
+
   // Exit with the first non-zero code, or 0 if all passed
   const failed = results.find((r) => r.code !== 0);
   process.exit(failed ? failed.code : 0);
@@ -142,3 +152,346 @@ function findConfig(candidates) {
   }
   return null;
 }
+
+/**
+ * Detect if the current directory is a sunpeak app project (has tools to serve).
+ * Just having sunpeak as a dependency is not enough — the testing framework
+ * can be used with any MCP server. A sunpeak app project has src/tools/.
+ */
+function isSunpeakProject() {
+  const pkgPath = join(process.cwd(), 'package.json');
+  if (!existsSync(pkgPath)) return false;
+  try {
+    const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+    const deps = { ...pkg.dependencies, ...pkg.devDependencies };
+    if (!('sunpeak' in deps)) return false;
+    // Check for the sunpeak app convention (src/tools/ directory)
+    return existsSync(join(process.cwd(), 'src', 'tools'));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Start the dev server and wait for it to be ready.
+ * The dev server's inspector listens on inspectorPort (default 3000)
+ * and the MCP server listens on mcpPort (default 8000).
+ * @param {{ inspectorPort: number, mcpPort: number }} ports
+ * @returns {Promise<{ process: import('child_process').ChildProcess } | null>}
+ */
+async function startDevServer({ inspectorPort, mcpPort }) {
+  // Check if the MCP server is already running
+  try {
+    const resp = await fetch(`http://localhost:${mcpPort}/health`, {
+      signal: AbortSignal.timeout(2000),
+    });
+    if (resp.ok) return null; // Server already running
+  } catch {
+    // Not running, start it
+  }
+
+  console.log('Starting dev server for evals...');
+  const child = spawn(
+    'pnpm',
+    ['exec', 'sunpeak', 'dev', '--', '--prod-tools'],
+    {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      env: {
+        ...process.env,
+        PORT: String(inspectorPort),
+        SUNPEAK_MCP_PORT: String(mcpPort),
+        SUNPEAK_DEV_OVERLAY: 'false',
+      },
+    }
+  );
+
+  // Detect early exit (build errors, port conflicts, etc.)
+  let exited = false;
+  let stderrChunks = [];
+  child.stderr.on('data', (chunk) => stderrChunks.push(chunk.toString()));
+  child.on('exit', () => { exited = true; });
+
+  // Wait for the inspector health endpoint
+  const timeout = 60_000;
+  const start = Date.now();
+  while (Date.now() - start < timeout) {
+    if (exited) {
+      const stderr = stderrChunks.join('');
+      throw new Error(`Dev server exited before becoming ready.\n${stderr}`);
+    }
+    try {
+      const resp = await fetch(`http://localhost:${inspectorPort}/health`, {
+        signal: AbortSignal.timeout(2000),
+      });
+      if (resp.ok) {
+        // Also verify the MCP server is reachable
+        try {
+          await fetch(`http://localhost:${mcpPort}/health`, {
+            signal: AbortSignal.timeout(2000),
+          });
+        } catch {
+          // MCP server not ready yet, keep polling
+          await new Promise((r) => setTimeout(r, 500));
+          continue;
+        }
+        console.log('Dev server ready');
+        return { process: child };
+      }
+    } catch {
+      // Not ready yet
+    }
+    await new Promise((r) => setTimeout(r, 500));
+  }
+
+  child.kill();
+  throw new Error(`Dev server did not start within ${timeout / 1000}s`);
+}
+
+/**
+ * Load .env file from a directory into process.env (only sets vars not already set).
+ * @param {string} dir - Directory containing .env file
+ */
+function loadEnvFile(dir) {
+  const envPath = join(dir, '.env');
+  if (!existsSync(envPath)) return;
+
+  const content = readFileSync(envPath, 'utf-8');
+  for (const line of content.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eqIdx = trimmed.indexOf('=');
+    if (eqIdx === -1) continue;
+    const key = trimmed.slice(0, eqIdx).trim();
+    let value = trimmed.slice(eqIdx + 1).trim();
+    // Strip surrounding quotes
+    value = value.replace(/^(['"])(.*)\1$/, '$2');
+    if (key && value && !process.env[key]) {
+      process.env[key] = value;
+    }
+  }
+}
+
+/**
+ * Map of provider packages to their required env vars.
+ */
+const PROVIDER_ENV_VARS = {
+  '@ai-sdk/openai': 'OPENAI_API_KEY',
+  '@ai-sdk/anthropic': 'ANTHROPIC_API_KEY',
+  '@ai-sdk/google': 'GOOGLE_GENERATIVE_AI_API_KEY',
+};
+
+/**
+ * Map model ID prefix to provider package.
+ */
+function getProviderForModel(modelId) {
+  if (/^(gpt-|o[134]-|o[134]$|chatgpt-)/.test(modelId)) return '@ai-sdk/openai';
+  if (/^claude-/.test(modelId)) return '@ai-sdk/anthropic';
+  if (/^(gemini-|models\/gemini-)/.test(modelId)) return '@ai-sdk/google';
+  return null;
+}
+
+/**
+ * Validate that required API keys are set for the configured models.
+ * @param {string[]} models
+ * @returns {string[]} Array of warning messages (empty if all good)
+ */
+function validateApiKeys(models) {
+  const warnings = [];
+  const checked = new Set();
+  for (const modelId of models) {
+    const pkg = getProviderForModel(modelId);
+    if (!pkg || checked.has(pkg)) continue;
+    checked.add(pkg);
+    const envVar = PROVIDER_ENV_VARS[pkg];
+    if (envVar && !process.env[envVar]) {
+      warnings.push(`${envVar} not set (required for ${modelId}). Add it to tests/evals/.env or export it in your shell.`);
+    }
+  }
+  return warnings;
+}
+
+/**
+ * Run eval tests via vitest with the eval plugin.
+ */
+async function runEvals(args) {
+  const { dirname, resolve, basename } = await import('path');
+  const { fileURLToPath } = await import('url');
+
+  const __dirname = dirname(fileURLToPath(import.meta.url));
+  const evalDir = findEvalDir();
+
+  if (!evalDir) {
+    console.error('No eval directory found. Run "sunpeak test init" to scaffold eval tests.');
+    return 1;
+  }
+
+  const absEvalDir = resolve(process.cwd(), evalDir);
+
+  // Load .env from eval directory before anything else
+  loadEnvFile(absEvalDir);
+
+  // Find eval config and load it to check models before launching vitest
+  const configFile = findConfig([
+    join(evalDir, 'eval.config.ts'),
+    join(evalDir, 'eval.config.js'),
+  ]);
+
+  // Quick check: load the config to see if models are configured.
+  // We do this by reading the file and checking for non-empty models array.
+  // For a proper check, we'd need to evaluate the TS, but a quick heuristic
+  // is to warn if we can detect an empty array.
+  let configModels = null;
+  if (configFile) {
+    const configContent = readFileSync(configFile, 'utf-8');
+    // Check if models array appears to be empty (all lines commented out)
+    const modelsMatch = configContent.match(/models:\s*\[([\s\S]*?)\]/);
+    if (modelsMatch) {
+      const modelsBody = modelsMatch[1].trim();
+      // Remove comments and whitespace to check if anything is actually configured
+      const uncommented = modelsBody.replace(/\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').trim();
+      if (!uncommented || uncommented === ',') {
+        console.log(`\nNo models configured in ${configFile}.`);
+        console.log('To run evals, uncomment at least one model in your eval config:\n');
+        console.log('  models: [');
+        console.log("    'gpt-4o',              // requires OPENAI_API_KEY");
+        console.log("    'claude-sonnet-4-20250514',  // requires ANTHROPIC_API_KEY");
+        console.log("    'gemini-2.0-flash',    // requires GOOGLE_GENERATIVE_AI_API_KEY");
+        console.log('  ],\n');
+        return 0;
+      }
+      // Extract actual model strings for API key validation
+      const modelStrings = [...uncommented.matchAll(/['"]([^'"]+)['"]/g)].map(m => m[1]);
+      configModels = modelStrings;
+    }
+  }
+
+  // Validate API keys for configured models
+  if (configModels && configModels.length > 0) {
+    const warnings = validateApiKeys(configModels);
+    if (warnings.length > 0) {
+      console.log('');
+      for (const w of warnings) {
+        console.warn(`⚠  ${w}`);
+      }
+      console.log('');
+    }
+  }
+
+  // Use non-standard ports for auto-start to avoid conflicts with a running dev server.
+  const autoStartMcpPort = 18920;
+  const autoStartInspectorPort = 18921;
+  // Default server for non-sunpeak projects (standard MCP port)
+  const defaultMcpPort = 8000;
+  let devServerHandle = null;
+  let mcpPort = defaultMcpPort;
+
+  if (isSunpeakProject() && !process.env.SUNPEAK_EVAL_SERVER) {
+    // Auto-start dev server for sunpeak projects on non-standard ports
+    mcpPort = autoStartMcpPort;
+    try {
+      devServerHandle = await startDevServer({
+        inspectorPort: autoStartInspectorPort,
+        mcpPort: autoStartMcpPort,
+      });
+    } catch (err) {
+      console.error('Failed to start dev server:', err.message);
+      return 1;
+    }
+  } else if (!process.env.SUNPEAK_EVAL_SERVER) {
+    // For non-sunpeak projects, check if the server is running
+    try {
+      await fetch(`http://localhost:${defaultMcpPort}/health`, {
+        signal: AbortSignal.timeout(3000),
+      });
+    } catch {
+      const defaultServer = `http://localhost:${defaultMcpPort}/mcp`;
+      console.error(`MCP server not reachable at ${defaultServer}`);
+      console.error('Make sure your MCP server is running, or set the server URL in eval.config.ts.\n');
+      return 1;
+    }
+  }
+
+  const defaultServer = `http://localhost:${mcpPort}/mcp`;
+
+  // Use .ts extension so vitest handles TypeScript imports natively
+  const vitestConfigPath = join(absEvalDir, '.eval-vitest.config.ts');
+
+  const evalPluginPath = resolve(__dirname, '..', 'lib', 'eval', 'eval-vitest-plugin.mjs');
+  const evalReporterPath = resolve(__dirname, '..', 'lib', 'eval', 'eval-reporter.mjs');
+
+  // Clean up dev server and temp config
+  const cleanupResources = () => {
+    if (devServerHandle?.process) {
+      devServerHandle.process.kill();
+      devServerHandle = null;
+    }
+    try { unlinkSync(vitestConfigPath); } catch {}
+  };
+  // On unexpected signal, clean up and re-exit with conventional signal code
+  const onSignal = (signal) => {
+    cleanupResources();
+    process.exit(signal === 'SIGINT' ? 130 : 143);
+  };
+  const onSigInt = () => onSignal('SIGINT');
+  const onSigTerm = () => onSignal('SIGTERM');
+  process.on('SIGINT', onSigInt);
+  process.on('SIGTERM', onSigTerm);
+
+  // Resolve config import path relative to the generated config file location
+  const configImportPath = configFile
+    ? './' + basename(configFile)
+    : null;
+
+  // Generate a vitest config that loads the eval config and plugin
+  const vitestConfig = `
+import { defineConfig } from 'vitest/config';
+import { evalVitestPlugin } from ${JSON.stringify(evalPluginPath)};
+${configImportPath ? `import evalConfig from ${JSON.stringify(configImportPath)};` : 'const evalConfig = { models: [], defaults: {} };'}
+
+export default defineConfig({
+  test: {
+    globals: true,
+    include: ['**/*.eval.ts', '**/*.eval.js'],
+    reporters: ['default', ${JSON.stringify(evalReporterPath)}],
+    testTimeout: 600000,
+  },
+  plugins: [
+    evalVitestPlugin({
+      server: evalConfig.server || process.env.SUNPEAK_EVAL_SERVER || ${JSON.stringify(defaultServer)},
+      models: evalConfig.models || [],
+      defaults: evalConfig.defaults || {},
+    }),
+  ],
+});
+`;
+
+  writeFileSync(vitestConfigPath, vitestConfig);
+
+  try {
+    const code = await runChild(
+      'pnpm',
+      ['exec', 'vitest', 'run', '--config', vitestConfigPath, ...args]
+    );
+    return code;
+  } finally {
+    cleanupResources();
+    process.removeListener('SIGINT', onSigInt);
+    process.removeListener('SIGTERM', onSigTerm);
+  }
+}
+
+/**
+ * Find the eval tests directory.
+ */
+function findEvalDir() {
+  const candidates = [
+    'tests/evals',
+    'tests/sunpeak/evals',
+  ];
+
+  for (const candidate of candidates) {
+    const full = join(process.cwd(), candidate);
+    if (existsSync(full)) return candidate;
+  }
+  return null;
+}

package/bin/lib/eval/eval-reporter.mjs

@@ -0,0 +1,105 @@
+/**
+ * Custom vitest reporter for eval results.
+ *
+ * Parses __SUNPEAK_EVAL__ log lines from test console output and prints
+ * a statistical summary with pass/fail counts per model per case.
+ */
+
+const EVAL_PREFIX = '__SUNPEAK_EVAL__';
+
+export default class EvalReporter {
+  /** @type {import('./eval-types.d.mts').EvalCaseResult[]} */
+  results = [];
+  /** @type {boolean} */
+  printed = false;
+
+  /**
+   * Called for each console.log from test code.
+   * @param {{ content: string, type: 'stdout' | 'stderr' }} log
+   */
+  onUserConsoleLog(log) {
+    if (log.type !== 'stdout') return;
+    const idx = log.content.indexOf(EVAL_PREFIX);
+    if (idx === -1) return;
+
+    try {
+      const json = log.content.slice(idx + EVAL_PREFIX.length);
+      const data = JSON.parse(json);
+      if (data.type === 'eval-result') {
+        this.results.push(data);
+      }
+    } catch {
+      // Ignore parse errors
+    }
+  }
+
+  /**
+   * Called when the test run ends (vitest v4+).
+   */
+  onTestRunEnd() {
+    this.printSummary();
+  }
+
+  /**
+   * Fallback for older vitest versions.
+   */
+  onFinished() {
+    this.printSummary();
+  }
+
+  printSummary() {
+    if (this.results.length === 0 || this.printed) return;
+    this.printed = true;
+
+    console.log('\n' + '='.repeat(60));
+    console.log('Eval Results');
+    console.log('='.repeat(60));
+
+    // Group by case name
+    const byCase = new Map();
+    for (const r of this.results) {
+      if (!byCase.has(r.caseName)) byCase.set(r.caseName, []);
+      byCase.get(r.caseName).push(r);
+    }
+
+    let totalPassed = 0;
+    let totalRuns = 0;
+
+    for (const [caseName, caseResults] of byCase) {
+      console.log(`\n  ${caseName}`);
+
+      const maxLen = Math.max(...caseResults.map((r) => r.modelId.length));
+
+      for (const r of caseResults) {
+        totalPassed += r.passed;
+        totalRuns += r.runs;
+
+        const pct = (r.passRate * 100).toFixed(0);
+        const avgMs = r.avgDurationMs.toFixed(0);
+        const status =
+          r.passRate === 1
+            ? '\x1b[32m\u2713\x1b[0m'
+            : r.passRate >= 0.8
+              ? '\x1b[33m~\x1b[0m'
+              : '\x1b[31m\u2717\x1b[0m';
+
+        console.log(
+          `    ${status} ${r.modelId.padEnd(maxLen)}  ${r.passed}/${r.runs} passed (${pct}%)  avg ${avgMs}ms`
+        );
+
+        if (r.failures.length > 0) {
+          for (const f of r.failures) {
+            console.log(`      \x1b[2m\u2514 ${f.error} (${f.count}x)\x1b[0m`);
+          }
+        }
+      }
+    }
+
+    const totalPct = totalRuns > 0 ? ((totalPassed / totalRuns) * 100).toFixed(0) : 0;
+    const modelCount = new Set(this.results.map((r) => r.modelId)).size;
+    console.log(
+      `\n  Summary: ${totalPassed}/${totalRuns} passed (${totalPct}%) across ${modelCount} model(s)`
+    );
+    console.log('='.repeat(60) + '\n');
+  }
+}