sunpeak 0.20.5 → 0.20.7

package/bin/commands/dev.mjs

@@ -149,11 +149,15 @@ export async function dev(projectRoot = process.cwd(), args = []) {
   const tailwindPlugin = await importFromProject(require, '@tailwindcss/vite');
   const tailwindcss = tailwindPlugin.default;
 
-  // Parse port from args or use default
-  let port = parseInt(process.env.PORT || '3000');
+  // Parse port from args or env. When neither is set, leave undefined so
+  // inspectServer auto-discovers a free port (and doesn't use strictPort,
+  // which would crash instead of falling back when port 3000 is busy).
+  let port = undefined;
   const portArgIndex = args.findIndex(arg => arg === '--port' || arg === '-p');
   if (portArgIndex !== -1 && args[portArgIndex + 1]) {
     port = parseInt(args[portArgIndex + 1]);
+  } else if (process.env.PORT) {
+    port = parseInt(process.env.PORT);
   }
 
   // Parse --no-begging flag
@@ -166,7 +170,7 @@ export async function dev(projectRoot = process.cwd(), args = []) {
   if (isProdTools) console.log('Prod Tools: MCP tool calls will use real handlers instead of simulation mocks');
   if (isProdResources) console.log('Prod Resources: resources will use production-built HTML from dist/');
 
-  console.log(`Starting dev server on port ${port}...`);
+  console.log(`Starting dev server${port ? ` on port ${port}` : ''}...`);
 
   // Check if we're in the sunpeak workspace (directory is named "template")
   const isTemplate = basename(projectRoot) === 'template';
@@ -261,6 +265,8 @@ export async function dev(projectRoot = process.cwd(), args = []) {
 
   // Build path map for prod-tools handler reloading (re-imports on each call for HMR).
   // Also do an initial load to validate handlers and populate toolHandlerMap for the MCP server.
+  // Extract the raw Zod shape (schema export) so the MCP server can register tools
+  // with their actual inputSchema instead of z.object({}).passthrough().
   const toolHandlerMap = new Map();
   for (const [toolName, { tool, path: toolPath }] of toolMap) {
     void tool; // Used for metadata; handler loaded unconditionally
@@ -268,7 +274,15 @@ export async function dev(projectRoot = process.cwd(), args = []) {
     try {
       const mod = await toolLoaderServer.ssrLoadModule(`./${relativePath}`);
       if (typeof mod.default === 'function') {
-        toolHandlerMap.set(toolName, { handler: mod.default, outputSchema: mod.outputSchema });
+        toolHandlerMap.set(toolName, {
+          handler: mod.default,
+          outputSchema: mod.outputSchema,
+          // The raw Zod shape from the tool file (e.g., { query: z.string(), limit: z.number() }).
+          // Passed to the MCP server so tools/list reports actual parameter schemas instead of
+          // empty objects. The MCP SDK duck-types Zod values (checks for parse/safeParse) so
+          // this works across module instances.
+          schema: mod.schema,
+        });
       }
     } catch (err) {
       console.warn(`Warning: Could not load handler for tool "${toolName}" (${relativePath}):\n  ${err.message}`);
@@ -327,6 +341,10 @@ export async function dev(projectRoot = process.cwd(), args = []) {
       ...(toolHandlerMap.has(toolName) ? {
         handler: toolHandlerMap.get(toolName).handler,
       } : {}),
+      // Attach the raw Zod shape so the MCP server registers tools with real schemas.
+      ...(toolHandlerMap.has(toolName) && toolHandlerMap.get(toolName).schema ? {
+        inputSchema: toolHandlerMap.get(toolName).schema,
+      } : {}),
     });
   }
 
@@ -346,6 +364,7 @@ export async function dev(projectRoot = process.cwd(), args = []) {
       tool: { name: toolName, ...tool },
       ...(handlerInfo?.outputSchema ? { outputSchema: handlerInfo.outputSchema } : {}),
       ...(handlerInfo ? { handler: handlerInfo.handler } : {}),
+      ...(handlerInfo?.schema ? { inputSchema: handlerInfo.schema } : {}),
     });
   }
 

package/bin/commands/inspect.mjs

@@ -1429,8 +1429,14 @@ export async function inspectServer(opts) {
     ownsSandbox = true;
   }
 
-  // Determine server port
-  const port = preferredPort || Number(process.env.PORT) || (await getPort(3000));
+  // Determine server port.
+  // Track whether the port was explicitly requested (via option or env var) vs
+  // auto-discovered. When explicit, use strictPort so Vite fails fast instead of
+  // silently picking another port — Playwright tests set baseURL from the same port
+  // and a silent fallback causes ERR_CONNECTION_REFUSED. When auto-discovered,
+  // the port is guaranteed free so strictPort is irrelevant.
+  const explicitPort = preferredPort || (process.env.PORT ? Number(process.env.PORT) : null);
+  const port = explicitPort || (await getPort(3000));
 
   // Import Vite
   const { createServer } = await import('vite');
@@ -1562,6 +1568,12 @@ export async function inspectServer(opts) {
     ],
     server: {
       port,
+      // When the port was explicitly requested (Playwright tests, --port flag, PORT env),
+      // fail fast if busy instead of silently picking another port. Playwright tests
+      // configure baseURL from the same port, so a silent fallback causes
+      // ERR_CONNECTION_REFUSED. When auto-discovered via getPort(), the port is
+      // already free so this doesn't apply.
+      ...(explicitPort ? { strictPort: true } : {}),
       // Listen on all interfaces so both 127.0.0.1 (used by Playwright tests)
       // and localhost (used by interactive browsing) connect successfully.
       // Without this, Vite defaults to localhost which may resolve to IPv6-only

package/bin/commands/test-init.mjs

@@ -61,12 +61,14 @@ export const defaultDeps = {
  * - JS/TS projects: root-level config + test files
  * - sunpeak projects: migrate to defineConfig()
  *
- * Scaffolds all 5 test types:
+ * Scaffolds 4 test types:
  * 1. E2E tests — Playwright-based inspector tests (mcp fixture)
  * 2. Visual regression — Screenshot comparison via result.screenshot()
  * 3. Live tests — Test against real ChatGPT/Claude hosts
  * 4. Evals — Multi-model tool calling reliability tests
- * 5. Unit tests — Direct tool handler tests (JS/TS projects only)
+ *
+ * Unit tests are not scaffolded here — they're part of the sunpeak app
+ * framework (`sunpeak new`) where tool handlers can be imported directly.
  */
 export async function testInit(args = [], deps = defaultDeps) {
   const d = { ...defaultDeps, ...deps };
@@ -532,56 +534,6 @@ export default defineLiveConfig({${serverOption}
   d.log.success(`Created ${liveDir}/ with live test config and example.`);
 }
 
-/**
- * Scaffold a unit test example for JS/TS projects.
- * @param {string} filePath - Full path to the unit test file
- * @param {object} d - Dependencies
- */
-function scaffoldUnitTest(filePath, d) {
-  if (d.existsSync(filePath)) {
-    d.log.info('Unit test already exists. Skipping.');
-    return;
-  }
-
-  d.mkdirSync(dirname(filePath), { recursive: true });
-
-  d.writeFileSync(
-    filePath,
-    `import { describe, it, expect } from 'vitest';
-
-/**
- * Unit tests for your MCP tool handlers.
- *
- * Import your tool handler directly and test its input/output
- * without starting the MCP server or inspector.
- *
- * Run with: npx sunpeak test --unit
- *
- * To set up vitest, add it to your devDependencies:
- *   npm install -D vitest
- *
- * Uncomment and customize the tests below for your tools.
- */
-
-// import handler, { tool, schema } from '../../src/tools/your-tool';
-// const extra = {} as Parameters<typeof handler>[1];
-
-// describe('your tool', () => {
-//   it('returns expected output', async () => {
-//     const result = await handler({ key: 'value' }, extra);
-//     expect(result.structuredContent).toBeDefined();
-//   });
-//
-//   it('exports correct tool config', () => {
-//     expect(tool.title).toBe('Your Tool');
-//     expect(tool.annotations?.readOnlyHint).toBe(true);
-//   });
-// });
-`
-  );
-  d.log.success(`Created ${filePath}`);
-}
-
 async function initExternalProject(cliServer, d) {
   d.log.info('Detected non-JS project. Creating self-contained test directory.');
 
@@ -715,6 +667,22 @@ async function initJsProject(cliServer, d) {
   const server = await getServerConfig(cliServer, d);
   const cwd = d.cwd();
 
+  // Ensure "type": "module" — sunpeak exports are ESM-only and Playwright's
+  // CJS resolver won't find them without it.
+  const pkgPath = join(cwd, 'package.json');
+  if (d.existsSync(pkgPath)) {
+    try {
+      const pkg = JSON.parse(d.readFileSync(pkgPath, 'utf-8'));
+      if (pkg.type !== 'module') {
+        pkg.type = 'module';
+        d.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n');
+        d.log.success('Set "type": "module" in package.json (required for sunpeak imports)');
+      }
+    } catch {
+      d.log.warn('Could not read package.json. Make sure "type": "module" is set.');
+    }
+  }
+
   // Create playwright.config.ts
   const configPath = join(cwd, 'playwright.config.ts');
   if (d.existsSync(configPath)) {
@@ -775,19 +743,15 @@ test('server exposes tools', async ({ mcp }) => {
   // 4. Eval boilerplate
   scaffoldEvals(join(cwd, 'tests', 'evals'), { server, d });
 
-  // 5. Unit test
-  scaffoldUnitTest(join(cwd, 'tests', 'unit', 'example.test.ts'), d);
-
   if (server.type === 'later') {
     d.log.warn('Server not configured. Edit playwright.config.ts before running tests.');
   }
   const pkgMgr = d.detectPackageManager();
   d.log.step('Next steps:');
-  d.log.message(`  ${pkgMgr} add -D sunpeak @playwright/test vitest`);
+  d.log.message(`  ${pkgMgr} add -D sunpeak @playwright/test`);
   d.log.message(`  ${pkgMgr} exec playwright install chromium`);
   d.log.message('');
   d.log.message('  npx sunpeak test              # E2E tests');
-  d.log.message('  npx sunpeak test --unit        # Unit tests (vitest)');
   d.log.message('  npx sunpeak test --visual      # Visual regression');
   d.log.message('  npx sunpeak test --live         # Live tests against real hosts');
   d.log.message('  npx sunpeak test --eval         # Multi-model evals');
@@ -833,14 +797,10 @@ export default defineConfig();
   // 3. Eval boilerplate
   scaffoldEvals(join(cwd, 'tests', 'evals'), { isSunpeak: true, d });
 
-  // 4. Unit test
-  scaffoldUnitTest(join(cwd, 'tests', 'unit', 'example.test.ts'), d);
-
   d.log.step('Scaffolded test types:');
   d.log.message('  tests/e2e/visual.test.ts    — Visual regression (npx sunpeak test --visual)');
   d.log.message('  tests/live/                 — Live host tests (npx sunpeak test --live)');
   d.log.message('  tests/evals/                — Multi-model evals (npx sunpeak test --eval)');
-  d.log.message('  tests/unit/example.test.ts  — Unit tests (npx sunpeak test --unit)');
   d.log.message('');
   d.log.message('  Migrate existing e2e tests:');
   d.log.message('  Replace: import { test, expect } from "@playwright/test"');

package/bin/commands/test.mjs

@@ -59,8 +59,17 @@ export async function runTest(args) {
   const results = [];
 
   if (runUnit) {
-    const code = await runChild('pnpm', ['exec', 'vitest', 'run', ...filteredArgs]);
-    results.push({ suite: 'unit', code });
+    // Only run unit tests if vitest is available (app framework projects have it,
+    // standalone testing framework projects don't).
+    const hasVitest = existsSync(join(process.cwd(), 'node_modules', '.bin', 'vitest'));
+    if (hasVitest) {
+      const code = await runChild('pnpm', ['exec', 'vitest', 'run', ...filteredArgs]);
+      results.push({ suite: 'unit', code });
+    } else if (isUnit) {
+      // Only warn if the user explicitly asked for --unit
+      console.error('vitest is not installed. Install it with: npm add -D vitest');
+      results.push({ suite: 'unit', code: 1 });
+    }
   }
 
   if (runE2e) {
@@ -418,11 +427,12 @@ async function runEvals(args) {
 
     const warnings = validateApiKeys(configModels);
     if (warnings.length > 0) {
-      console.log('');
+      console.error('');
       for (const w of warnings) {
-        console.warn(`⚠  ${w}`);
+        console.error(`✗  ${w}`);
       }
-      console.log('');
+      console.error('');
+      return 1;
     }
   }
 

package/bin/lib/eval/eval-runner.mjs

@@ -112,9 +112,35 @@ export async function discoverAndConvertTools(client) {
   const tools = {};
 
   for (const t of mcpTools) {
+    // Clean up the MCP inputSchema for AI provider compatibility.
+    // OpenAI rejects $schema, additionalProperties: {} (empty schema has no type),
+    // and other JSON Schema features that MCP servers may include.
+    const rawSchema = t.inputSchema || { type: 'object', properties: {} };
+    const cleanSchema = { ...rawSchema };
+    delete cleanSchema.$schema;
+    if (
+      cleanSchema.additionalProperties != null &&
+      typeof cleanSchema.additionalProperties === 'object' &&
+      Object.keys(cleanSchema.additionalProperties).length === 0
+    ) {
+      // Empty additionalProperties ({}) causes OpenAI to report type: "None".
+      // Remove it so the schema is a plain { type: "object", properties: {...} }.
+      delete cleanSchema.additionalProperties;
+    }
+    if (!cleanSchema.type) cleanSchema.type = 'object';
+    if (!cleanSchema.properties) cleanSchema.properties = {};
+    // Remove `required` so the model isn't forced to ask the user for every
+    // parameter before calling a tool. Eval prompts are intentionally vague
+    // ("show me photo albums") and the model should call the tool with
+    // reasonable defaults, not refuse because required fields are missing.
+    delete cleanSchema.required;
+
     tools[t.name] = aiTool({
       description: t.description || '',
-      parameters: jsonSchema(t.inputSchema || { type: 'object', properties: {} }),
+      // Set both so the tool works with ai v4/v5 (reads `parameters`)
+      // and ai v6 (reads `inputSchema`). tool() passes through both.
+      inputSchema: jsonSchema(cleanSchema),
+      parameters: jsonSchema(cleanSchema),
       execute: async (args) => {
         const result = await client.callTool({ name: t.name, arguments: args });
         // Return a simplified version for the model to consume

package/bin/lib/eval/model-registry.mjs

@@ -44,7 +44,12 @@ export async function resolveModel(modelId) {
   // that creates model instances: openai('gpt-4o'), anthropic('claude-...'), google('gemini-...')
   if (pkg === '@ai-sdk/openai') {
     const { openai } = provider;
-    return openai(modelId);
+    // @ai-sdk/openai v3 defaults to the Responses API, which requires strict
+    // JSON Schema (additionalProperties: false at every level, all properties
+    // required) — incompatible with arbitrary MCP server schemas. Use .chat()
+    // (Chat Completions API) when available. v1/v2 default to Chat Completions
+    // already and may not have .chat(), so fall back to the default.
+    return typeof openai.chat === 'function' ? openai.chat(modelId) : openai(modelId);
   }
   if (pkg === '@ai-sdk/anthropic') {
     const { anthropic } = provider;

package/bin/lib/test/base-config.mjs

@@ -5,8 +5,6 @@
  * Produces a config with per-host Playwright projects, sensible defaults for
  * MCP App testing, and a webServer entry to launch the inspector backend.
  */
-import { getPortSync } from '../get-port.mjs';
-
 /**
  * @param {Object} options
  * @param {string[]} options.hosts - Host shells to create projects for
@@ -63,10 +61,19 @@ export function createBaseConfig({ hosts, testDir, webServer, port, use, globalS
 /**
  * Resolve ports for the inspector and sandbox proxy.
  * Respects env vars for CI where validate.mjs assigns unique ports.
+ *
+ * Uses FIXED default ports (no dynamic probing) so all Playwright workers
+ * resolve the same baseURL. Dynamic port probing (getPortSync) caused flaky
+ * tests: the main process would pick port X, start the webServer on it, then
+ * worker processes re-evaluating the config would find X occupied and resolve
+ * to random ports Y/Z — causing ERR_CONNECTION_REFUSED.
+ *
+ * If the default port is busy, Playwright's reuseExistingServer (local) reuses
+ * it, or strictPort (CI) fails fast with a clear error.
  */
 export function resolvePorts() {
-  const port = parsePort(process.env.SUNPEAK_TEST_PORT) ?? getPortSync(6776);
-  const sandboxPort = parsePort(process.env.SUNPEAK_SANDBOX_PORT) ?? getPortSync(24680);
+  const port = parsePort(process.env.SUNPEAK_TEST_PORT) ?? 6776;
+  const sandboxPort = parsePort(process.env.SUNPEAK_SANDBOX_PORT) ?? 24680;
   return { port, sandboxPort };
 }
 

package/dist/mcp/index.cjs

@@ -9277,6 +9277,26 @@ function injectViteCSP(existingMeta) {
 	};
 }
 var startupTimestamp = Date.now().toString(36);
+/**
+* Make all properties in a Zod raw shape optional.
+*
+* Tool schemas from `src/tools/*.ts` have required fields by default (e.g.
+* `z.string()`). The dev server needs to accept partial args because:
+* - Mock mode returns fixture data regardless of args
+* - Models may not send every required field
+* - The inspector's "Re-run" button sends args from the last run
+*
+* Making fields optional preserves property types/descriptions in `tools/list`
+* (so models know what args to send) while letting the SDK accept any subset.
+*/
+function makeSchemaOptional(shape) {
+	const optional = {};
+	for (const [key, value] of Object.entries(shape)) {
+		const v = value;
+		optional[key] = typeof v.optional === "function" ? v.optional() : value;
+	}
+	return optional;
+}
 function createAppServer(config, simulations, viteMode) {
 	const { name = "sunpeak-app", version = "0.1.0", serverInfo } = config;
 	const mcpServer = new McpServer({
@@ -9350,6 +9370,7 @@ function createAppServer(config, simulations, viteMode) {
 				});
 				resourceHandles.set(resourceName, handle);
 			}
+			const toolInputSchema = simulation.inputSchema ? makeSchemaOptional(simulation.inputSchema) : zod.z.object({}).passthrough();
 			const fullToolMeta = {
 				...toolMeta,
 				ui: {
@@ -9359,7 +9380,7 @@ function createAppServer(config, simulations, viteMode) {
 			};
 			const toolHandle = hZ(mcpServer, tool.name, {
 				description: tool.description,
-				inputSchema: zod.z.object({}).passthrough(),
+				inputSchema: toolInputSchema,
 				...simulation.outputSchema ? { outputSchema: simulation.outputSchema } : {},
 				annotations: tool.annotations,
 				_meta: fullToolMeta
@@ -9424,7 +9445,7 @@ function createAppServer(config, simulations, viteMode) {
 			const realHandler = simulation.handler;
 			const plainToolConfig = {
 				description: tool.description,
-				inputSchema: zod.z.object({}).passthrough(),
+				inputSchema: simulation.inputSchema ? makeSchemaOptional(simulation.inputSchema) : zod.z.object({}).passthrough(),
 				...simulation.outputSchema ? { outputSchema: simulation.outputSchema } : {},
 				annotations: tool.annotations,
 				_meta: toolMeta