sunpeak 0.19.2 → 0.19.10

package/template/tests/evals/.env.example

@@ -0,0 +1,5 @@
+# Copy this file to .env and fill in your API keys.
+# .env is gitignored — never commit API keys.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GOOGLE_GENERATIVE_AI_API_KEY=...

package/template/tests/evals/albums.eval.ts

@@ -0,0 +1,28 @@
+import { expect } from 'vitest';
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks for photo albums',
+      prompt: 'Show me my photo albums',
+      expect: { tool: 'show-albums' },
+    },
+    {
+      name: 'asks for food photos',
+      prompt: 'Show me photos from my Austin pizza tour',
+      expect: {
+        tool: 'show-albums',
+        args: { search: expect.stringMatching(/pizza|austin/i) },
+      },
+    },
+    {
+      name: 'asks for a specific category',
+      prompt: 'Show me my travel photos',
+      expect: {
+        tool: 'show-albums',
+        args: { category: expect.stringMatching(/travel/i) },
+      },
+    },
+  ],
+});

package/template/tests/evals/carousel.eval.ts

@@ -0,0 +1,26 @@
+import { expect } from 'vitest';
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks for popular places',
+      prompt: 'Show me popular places to visit in Tokyo',
+      expect: {
+        tool: 'show-carousel',
+        args: { city: expect.stringMatching(/tokyo/i) },
+      },
+    },
+    {
+      name: 'asks for restaurants',
+      prompt: 'What are some good restaurants in Paris?',
+      expect: {
+        tool: 'show-carousel',
+        args: {
+          city: expect.stringMatching(/paris/i),
+          categories: expect.arrayContaining([expect.stringMatching(/restaurant|food|dining/i)]),
+        },
+      },
+    },
+  ],
+});

package/template/tests/evals/eval.config.ts

@@ -0,0 +1,26 @@
+import { defineEvalConfig } from 'sunpeak/eval';
+
+// API keys are loaded automatically from tests/evals/.env (gitignored).
+// See .env.example for the format.
+
+export default defineEvalConfig({
+  // Server is auto-detected for sunpeak projects.
+  // For non-sunpeak projects, uncomment:
+  // server: 'http://localhost:8000/mcp',
+
+  models: [
+    // Uncomment models and install their provider packages:
+    // 'gpt-4o',                      // OPENAI_API_KEY
+    // 'gpt-4o-mini',                 // OPENAI_API_KEY
+    // 'o4-mini',                     // OPENAI_API_KEY
+    // 'claude-sonnet-4-20250514',    // ANTHROPIC_API_KEY
+    // 'gemini-2.0-flash',            // GOOGLE_GENERATIVE_AI_API_KEY
+  ],
+
+  defaults: {
+    runs: 10, // Number of times to run each case per model
+    maxSteps: 1, // Max tool call steps per run
+    temperature: 0, // 0 for most deterministic results
+    timeout: 30_000, // Timeout per run in ms
+  },
+});

package/template/tests/evals/map.eval.ts

@@ -0,0 +1,23 @@
+import { expect } from 'vitest';
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks for a map',
+      prompt: 'Show me a map of coffee shops near downtown Austin',
+      expect: {
+        tool: 'show-map',
+        args: { query: expect.stringMatching(/coffee/i) },
+      },
+    },
+    {
+      name: 'asks for nearby places',
+      prompt: 'Find me some parks nearby',
+      expect: {
+        tool: 'show-map',
+        args: { query: expect.stringMatching(/park/i) },
+      },
+    },
+  ],
+});

package/template/tests/evals/review.eval.ts

@@ -0,0 +1,48 @@
+import { expect } from 'vitest';
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks to review a code diff',
+      prompt: 'Review my code changes to the auth module',
+      expect: { tool: 'review-diff' },
+    },
+    {
+      name: 'asks to draft a social post',
+      prompt: 'Write a launch announcement for X and LinkedIn',
+      expect: {
+        tool: 'review-post',
+        args: { platforms: expect.arrayContaining([expect.stringMatching(/x|twitter/i)]) },
+      },
+    },
+    {
+      name: 'asks to review a purchase',
+      prompt: 'Review my order for the Pro plan upgrade',
+      expect: { tool: 'review-purchase' },
+    },
+
+    // --- Other assertion patterns ---
+    //
+    // Multi-step (ordered tool call sequence):
+    //   {
+    //     name: 'multi-step flow',
+    //     prompt: 'Draft a post and then review it',
+    //     maxSteps: 3,
+    //     expect: [
+    //       { tool: 'review-post' },
+    //       { tool: 'publish-post' },
+    //     ],
+    //   },
+    //
+    // Custom assertion (full access to result):
+    //   {
+    //     name: 'custom check',
+    //     prompt: 'Show me my recent reviews',
+    //     assert: (result) => {
+    //       expect(result.toolCalls).toHaveLength(1);
+    //       expect(result.toolCalls[0].name).toBe('review-diff');
+    //     },
+    //   },
+  ],
+});