sunpeak 0.19.4 → 0.19.12

package/template/tests/evals/_env.example

@@ -0,0 +1,5 @@
+# Copy this file to .env and fill in your API keys.
+# .env is gitignored — never commit API keys.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GOOGLE_GENERATIVE_AI_API_KEY=...

package/template/tests/evals/albums.eval.ts

@@ -0,0 +1,31 @@
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks for photo albums',
+      prompt: 'Show me my photo albums',
+      expect: { tool: 'show-albums' },
+    },
+    {
+      name: 'asks for food photos',
+      prompt: 'Show me photos from my Austin pizza tour',
+      expect: { tool: 'show-albums' },
+    },
+    {
+      name: 'asks for a specific category',
+      prompt: 'Show me my travel photos',
+      expect: { tool: 'show-albums' },
+    },
+    // To also check argument extraction, add args expectations
+    // (import { expect } from 'vitest' to use matchers):
+    // {
+    //   name: 'passes search term',
+    //   prompt: 'Show me photos from my Austin pizza tour',
+    //   expect: {
+    //     tool: 'show-albums',
+    //     args: { search: expect.stringMatching(/pizza|austin/i) },
+    //   },
+    // },
+  ],
+});

package/template/tests/evals/carousel.eval.ts

@@ -0,0 +1,16 @@
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks for popular places',
+      prompt: 'Show me popular places to visit in Tokyo',
+      expect: { tool: 'show-carousel' },
+    },
+    {
+      name: 'asks for places to eat',
+      prompt: 'Where should I eat in Paris?',
+      expect: { tool: 'show-carousel' },
+    },
+  ],
+});

package/template/tests/evals/eval.config.ts

@@ -0,0 +1,26 @@
+import { defineEvalConfig } from 'sunpeak/eval';
+
+// API keys are loaded automatically from tests/evals/.env (gitignored).
+// See .env.example for the format.
+
+export default defineEvalConfig({
+  // Server is auto-detected for sunpeak projects.
+  // For non-sunpeak projects, uncomment:
+  // server: 'http://localhost:8000/mcp',
+
+  models: [
+    // Uncomment models and install their provider packages:
+    // 'gpt-4o',                   // OPENAI_API_KEY
+    // 'gpt-4o-mini',              // OPENAI_API_KEY
+    // 'o4-mini',                  // OPENAI_API_KEY
+    // 'claude-sonnet-4-20250514', // ANTHROPIC_API_KEY
+    // 'gemini-2.0-flash',         // GOOGLE_GENERATIVE_AI_API_KEY
+  ],
+
+  defaults: {
+    runs: 5, // Number of times to run each case per model
+    maxSteps: 1, // Max tool call steps per run
+    temperature: 0, // 0 for most deterministic results
+    timeout: 30_000, // Timeout per run in ms
+  },
+});

package/template/tests/evals/map.eval.ts

@@ -0,0 +1,16 @@
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks for a map',
+      prompt: 'Show me a map of coffee shops near downtown Austin',
+      expect: { tool: 'show-map' },
+    },
+    {
+      name: 'asks for nearby places',
+      prompt: 'Find me some parks nearby',
+      expect: { tool: 'show-map' },
+    },
+  ],
+});

package/template/tests/evals/review.eval.ts

@@ -0,0 +1,53 @@
+import { defineEval } from 'sunpeak/eval';
+
+export default defineEval({
+  cases: [
+    {
+      name: 'asks to review a code diff',
+      prompt: 'Review my code changes to the auth module',
+      expect: { tool: 'review-diff' },
+    },
+    {
+      name: 'asks to draft a social post',
+      prompt: 'Write a launch announcement for X and LinkedIn',
+      expect: { tool: 'review-post' },
+    },
+    {
+      name: 'asks to review a purchase',
+      prompt: 'Review my order for the Pro plan upgrade',
+      expect: { tool: 'review-purchase' },
+    },
+
+    // To also check argument extraction, add args expectations
+    // (import { expect } from 'vitest' to use matchers):
+    // {
+    //   name: 'passes platforms',
+    //   prompt: 'Write a launch announcement for X and LinkedIn',
+    //   expect: {
+    //     tool: 'review-post',
+    //     args: { platforms: expect.arrayContaining([expect.stringMatching(/x|twitter/i)]) },
+    //   },
+    // },
+    //
+    // Multi-step (ordered tool call sequence):
+    // {
+    //   name: 'multi-step flow',
+    //   prompt: 'Draft a post and then review it',
+    //   maxSteps: 3,
+    //   expect: [
+    //     { tool: 'review-post' },
+    //     { tool: 'publish-post' },
+    //   ],
+    // },
+    //
+    // Custom assertion (full access to result):
+    // {
+    //   name: 'custom check',
+    //   prompt: 'Show me my recent reviews',
+    //   assert: (result) => {
+    //     expect(result.toolCalls).toHaveLength(1);
+    //     expect(result.toolCalls[0].name).toBe('review-diff');
+    //   },
+    // },
+  ],
+});