windmill-components 1.655.2 → 1.665.1

package/package/components/copilot/chat/__tests__/app/appChat.eval.test.js

@@ -6,50 +6,66 @@ import { loadAppFixtureForEval } from './appFixtureLoader';
 import { dirname, join } from 'path';
 // @ts-ignore - Node.js url
 import { fileURLToPath } from 'url';
-// Get API key from environment - tests will be skipped if not set
+// Get API keys from environment - tests will be skipped if none are set
 // @ts-ignore
-const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
-// Skip all tests if no API key is provided
-const describeWithApiKey = OPENROUTER_API_KEY ? describe : describe.skip;
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+// @ts-ignore
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
+const hasAnyKey = OPENAI_API_KEY || ANTHROPIC_API_KEY;
+const describeWithApiKey = hasAnyKey ? describe : describe.skip;
 // Get __dirname equivalent for ES modules
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const MODELS = ['google/gemini-2.5-flash', 'anthropic/claude-haiku-4.5', 'openai/gpt-4o'];
+const MODEL_VARIANTS = [
+    ...(OPENAI_API_KEY
+        ? [{ model: 'gpt-4o', provider: 'openai', apiKey: OPENAI_API_KEY }]
+        : []),
+    ...(ANTHROPIC_API_KEY
+        ? [
+            {
+                model: 'claude-haiku-4-5-20241022',
+                provider: 'anthropic',
+                apiKey: ANTHROPIC_API_KEY
+            }
+        ]
+        : [])
+];
 const VARIANTS = [
-    ...MODELS.map((model) => ({
+    ...MODEL_VARIANTS.map((mv) => ({
         ...BASELINE_VARIANT,
-        model,
-        name: `baseline-${model.replace('/', '-')}`
+        model: mv.model,
+        name: `baseline-${mv.provider}-${mv.model}`,
+        _provider: mv.provider,
+        _apiKey: mv.apiKey
     })),
-    ...MODELS.map((model) => ({
+    ...MODEL_VARIANTS.map((mv) => ({
         ...STREAMLINED_VARIANT,
-        model,
-        name: `streamlined-${model.replace('/', '-')}`
+        model: mv.model,
+        name: `streamlined-${mv.provider}-${mv.model}`,
+        _provider: mv.provider,
+        _apiKey: mv.apiKey
     }))
 ];
 describeWithApiKey('App Chat LLM Evaluation', () => {
     const TEST_TIMEOUT = 120_000;
-    if (!OPENROUTER_API_KEY) {
-        console.warn('OPENROUTER_API_KEY is not set, skipping tests');
+    if (!hasAnyKey) {
+        console.warn('No API keys set (OPENAI_API_KEY or ANTHROPIC_API_KEY), skipping tests');
     }
     it('test1: creates a simple counter app', async () => {
         const USER_PROMPT = `Create a counter app with increment/decrement buttons`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY);
-        // Write results to files
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, undefined, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
         expect(true).toBe(true);
     }, TEST_TIMEOUT);
     it('test2: modifies existing counter app to add reset button', async () => {
-        // Load initial app from fixture folder
         const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'test1_counter_app'));
         const USER_PROMPT = `Add a reset button that sets the counter back to 0`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialFrontend,
             initialBackend
-        });
-        // Write results to files
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
@@ -59,10 +75,10 @@ describeWithApiKey('App Chat LLM Evaluation', () => {
     it('test3: shopping cart - add quantity selector', async () => {
         const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'shopping_cart'));
         const USER_PROMPT = `Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialFrontend,
             initialBackend
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
@@ -71,10 +87,10 @@ describeWithApiKey('App Chat LLM Evaluation', () => {
     it('test4: shopping cart - add discount code', async () => {
         const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'shopping_cart'));
         const USER_PROMPT = `Add a discount code input field in the cart. When the code "SAVE10" is entered, apply a 10% discount to the total`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialFrontend,
             initialBackend
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
@@ -84,10 +100,10 @@ describeWithApiKey('App Chat LLM Evaluation', () => {
     it('test5: file manager - add search bar', async () => {
         const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
         const USER_PROMPT = `Add a search bar in the toolbar that filters files and folders by name as the user types`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialFrontend,
             initialBackend
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
@@ -96,10 +112,10 @@ describeWithApiKey('App Chat LLM Evaluation', () => {
     it('test6: file manager - show file details', async () => {
         const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
         const USER_PROMPT = `Show file size (formatted as KB/MB) and modified date in the file list for each item`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialFrontend,
             initialBackend
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
@@ -108,10 +124,10 @@ describeWithApiKey('App Chat LLM Evaluation', () => {
     it('test7: file manager - add select all checkbox', async () => {
         const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
         const USER_PROMPT = `Add a "Select All" checkbox in the file list header and individual checkboxes for each file. Add a "Delete Selected" button that appears when items are selected`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialFrontend,
             initialBackend
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
@@ -120,7 +136,7 @@ describeWithApiKey('App Chat LLM Evaluation', () => {
     // ==================== From-Scratch Creation Tests ====================
     it('test8: create quiz app from scratch', async () => {
         const USER_PROMPT = `Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct.`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY);
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, undefined, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);
@@ -128,7 +144,7 @@ describeWithApiKey('App Chat LLM Evaluation', () => {
     }, TEST_TIMEOUT);
     it('test9: create recipe book from scratch', async () => {
         const USER_PROMPT = `Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes.`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY);
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, undefined, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`App files: ${appPaths.join(', ')}`);

package/package/components/copilot/chat/__tests__/app/appEvalComparison.d.ts

@@ -16,11 +16,6 @@ export interface InitialApp {
 }
 /**
  * Evaluates how well a generated app fulfills the user's request, considering any initial app state.
- * This evaluator does not require an expected reference app - it evaluates based on the request alone.
- *
- * @param userPrompt The original user request
- * @param generatedApp The app generated by the AI
- * @param initialApp Optional initial app state (what the app looked like before AI changes)
- * @returns Evaluation result with score, statement, and missing requirements
+ * Uses Anthropic API directly.
  */
 export declare function evaluateAppGeneration(userPrompt: string, generatedApp: AppFiles, initialApp?: InitialApp): Promise<EvaluationResult>;

package/package/components/copilot/chat/__tests__/app/appEvalComparison.js

@@ -1,4 +1,4 @@
-import OpenAI from 'openai';
+import Anthropic from '@anthropic-ai/sdk';
 import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared';
 /**
  * System prompt for evaluating app generation without a reference expected app.
@@ -51,17 +51,20 @@ const APP_GENERATION_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for
 ${BASE_EVALUATOR_RESPONSE_FORMAT}`;
 /**
  * Evaluates how well a generated app fulfills the user's request, considering any initial app state.
- * This evaluator does not require an expected reference app - it evaluates based on the request alone.
- *
- * @param userPrompt The original user request
- * @param generatedApp The app generated by the AI
- * @param initialApp Optional initial app state (what the app looked like before AI changes)
- * @returns Evaluation result with score, statement, and missing requirements
+ * Uses Anthropic API directly.
  */
 export async function evaluateAppGeneration(userPrompt, generatedApp, initialApp) {
     // @ts-ignore
-    const apiKey = process.env.OPENROUTER_API_KEY;
-    const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey });
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey) {
+        return {
+            success: false,
+            resemblanceScore: 0,
+            statement: 'No API key available for evaluation',
+            error: 'ANTHROPIC_API_KEY not set'
+        };
+    }
+    const client = new Anthropic({ apiKey });
     let userMessage = `## User's Original Request
 ${userPrompt}
 
@@ -89,15 +92,17 @@ Please evaluate how well the generated app:
 1. Fulfills ALL requirements from the user's original request
 2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}`;
     try {
-        const response = await client.chat.completions.create({
-            model: 'anthropic/claude-sonnet-4.5',
+        const response = await client.messages.create({
+            model: 'claude-sonnet-4-5-20250514',
+            max_tokens: 2048,
+            system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT,
             messages: [
-                { role: 'system', content: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT },
                 { role: 'user', content: userMessage }
             ],
             temperature: 0
         });
-        const content = response.choices[0]?.message?.content;
+        const textBlock = response.content.find((block) => block.type === 'text');
+        const content = textBlock?.text;
         if (!content) {
             return {
                 success: false,

package/package/components/copilot/chat/__tests__/app/appEvalRunner.d.ts

@@ -1,5 +1,6 @@
 import type { AppFiles, BackendRunnable } from '../../app/core';
 import { type VariantConfig, type BaseEvalResult } from '../shared';
+import type { AIProvider } from '../../../../../gen/types.gen';
 export type { InitialApp } from './appEvalComparison';
 /**
  * App-specific evaluation result.
@@ -20,17 +21,25 @@ export interface AppEvalOptions {
     variant?: VariantConfig;
     /** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */
     evaluateWithLLM?: boolean;
+    /** AI provider (inferred from model name if omitted) */
+    provider?: AIProvider;
 }
 /**
- * Runs an app chat evaluation with real OpenAI API calls.
- * Executes tool calls using the actual app tools from core.ts or variant-configured tools.
+ * Runs an app chat evaluation using the shared chat loop (same code path as production).
  */
-export declare function runAppEval(userPrompt: string, openaiApiKey: string, options?: AppEvalOptions): Promise<AppEvalResult>;
+export declare function runAppEval(userPrompt: string, apiKey: string, options?: AppEvalOptions): Promise<AppEvalResult>;
+/**
+ * Per-variant provider override.
+ */
+export interface VariantProviderOverride {
+    provider: AIProvider;
+    apiKey: string;
+}
 /**
  * Runs the same prompt against multiple variants sequentially for comparison.
- * Returns results in the same order as the input variants.
+ * Accepts optional per-variant provider/apiKey overrides.
  */
-export declare function runVariantComparison(userPrompt: string, variants: VariantConfig[], openaiApiKey: string, baseOptions?: Omit<AppEvalOptions, 'variant'>): Promise<AppEvalResult[]>;
+export declare function runVariantComparison(userPrompt: string, variants: VariantConfig[], defaultApiKey: string, baseOptions?: Omit<AppEvalOptions, 'variant'>, providerOverrides?: VariantProviderOverride[]): Promise<AppEvalResult[]>;
 /**
  * Writes app comparison results to a folder-based structure.
  * Each variant gets its own folder with frontend/, backend/, and details.json.

package/package/components/copilot/chat/__tests__/app/appEvalRunner.js

@@ -11,15 +11,14 @@ const appDefaults = {
     tools: getAppTools()
 };
 /**
- * Runs an app chat evaluation with real OpenAI API calls.
- * Executes tool calls using the actual app tools from core.ts or variant-configured tools.
+ * Runs an app chat evaluation using the shared chat loop (same code path as production).
  */
-export async function runAppEval(userPrompt, openaiApiKey, options) {
+export async function runAppEval(userPrompt, apiKey, options) {
     const { helpers, getFiles } = createAppEvalHelpers(options?.initialFrontend ?? {}, options?.initialBackend ?? {});
     // Resolve variant configuration
     const variantName = options?.variant?.name ?? 'baseline';
     const systemMessage = resolveSystemPrompt(options?.variant, appDefaults, options?.customSystemPrompt);
-    const { toolDefs, tools } = resolveTools(options?.variant, appDefaults);
+    const { tools } = resolveTools(options?.variant, appDefaults);
     const model = resolveModel(options?.variant, options?.model);
     // Build user message
     const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext());
@@ -28,15 +27,15 @@ export async function runAppEval(userPrompt, openaiApiKey, options) {
         userPrompt,
         systemMessage,
         userMessage,
-        toolDefs,
         tools,
         helpers,
-        apiKey: openaiApiKey,
+        apiKey,
         getOutput: getFiles,
         options: {
             maxIterations: options?.maxIterations,
             model,
-            workspace: 'test-workspace'
+            workspace: 'test-workspace',
+            provider: options?.provider
         }
     });
     // Run LLM evaluation unless explicitly disabled
@@ -60,13 +59,15 @@ export async function runAppEval(userPrompt, openaiApiKey, options) {
 }
 /**
  * Runs the same prompt against multiple variants sequentially for comparison.
- * Returns results in the same order as the input variants.
+ * Accepts optional per-variant provider/apiKey overrides.
  */
-export async function runVariantComparison(userPrompt, variants, openaiApiKey, baseOptions) {
-    const results = await Promise.all(variants.map(async (variant) => {
-        return await runAppEval(userPrompt, openaiApiKey, {
+export async function runVariantComparison(userPrompt, variants, defaultApiKey, baseOptions, providerOverrides) {
+    const results = await Promise.all(variants.map(async (variant, i) => {
+        const override = providerOverrides?.[i];
+        return await runAppEval(userPrompt, override?.apiKey ?? defaultApiKey, {
             ...baseOptions,
-            variant
+            variant,
+            provider: override?.provider ?? baseOptions?.provider
         });
     }));
     return results;

package/package/components/copilot/chat/__tests__/flow/flowChat.eval.test.js

@@ -21,30 +21,47 @@ import initialTest5 from './initial/test5_initial.json';
 import initialTest6 from './initial/test6_initial.json';
 // @ts-ignore - JSON import
 import initialTest7 from './initial/test7_initial.json';
-// Get API key from environment - tests will be skipped if not set
+// Get API keys from environment - tests will be skipped if none are set
 // @ts-ignore
-// const OPENAI_API_KEY = process.env.OPENAI_API_KEY
-const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
-// Skip all tests if no API key is provided
-// const describeWithApiKey = OPENAI_API_KEY ? describe : describe.skip
-const describeWithApiKey = OPENROUTER_API_KEY ? describe : describe.skip;
-const MODELS = ['google/gemini-2.5-flash', 'anthropic/claude-haiku-4.5', 'openai/gpt-4o'];
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+// @ts-ignore
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
+const hasAnyKey = OPENAI_API_KEY || ANTHROPIC_API_KEY;
+const describeWithApiKey = hasAnyKey ? describe : describe.skip;
+const MODEL_VARIANTS = [
+    ...(OPENAI_API_KEY
+        ? [{ model: 'gpt-4o', provider: 'openai', apiKey: OPENAI_API_KEY }]
+        : []),
+    ...(ANTHROPIC_API_KEY
+        ? [
+            {
+                model: 'claude-haiku-4-5-20241022',
+                provider: 'anthropic',
+                apiKey: ANTHROPIC_API_KEY
+            }
+        ]
+        : [])
+];
 const VARIANTS = [
-    ...MODELS.map((model) => ({
+    ...MODEL_VARIANTS.map((mv) => ({
         ...BASELINE_VARIANT,
-        model,
-        name: `baseline-${model.replace('/', '-')}`
+        model: mv.model,
+        name: `baseline-${mv.provider}-${mv.model}`,
+        _provider: mv.provider,
+        _apiKey: mv.apiKey
     })),
-    ...MODELS.map((model) => ({
+    ...MODEL_VARIANTS.map((mv) => ({
         ...MINIMAL_SINGLE_TOOL_VARIANT,
-        model,
-        name: `minimal-single-tool-${model.replace('/', '-')}`
+        model: mv.model,
+        name: `minimal-single-tool-${mv.provider}-${mv.model}`,
+        _provider: mv.provider,
+        _apiKey: mv.apiKey
     }))
 ];
 describeWithApiKey('Flow Chat LLM Evaluation', () => {
     const TEST_TIMEOUT = 120_000;
-    if (!OPENROUTER_API_KEY) {
-        console.warn('OPENROUTER_API_KEY is not set, skipping tests');
+    if (!hasAnyKey) {
+        console.warn('No API keys set (OPENAI_API_KEY or ANTHROPIC_API_KEY), skipping tests');
     }
     it('test1: user role-based actions with loop and branches', async () => {
         const USER_PROMPT = `
@@ -56,9 +73,9 @@ STEP 3: Loop on all users
 STEP 4: Do branches based on user's role, do different action based on that. Roles are admin, user, moderator
 STEP 5: Return action taken for each user
 `;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             expectedFlow: expectedTest1
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         // Write results to files
         const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
@@ -89,9 +106,9 @@ STEP 5: Branch based on inventory - if all items available, create shipment reco
 STEP 6: Send confirmation (mock email to customer_email)
 STEP 7: Return final order summary with status
 `;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             expectedFlow: expectedTest2
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`Flow files: ${flowPaths.join(', ')}`);
@@ -124,9 +141,9 @@ STEP 5: Branch based on quality score:
   - If score < 70: Store in quarantine and send alert
 STEP 6: Return processing report with statistics (total records, quality score, destination)
 `;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             expectedFlow: expectedTest3
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`Flow files: ${flowPaths.join(', ')}`);
@@ -159,9 +176,9 @@ STEP 3: Use an AI agent to handle the customer query. The agent should have acce
 STEP 4: Log the interaction to audit trail (customer_id, query, response summary)
 STEP 5: Return the agent's response and any actions taken
 `;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             expectedFlow: expectedTest4
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`Flow files: ${flowPaths.join(', ')}`);
@@ -190,11 +207,11 @@ Modify this existing flow to add error handling:
 - If validation passes, return the data for the next step
 - Update save_results to handle the validation result appropriately
 `;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialModules: initialTest5.value.modules,
             initialSchema: initialTest5.schema,
             expectedFlow: expectedTest5
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`Flow files: ${flowPaths.join(', ')}`);
@@ -222,11 +239,11 @@ Modify the order processing loop to handle different order types:
 - Move the original process_order step to the default branch for unknown order types
 - Each branch step should return the orderId, shipping cost, and shipping type
 `;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialModules: initialTest6.value.modules,
             initialSchema: initialTest6.schema,
             expectedFlow: expectedTest6
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`Flow files: ${flowPaths.join(', ')}`);
@@ -254,11 +271,11 @@ Refactor this flow for better performance by parallelizing the enrichment steps:
 - The combine_data step should check if any enrichment used a fallback value and set a hasFallbacks flag
 - Keep get_item as the first step and return_result as the last step unchanged
 `;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
             initialModules: initialTest7.value.modules,
             initialSchema: initialTest7.schema,
             expectedFlow: expectedTest7
-        });
+        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
         const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results);
         console.log(`\nResults written to: ${summaryPath}`);
         console.log(`Flow files: ${flowPaths.join(', ')}`);

package/package/components/copilot/chat/__tests__/flow/flowEvalComparison.js

@@ -40,13 +40,10 @@ ${BASE_EVALUATOR_RESPONSE_FORMAT}`;
  * Returns a resemblance score (0-100), a qualitative statement, and any missing requirements.
  */
 export async function evaluateFlowComparison(generatedFlow, expectedFlow, userPrompt) {
-    // @ts-ignore
-    const apiKey = process.env.OPENROUTER_API_KEY;
     return evaluateWithLLM({
         userPrompt,
         generatedOutput: generatedFlow,
         expectedOutput: expectedFlow,
-        evaluatorSystemPrompt: FLOW_EVALUATOR_SYSTEM_PROMPT,
-        apiKey
+        evaluatorSystemPrompt: FLOW_EVALUATOR_SYSTEM_PROMPT
     });
 }

package/package/components/copilot/chat/__tests__/flow/flowEvalRunner.d.ts

@@ -1,4 +1,5 @@
 import type { FlowModule } from '../../../../../gen';
+import type { AIProvider } from '../../../../../gen/types.gen';
 import type { ExtendedOpenFlow } from '../../../../flows/types';
 import { type ExpectedFlow } from './flowEvalComparison';
 import { type VariantConfig, type BaseEvalResult } from '../shared';
@@ -21,17 +22,25 @@ export interface FlowEvalOptions {
     maxIterations?: number;
     variant?: VariantConfig;
     expectedFlow?: ExpectedFlow;
+    /** AI provider (inferred from model name if omitted) */
+    provider?: AIProvider;
 }
 /**
- * Runs a flow chat evaluation with real OpenAI API calls.
- * Executes tool calls using the actual flowTools from core.ts or variant-configured tools.
+ * Runs a flow chat evaluation using the shared chat loop (same code path as production).
  */
-export declare function runFlowEval(userPrompt: string, openaiApiKey: string, options?: FlowEvalOptions): Promise<FlowEvalResult>;
+export declare function runFlowEval(userPrompt: string, apiKey: string, options?: FlowEvalOptions): Promise<FlowEvalResult>;
+/**
+ * Per-variant provider override.
+ */
+export interface VariantProviderOverride {
+    provider: AIProvider;
+    apiKey: string;
+}
 /**
  * Runs the same prompt against multiple variants sequentially for comparison.
- * Returns results in the same order as the input variants.
+ * Accepts optional per-variant provider/apiKey overrides.
  */
-export declare function runVariantComparison(userPrompt: string, variants: VariantConfig[], openaiApiKey: string, baseOptions?: Omit<FlowEvalOptions, 'variant'>): Promise<FlowEvalResult[]>;
+export declare function runVariantComparison(userPrompt: string, variants: VariantConfig[], defaultApiKey: string, baseOptions?: Omit<FlowEvalOptions, 'variant'>, providerOverrides?: VariantProviderOverride[]): Promise<FlowEvalResult[]>;
 /**
  * Writes flow comparison results to files.
  */

package/package/components/copilot/chat/__tests__/flow/flowEvalRunner.js

@@ -10,15 +10,14 @@ const flowDefaults = {
     tools: flowTools
 };
 /**
- * Runs a flow chat evaluation with real OpenAI API calls.
- * Executes tool calls using the actual flowTools from core.ts or variant-configured tools.
+ * Runs a flow chat evaluation using the shared chat loop (same code path as production).
  */
-export async function runFlowEval(userPrompt, openaiApiKey, options) {
+export async function runFlowEval(userPrompt, apiKey, options) {
     const { helpers, getFlow } = createFlowEvalHelpers(options?.initialModules ?? [], options?.initialSchema);
     // Resolve variant configuration
     const variantName = options?.variant?.name ?? 'baseline';
     const systemMessage = resolveSystemPrompt(options?.variant, flowDefaults, options?.customSystemPrompt);
-    const { toolDefs, tools } = resolveTools(options?.variant, flowDefaults);
+    const { tools } = resolveTools(options?.variant, flowDefaults);
     const model = resolveModel(options?.variant, options?.model);
     // Build user message
     const userMessage = prepareFlowUserMessage(userPrompt, helpers.getFlowAndSelectedId(), []);
@@ -27,15 +26,15 @@ export async function runFlowEval(userPrompt, openaiApiKey, options) {
         userPrompt,
         systemMessage,
         userMessage,
-        toolDefs,
         tools,
         helpers,
-        apiKey: openaiApiKey,
+        apiKey,
         getOutput: getFlow,
         options: {
             maxIterations: options?.maxIterations,
             model,
-            workspace: 'test-workspace'
+            workspace: 'test-workspace',
+            provider: options?.provider
         }
     });
     // Run evaluation if expected flow is provided
@@ -57,13 +56,15 @@ export async function runFlowEval(userPrompt, openaiApiKey, options) {
 }
 /**
  * Runs the same prompt against multiple variants sequentially for comparison.
- * Returns results in the same order as the input variants.
+ * Accepts optional per-variant provider/apiKey overrides.
  */
-export async function runVariantComparison(userPrompt, variants, openaiApiKey, baseOptions) {
-    const results = await Promise.all(variants.map(async (variant) => {
-        return await runFlowEval(userPrompt, openaiApiKey, {
+export async function runVariantComparison(userPrompt, variants, defaultApiKey, baseOptions, providerOverrides) {
+    const results = await Promise.all(variants.map(async (variant, i) => {
+        const override = providerOverrides?.[i];
+        return await runFlowEval(userPrompt, override?.apiKey ?? defaultApiKey, {
             ...baseOptions,
-            variant
+            variant,
+            provider: override?.provider ?? baseOptions?.provider
         });
     }));
     return results;

package/package/components/copilot/chat/__tests__/shared/baseEvalRunner.d.ts

@@ -1,5 +1,4 @@
 import type { ChatCompletionMessageParam, ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs';
-import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs';
 import type { TokenUsage, ToolCallDetail, EvalRunnerOptions } from './types';
 import type { Tool } from './baseVariants';
 /**
@@ -26,13 +25,13 @@ export interface RunEvalParams<THelpers, TOutput> {
     systemMessage: ChatCompletionSystemMessageParam;
     /** User message for the LLM */
     userMessage: ChatCompletionMessageParam;
-    /** Tool definitions for the LLM API */
-    toolDefs: ChatCompletionTool[];
+    /** Tool definitions for the LLM API (unused — derived from tools) */
+    toolDefs?: unknown;
     /** Full tool implementations for execution */
     tools: Tool<THelpers>[];
     /** Domain-specific helpers for tool execution */
     helpers: THelpers;
-    /** API key for OpenRouter */
+    /** API key for the provider */
     apiKey: string;
     /** Function to get the current output state */
     getOutput: () => TOutput;
@@ -40,9 +39,7 @@ export interface RunEvalParams<THelpers, TOutput> {
     options?: EvalRunnerOptions;
 }
 /**
- * Runs a generic evaluation with real LLM API calls.
- * Executes tool calls in a loop until the LLM stops calling tools.
- *
- * This is the core execution loop shared across all chat eval tests.
+ * Runs a generic evaluation using the shared chat loop (same code path as production).
+ * Uses streaming via real provider SDKs instead of OpenRouter non-streaming.
  */
 export declare function runEval<THelpers, TOutput>(params: RunEvalParams<THelpers, TOutput>): Promise<RawEvalResult<TOutput>>;