windmill-components 1.677.0 → 1.687.0

package/package/components/copilot/chat/__tests__/app/appChat.eval.test.js

@@ -1,153 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { runVariantComparison, writeAppComparisonResults } from './appEvalRunner';
-import { BASELINE_VARIANT, STREAMLINED_VARIANT } from './variants';
-import { loadAppFixtureForEval } from './appFixtureLoader';
-// @ts-ignore - Node.js path
-import { dirname, join } from 'path';
-// @ts-ignore - Node.js url
-import { fileURLToPath } from 'url';
-// Get API keys from environment - tests will be skipped if none are set
-// @ts-ignore
-const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
-// @ts-ignore
-const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
-const hasAnyKey = OPENAI_API_KEY || ANTHROPIC_API_KEY;
-const describeWithApiKey = hasAnyKey ? describe : describe.skip;
-// Get __dirname equivalent for ES modules
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-const MODEL_VARIANTS = [
-    ...(OPENAI_API_KEY
-        ? [{ model: 'gpt-4o', provider: 'openai', apiKey: OPENAI_API_KEY }]
-        : []),
-    ...(ANTHROPIC_API_KEY
-        ? [
-            {
-                model: 'claude-haiku-4-5-20241022',
-                provider: 'anthropic',
-                apiKey: ANTHROPIC_API_KEY
-            }
-        ]
-        : [])
-];
-const VARIANTS = [
-    ...MODEL_VARIANTS.map((mv) => ({
-        ...BASELINE_VARIANT,
-        model: mv.model,
-        name: `baseline-${mv.provider}-${mv.model}`,
-        _provider: mv.provider,
-        _apiKey: mv.apiKey
-    })),
-    ...MODEL_VARIANTS.map((mv) => ({
-        ...STREAMLINED_VARIANT,
-        model: mv.model,
-        name: `streamlined-${mv.provider}-${mv.model}`,
-        _provider: mv.provider,
-        _apiKey: mv.apiKey
-    }))
-];
-describeWithApiKey('App Chat LLM Evaluation', () => {
-    const TEST_TIMEOUT = 120_000;
-    if (!hasAnyKey) {
-        console.warn('No API keys set (OPENAI_API_KEY or ANTHROPIC_API_KEY), skipping tests');
-    }
-    it('test1: creates a simple counter app', async () => {
-        const USER_PROMPT = `Create a counter app with increment/decrement buttons`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, undefined, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    it('test2: modifies existing counter app to add reset button', async () => {
-        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'test1_counter_app'));
-        const USER_PROMPT = `Add a reset button that sets the counter back to 0`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
-            initialFrontend,
-            initialBackend
-        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    // ==================== Shopping Cart Tests ====================
-    it('test3: shopping cart - add quantity selector', async () => {
-        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'shopping_cart'));
-        const USER_PROMPT = `Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
-            initialFrontend,
-            initialBackend
-        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    it('test4: shopping cart - add discount code', async () => {
-        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'shopping_cart'));
-        const USER_PROMPT = `Add a discount code input field in the cart. When the code "SAVE10" is entered, apply a 10% discount to the total`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
-            initialFrontend,
-            initialBackend
-        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    // ==================== File Manager Tests ====================
-    it('test5: file manager - add search bar', async () => {
-        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
-        const USER_PROMPT = `Add a search bar in the toolbar that filters files and folders by name as the user types`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
-            initialFrontend,
-            initialBackend
-        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    it('test6: file manager - show file details', async () => {
-        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
-        const USER_PROMPT = `Show file size (formatted as KB/MB) and modified date in the file list for each item`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
-            initialFrontend,
-            initialBackend
-        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    it('test7: file manager - add select all checkbox', async () => {
-        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
-        const USER_PROMPT = `Add a "Select All" checkbox in the file list header and individual checkboxes for each file. Add a "Delete Selected" button that appears when items are selected`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, {
-            initialFrontend,
-            initialBackend
-        }, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    // ==================== From-Scratch Creation Tests ====================
-    it('test8: create quiz app from scratch', async () => {
-        const USER_PROMPT = `Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct.`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, undefined, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-    it('test9: create recipe book from scratch', async () => {
-        const USER_PROMPT = `Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes.`;
-        const results = await runVariantComparison(USER_PROMPT, VARIANTS, VARIANTS[0]._apiKey, undefined, VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })));
-        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
-        console.log(`\nResults written to: ${summaryPath}`);
-        console.log(`App files: ${appPaths.join(', ')}`);
-        expect(true).toBe(true);
-    }, TEST_TIMEOUT);
-});

package/package/components/copilot/chat/__tests__/app/appEvalComparison.d.ts

@@ -1,21 +0,0 @@
-import type { AppFiles, BackendRunnable } from '../../app/core';
-import type { EvaluationResult } from '../shared';
-/**
- * Expected app structure for evaluation.
- */
-export interface ExpectedApp {
-    frontend: Record<string, string>;
-    backend: Record<string, BackendRunnable>;
-}
-/**
- * Initial app state for evaluation context.
- */
-export interface InitialApp {
-    frontend: Record<string, string>;
-    backend: Record<string, BackendRunnable>;
-}
-/**
- * Evaluates how well a generated app fulfills the user's request, considering any initial app state.
- * Uses Anthropic API directly.
- */
-export declare function evaluateAppGeneration(userPrompt: string, generatedApp: AppFiles, initialApp?: InitialApp): Promise<EvaluationResult>;

package/package/components/copilot/chat/__tests__/app/appEvalComparison.js

@@ -1,136 +0,0 @@
-import Anthropic from '@anthropic-ai/sdk';
-import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared';
-/**
- * System prompt for evaluating app generation without a reference expected app.
- * Evaluates based on user request fulfillment and appropriate modifications to initial state.
- */
-const APP_GENERATION_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill Raw App definitions. Your task is to evaluate a generated app based on:
-1. The original user request/prompt
-2. The initial app state (if any) - this is what the app looked like before the AI made changes
-
-## Windmill Raw App Context
-- Raw Apps consist of frontend files and backend runnables
-- Frontend files are TypeScript/JavaScript files bundled with esbuild (entrypoint: index.tsx)
-- Backend runnables can be: inline scripts (TypeScript/Python), workspace scripts, workspace flows, or hub scripts
-- Frontend calls backend using \`await backend.<runnable_key>(args...)\`
-- Each backend runnable has a key (identifier), name (description), type, and configuration
-
-## Backend Runnable Types
-- **inline**: Custom code with \`inlineScript.language\` and \`inlineScript.content\`
-- **script**: Workspace script reference with \`path\`
-- **flow**: Workspace flow reference with \`path\`
-- **hubscript**: Hub script reference with \`path\`
-
-## Evaluation Criteria
-1. **User Request Fulfillment**: Does the generated app address ALL requirements from the user's original prompt?
-   - Are all requested features implemented?
-   - Does the frontend UI match the requirements?
-   - Are the correct backend runnables created?
-2. **Appropriate Modifications** (if initial app was provided):
-   - Were the changes made relevant to the user's request?
-   - Was existing functionality preserved where appropriate?
-   - Were only necessary changes made (no unnecessary removals or additions)?
-3. **Frontend Structure**: Are the frontend files correctly organized and implemented?
-   - Is the code valid TypeScript/JavaScript?
-   - Are components properly structured?
-   - Are backend calls correctly made?
-4. **Backend Structure**: Are the backend runnables correctly configured?
-   - Do inline scripts have proper main functions?
-   - Are types and paths correct for non-inline runnables?
-5. **Integration**: Does the frontend correctly call the backend?
-   - Are the runnable keys correctly referenced?
-   - Are arguments passed correctly?
-6. **Code Quality**: Is the code functionally correct and well-structured?
-
-## Important Notes
-- Focus on whether the user's request was fulfilled, not on stylistic preferences
-- If an initial app was provided, evaluate the appropriateness of the changes made
-- For new apps (no initial state), evaluate completeness and correctness
-- Extra helper functions or slightly different approaches can still score high if they accomplish the goal
-
-${BASE_EVALUATOR_RESPONSE_FORMAT}`;
-/**
- * Evaluates how well a generated app fulfills the user's request, considering any initial app state.
- * Uses Anthropic API directly.
- */
-export async function evaluateAppGeneration(userPrompt, generatedApp, initialApp) {
-    // @ts-ignore
-    const apiKey = process.env.ANTHROPIC_API_KEY;
-    if (!apiKey) {
-        return {
-            success: false,
-            resemblanceScore: 0,
-            statement: 'No API key available for evaluation',
-            error: 'ANTHROPIC_API_KEY not set'
-        };
-    }
-    const client = new Anthropic({ apiKey });
-    let userMessage = `## User's Original Request
-${userPrompt}
-
-`;
-    if (initialApp) {
-        userMessage += `## Initial App State (before AI modifications)
-\`\`\`json
-${JSON.stringify(initialApp, null, 2)}
-\`\`\`
-
-`;
-    }
-    else {
-        userMessage += `## Initial App State
-No initial app was provided - this is a new app created from scratch.
-
-`;
-    }
-    userMessage += `## Generated App
-\`\`\`json
-${JSON.stringify(generatedApp, null, 2)}
-\`\`\`
-
-Please evaluate how well the generated app:
-1. Fulfills ALL requirements from the user's original request
-2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}`;
-    try {
-        const response = await client.messages.create({
-            model: 'claude-sonnet-4-5-20250514',
-            max_tokens: 2048,
-            system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT,
-            messages: [
-                { role: 'user', content: userMessage }
-            ],
-            temperature: 0
-        });
-        const textBlock = response.content.find((block) => block.type === 'text');
-        const content = textBlock?.text;
-        if (!content) {
-            return {
-                success: false,
-                resemblanceScore: 0,
-                statement: 'No response from evaluator',
-                error: 'Empty response from LLM'
-            };
-        }
-        // Parse JSON response - handle potential markdown code blocks
-        let jsonContent = content.trim();
-        if (jsonContent.startsWith('```')) {
-            jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
-        }
-        const parsed = JSON.parse(jsonContent);
-        return {
-            success: true,
-            resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
-            statement: parsed.statement,
-            missingRequirements: parsed.missingRequirements ?? []
-        };
-    }
-    catch (err) {
-        const errorMessage = err instanceof Error ? err.message : String(err);
-        return {
-            success: false,
-            resemblanceScore: 0,
-            statement: 'Evaluation failed',
-            error: errorMessage
-        };
-    }
-}

package/package/components/copilot/chat/__tests__/app/appEvalHelpers.d.ts

@@ -1,15 +0,0 @@
-import type { AppAIChatHelpers, AppFiles, BackendRunnable } from '../../app/core';
-/**
- * Creates mock AppAIChatHelpers for eval testing.
- * Tracks app files state in memory and allows tool functions to modify it.
- */
-export declare function createAppEvalHelpers(initialFrontend?: Record<string, string>, initialBackend?: Record<string, BackendRunnable>): {
-    helpers: AppAIChatHelpers;
-    getFiles: () => AppFiles;
-    getFrontend: () => {
-        [x: string]: string;
-    };
-    getBackend: () => {
-        [x: string]: BackendRunnable;
-    };
-};

package/package/components/copilot/chat/__tests__/app/appEvalHelpers.js

@@ -1,107 +0,0 @@
-/**
- * Creates an empty lint result (no errors or warnings).
- */
-function createEmptyLintResult() {
-    return {
-        errorCount: 0,
-        warningCount: 0,
-        errors: { frontend: {}, backend: {} },
-        warnings: { frontend: {}, backend: {} }
-    };
-}
-/**
- * Creates mock AppAIChatHelpers for eval testing.
- * Tracks app files state in memory and allows tool functions to modify it.
- */
-export function createAppEvalHelpers(initialFrontend = {}, initialBackend = {}) {
-    // In-memory state
-    let frontend = { ...initialFrontend };
-    let backend = { ...initialBackend };
-    let snapshotId = 0;
-    const snapshots = new Map();
-    const helpers = {
-        // Frontend file operations
-        listFrontendFiles: () => Object.keys(frontend),
-        getFrontendFile: (path) => frontend[path],
-        getFrontendFiles: () => ({ ...frontend }),
-        setFrontendFile: (path, content) => {
-            frontend[path] = content;
-            // Return mock lint result - in real usage this would validate the file
-            return createEmptyLintResult();
-        },
-        deleteFrontendFile: (path) => {
-            delete frontend[path];
-        },
-        // Backend runnable operations
-        listBackendRunnables: () => {
-            return Object.entries(backend).map(([key, runnable]) => ({
-                key,
-                name: runnable.name
-            }));
-        },
-        getBackendRunnable: (key) => backend[key],
-        getBackendRunnables: () => ({ ...backend }),
-        setBackendRunnable: async (key, runnable) => {
-            backend[key] = runnable;
-            // Return mock lint result - in real usage this would validate the runnable
-            return createEmptyLintResult();
-        },
-        deleteBackendRunnable: (key) => {
-            delete backend[key];
-        },
-        // Combined view
-        getFiles: () => ({
-            frontend: { ...frontend },
-            backend: { ...backend }
-        }),
-        getSelectedContext: () => ({
-            type: 'none'
-        }),
-        // Snapshot management
-        snapshot: () => {
-            const id = ++snapshotId;
-            snapshots.set(id, {
-                frontend: { ...frontend },
-                backend: { ...backend }
-            });
-            return id;
-        },
-        revertToSnapshot: (id) => {
-            const snap = snapshots.get(id);
-            if (snap) {
-                frontend = { ...snap.frontend };
-                backend = { ...snap.backend };
-            }
-        },
-        // Linting
-        lint: () => {
-            // Return mock lint result - no actual linting in eval
-            return createEmptyLintResult();
-        },
-        // Data table operations (mock implementation for testing)
-        getDatatables: async () => {
-            // Return empty array for eval testing - no real datatables in test context
-            return [];
-        },
-        getAvailableDatatableNames: () => {
-            // Return empty array for eval testing - no real datatables in test context
-            return [];
-        },
-        execDatatableSql: async (_datatableName, _sql, _newTable) => {
-            // Return success with empty result for eval testing
-            return { success: true, result: [] };
-        },
-        addTableToWhitelist: (_datatableName, _schemaName, _tableName) => {
-            // No-op for eval testing - tables are not tracked in test context
-        }
-    };
-    return {
-        helpers,
-        getFiles: () => ({
-            frontend: { ...frontend },
-            backend: { ...backend }
-        }),
-        getFrontend: () => ({ ...frontend }),
-        getBackend: () => ({ ...backend })
-    };
-}

package/package/components/copilot/chat/__tests__/app/appEvalRunner.d.ts

@@ -1,50 +0,0 @@
-import type { AppFiles, BackendRunnable } from '../../app/core';
-import { type VariantConfig, type BaseEvalResult } from '../shared';
-import type { AIProvider } from '../../../../../gen/types.gen';
-export type { InitialApp } from './appEvalComparison';
-/**
- * App-specific evaluation result.
- */
-export interface AppEvalResult extends BaseEvalResult<AppFiles> {
-    /** Alias for output to maintain API compatibility */
-    files: AppFiles;
-}
-/**
- * Options for running an app evaluation.
- */
-export interface AppEvalOptions {
-    initialFrontend?: Record<string, string>;
-    initialBackend?: Record<string, BackendRunnable>;
-    model?: string;
-    customSystemPrompt?: string;
-    maxIterations?: number;
-    variant?: VariantConfig;
-    /** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */
-    evaluateWithLLM?: boolean;
-    /** AI provider (inferred from model name if omitted) */
-    provider?: AIProvider;
-}
-/**
- * Runs an app chat evaluation using the shared chat loop (same code path as production).
- */
-export declare function runAppEval(userPrompt: string, apiKey: string, options?: AppEvalOptions): Promise<AppEvalResult>;
-/**
- * Per-variant provider override.
- */
-export interface VariantProviderOverride {
-    provider: AIProvider;
-    apiKey: string;
-}
-/**
- * Runs the same prompt against multiple variants sequentially for comparison.
- * Accepts optional per-variant provider/apiKey overrides.
- */
-export declare function runVariantComparison(userPrompt: string, variants: VariantConfig[], defaultApiKey: string, baseOptions?: Omit<AppEvalOptions, 'variant'>, providerOverrides?: VariantProviderOverride[]): Promise<AppEvalResult[]>;
-/**
- * Writes app comparison results to a folder-based structure.
- * Each variant gets its own folder with frontend/, backend/, and details.json.
- */
-export declare function writeAppComparisonResults(userPrompt: string, results: AppEvalResult[], outputDir?: string): Promise<{
-    summaryPath: string;
-    appPaths: string[];
-}>;

package/package/components/copilot/chat/__tests__/app/appEvalRunner.js

@@ -1,93 +0,0 @@
-import { getAppTools, prepareAppSystemMessage, prepareAppUserMessage } from '../../app/core';
-import { createAppEvalHelpers } from './appEvalHelpers';
-import { evaluateAppGeneration } from './appEvalComparison';
-import { runEval, resolveSystemPrompt, resolveTools, resolveModel } from '../shared';
-import { writeAppComparisonResultsToFolders } from './appResultsWriter';
-/**
- * App-specific variant defaults.
- */
-const appDefaults = {
-    prepareSystemMessage: prepareAppSystemMessage,
-    tools: getAppTools()
-};
-/**
- * Runs an app chat evaluation using the shared chat loop (same code path as production).
- */
-export async function runAppEval(userPrompt, apiKey, options) {
-    const { helpers, getFiles } = createAppEvalHelpers(options?.initialFrontend ?? {}, options?.initialBackend ?? {});
-    // Resolve variant configuration
-    const variantName = options?.variant?.name ?? 'baseline';
-    const systemMessage = resolveSystemPrompt(options?.variant, appDefaults, options?.customSystemPrompt);
-    const { tools } = resolveTools(options?.variant, appDefaults);
-    const model = resolveModel(options?.variant, options?.model);
-    // Build user message
-    const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext());
-    // Run the base evaluation
-    const rawResult = await runEval({
-        userPrompt,
-        systemMessage,
-        userMessage,
-        tools,
-        helpers,
-        apiKey,
-        getOutput: getFiles,
-        options: {
-            maxIterations: options?.maxIterations,
-            model,
-            workspace: 'test-workspace',
-            provider: options?.provider
-        }
-    });
-    // Run LLM evaluation unless explicitly disabled
-    let evaluationResult;
-    if (options?.evaluateWithLLM !== false) {
-        const generatedApp = getFiles();
-        const initialApp = options?.initialFrontend || options?.initialBackend
-            ? {
-                frontend: options.initialFrontend ?? {},
-                backend: options.initialBackend ?? {}
-            }
-            : undefined;
-        evaluationResult = await evaluateAppGeneration(userPrompt, generatedApp, initialApp);
-    }
-    return {
-        ...rawResult,
-        variantName,
-        files: rawResult.output,
-        evaluationResult
-    };
-}
-/**
- * Runs the same prompt against multiple variants sequentially for comparison.
- * Accepts optional per-variant provider/apiKey overrides.
- */
-export async function runVariantComparison(userPrompt, variants, defaultApiKey, baseOptions, providerOverrides) {
-    const results = await Promise.all(variants.map(async (variant, i) => {
-        const override = providerOverrides?.[i];
-        return await runAppEval(userPrompt, override?.apiKey ?? defaultApiKey, {
-            ...baseOptions,
-            variant,
-            provider: override?.provider ?? baseOptions?.provider
-        });
-    }));
-    return results;
-}
-/**
- * Writes app comparison results to a folder-based structure.
- * Each variant gets its own folder with frontend/, backend/, and details.json.
- */
-export async function writeAppComparisonResults(userPrompt, results, outputDir) {
-    // @ts-ignore
-    const { dirname, join } = await import('path');
-    // @ts-ignore
-    const { fileURLToPath } = await import('url');
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
-    const resultsDir = outputDir ?? join(__dirname, 'results');
-    const result = await writeAppComparisonResultsToFolders({
-        userPrompt,
-        results,
-        outputDir: resultsDir
-    });
-    return { summaryPath: result.summaryPath, appPaths: result.variantPaths };
-}

package/package/components/copilot/chat/__tests__/app/appFixtureLoader.d.ts

@@ -1,29 +0,0 @@
-import type { AppFiles, BackendRunnable } from '../../app/core';
-/**
- * Loads an app fixture from a directory structure.
- *
- * Expected structure:
- * ```
- * fixturePath/
- * ├── frontend/
- * │   └── index.tsx           # → frontend["/index.tsx"]
- * │   └── components/
- * │       └── Button.tsx      # → frontend["/components/Button.tsx"]
- * └── backend/
- *     └── incrementCounter/
- *         ├── main.ts         # The code content
- *         └── meta.json       # { "name": "...", "language": "bun" }
- * ```
- *
- * @param fixturePath - Path to the fixture directory
- * @returns AppFiles object with frontend and backend
- */
-export declare function loadAppFixture(fixturePath: string): Promise<AppFiles>;
-/**
- * Loads an app fixture and returns the separate frontend and backend objects.
- * Convenience function for use with runAppEval options.
- */
-export declare function loadAppFixtureForEval(fixturePath: string): Promise<{
-    initialFrontend: Record<string, string>;
-    initialBackend: Record<string, BackendRunnable>;
-}>;