windmill-components 1.558.4 → 1.596.0

package/package/components/copilot/chat/ToolContentDisplay.svelte

@@ -1,9 +1,22 @@
 <script lang="ts">import { Loader2, Copy, Check } from 'lucide-svelte';
-let { title, content, error, loading, showCopy = true, showWhileLoading = true } = $props();
+import { TOOL_PRETTIFY_MAP } from './shared';
+let { title, content, error, loading, showCopy = true, showWhileLoading = true, streaming = false, toolName, showFade = false } = $props();
 let copied = $state(false);
 const hasContent = $derived(content !== undefined && content !== null);
+// Look up prettify function from the map using toolName
+const prettifyFn = $derived(toolName ? TOOL_PRETTIFY_MAP[toolName] : undefined);
 function formatJson(obj) {
     try {
+        // Apply prettify function if available for this tool
+        if (prettifyFn) {
+            if (typeof obj === 'object') {
+                return prettifyFn(JSON.stringify(obj, null, 2));
+            }
+            else {
+                return prettifyFn(obj);
+            }
+        }
+        // Original formatting logic as fallback
         if (typeof obj === 'string') {
             try {
                 const parsed = JSON.parse(obj);
@@ -40,15 +53,15 @@ async function copyToClipboard() {
 }
 </script>
 
-{#if showWhileLoading || (!loading && hasContent)}
+{#if showWhileLoading || (!loading && hasContent) || streaming}
 	<div class="space-y-2">
 		<div class="flex items-center justify-between">
-			<span class="text-secondary text-2xs font-semibold uppercase tracking-wide">
+			<span class="text-2xs">
 				{title}:
 			</span>
-			{#if showCopy && hasContent}
+			{#if showCopy && hasContent && !streaming}
 				<button
-					class="p-1 rounded hover:bg-surface-secondary text-tertiary hover:text-secondary transition-colors"
+					class="p-1 rounded hover:bg-surface-secondary text-primary hover:text-secondary transition-colors"
 					onclick={copyToClipboard}
 					title="Copy {title.toLowerCase()}"
 				>
@@ -61,32 +74,39 @@ async function copyToClipboard() {
 			{/if}
 		</div>
 
-		{#if loading}
+		{#if loading && !streaming && !hasContent}
 			<div
-				class="bg-surface-secondary border border-gray-200 dark:border-gray-700 rounded p-3 flex items-center gap-2 text-tertiary"
+				class="bg-surface-secondary border border-gray-200 dark:border-gray-700 rounded p-3 flex items-center gap-2 text-primary"
 			>
 				<Loader2 class="w-3 h-3 animate-spin" />
 				<span class="text-2xs">Executing...</span>
 			</div>
 		{:else if error}
 			<div
-				class="bg-red-50 dark:bg-red-900/20 border border-red-200 dark:border-red-800 rounded p-3 overflow-x-auto max-h-64 overflow-y-auto"
+				class="bg-red-50 dark:bg-red-900/20 border border-red-200 dark:border-red-800 rounded p-3 overflow-x-auto max-h-28 overflow-y-auto"
 			>
 				<pre class="text-2xs text-red-700 dark:text-red-300 whitespace-pre-wrap">{error}</pre>
 			</div>
 		{:else if hasContent}
 			<div
-				class="bg-surface-secondary border border-gray-200 dark:border-gray-700 rounded p-3 overflow-x-auto max-h-64 overflow-y-auto"
+				class="bg-surface-secondary border border-gray-200 dark:border-gray-700 rounded overflow-hidden relative"
 			>
-				<pre class="text-2xs text-primary whitespace-pre-wrap"
-					>{formatJson($state.snapshot(content))}</pre
-				>
+				<div class="p-3 overflow-x-auto max-h-28 overflow-y-auto">
+					<pre class="text-2xs text-primary whitespace-pre-wrap"
+						>{formatJson($state.snapshot(content))}</pre
+					>
+				</div>
+				{#if showFade}
+					<div
+						class="absolute bottom-0 left-0 right-0 h-16 pointer-events-none bg-gradient-to-t from-surface-secondary via-surface-secondary/70 via-surface-secondary/40 to-transparent"
+					></div>
+				{/if}
 			</div>
 		{:else}
 			<div
 				class="bg-surface-secondary border border-gray-200 dark:border-gray-700 rounded p-3 text-center"
 			>
-				<span class="text-2xs text-tertiary">No {title.toLowerCase()} yet</span>
+				<span class="text-2xs text-primary">No {title.toLowerCase()} yet</span>
 			</div>
 		{/if}
 	</div>

package/package/components/copilot/chat/ToolContentDisplay.svelte.d.ts

@@ -5,6 +5,9 @@ interface Props {
     loading?: boolean;
     showCopy?: boolean;
     showWhileLoading?: boolean;
+    streaming?: boolean;
+    toolName?: string;
+    showFade?: boolean;
 }
 declare const ToolContentDisplay: import("svelte").Component<Props, {}, "">;
 type ToolContentDisplay = ReturnType<typeof ToolContentDisplay>;

package/package/components/copilot/chat/ToolExecutionDisplay.svelte

@@ -4,8 +4,10 @@ import { aiChatManager } from './AIChatManager.svelte';
 import { twMerge } from 'tailwind-merge';
 import ToolContentDisplay from './ToolContentDisplay.svelte';
 let { message } = $props();
-let isExpanded = $state(message.showDetails || (message.isLoading && message.needsConfirmation));
 const hasParameters = $derived(message.parameters !== undefined && Object.keys(message.parameters).length > 0);
+let isExpanded = $derived(message.showDetails ||
+    (message.isStreamingArguments && hasParameters) ||
+    (message.isLoading && message.needsConfirmation));
 </script>
 
 <div
@@ -14,14 +16,14 @@ const hasParameters = $derived(message.parameters !== undefined && Object.keys(m
 	<!-- Collapsible Header -->
 	<button
 		class={twMerge(
-			'w-full p-3 bg-surface-secondary hover:bg-surface-hover transition-colors flex items-center justify-between text-left border-b border-gray-200 dark:border-gray-700',
+			'w-full p-2 bg-surface-secondary hover:bg-surface-hover transition-colors flex items-center justify-between text-left border-b border-gray-200 dark:border-gray-700',
 			message.needsConfirmation ? 'opacity-80' : ''
 		)}
 		onclick={() => (isExpanded = !isExpanded)}
-		disabled={!message.showDetails}
+		disabled={!message.showDetails && !message.isStreamingArguments}
 	>
 		<div class="flex items-center gap-2 flex-1">
-			{#if message.showDetails}
+			{#if message.showDetails || message.isStreamingArguments}
 				{#if isExpanded}
 					<ChevronDown class="w-3 h-3 text-secondary" />
 				{:else}
@@ -44,11 +46,19 @@ const hasParameters = $derived(message.parameters !== undefined && Object.keys(m
 
 	<!-- Expanded Content -->
 	{#if isExpanded}
-		<div class="p-3 bg-surface space-y-3">
-			<!-- Parameters Section -->
-			<div class={message.needsConfirmation ? 'opacity-80' : ''}>
-				<ToolContentDisplay title="Parameters" content={message.parameters} />
-			</div>
+		<div class="p-2 bg-surface space-y-3">
+			<!-- Parameters Section - show if we have parameters, or if confirmation is needed (even with empty params) -->
+			{#if hasParameters || message.needsConfirmation}
+				<div class={message.needsConfirmation ? 'opacity-80' : ''}>
+					<ToolContentDisplay
+						title="Parameters"
+						content={message.parameters}
+						streaming={message.isStreamingArguments}
+						toolName={message.toolName}
+						showFade={message.showFade}
+					/>
+				</div>
+			{/if}
 
 			<!-- Confirmation Footer -->
 			{#if message.needsConfirmation}
@@ -59,8 +69,7 @@ const hasParameters = $derived(message.parameters !== undefined && Object.keys(m
 					)}
 				>
 					<Button
-						variant="border"
-						color="gray"
+						variant="default"
 						size="xs"
 						on:click={() => {
 							if (message.tool_call_id) {
@@ -68,10 +77,10 @@ const hasParameters = $derived(message.parameters !== undefined && Object.keys(m
 							}
 						}}
 						startIcon={{ icon: XCircle }}
+						destructive
 					></Button>
 					<Button
-						variant="border"
-						color="green"
+						variant="accent"
 						size="xs"
 						on:click={() => {
 							if (message.tool_call_id) {
@@ -84,8 +93,8 @@ const hasParameters = $derived(message.parameters !== undefined && Object.keys(m
 					</Button>
 				</div>
 
-				<!-- Result Section -->
-			{:else}
+				<!-- Logs and Result - hide while streaming -->
+			{:else if !message.isStreamingArguments}
 				<ToolContentDisplay
 					title="Logs"
 					content={message.logs}

package/package/components/copilot/chat/__tests__/app/appChat.eval.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package/components/copilot/chat/__tests__/app/appChat.eval.test.js

@@ -0,0 +1,137 @@
+import { describe, expect, it } from 'vitest';
+import { runVariantComparison, writeAppComparisonResults } from './appEvalRunner';
+import { BASELINE_VARIANT, STREAMLINED_VARIANT } from './variants';
+import { loadAppFixtureForEval } from './appFixtureLoader';
+// @ts-ignore - Node.js path
+import { dirname, join } from 'path';
+// @ts-ignore - Node.js url
+import { fileURLToPath } from 'url';
+// Get API key from environment - tests will be skipped if not set
+// @ts-ignore
+const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
+// Skip all tests if no API key is provided
+const describeWithApiKey = OPENROUTER_API_KEY ? describe : describe.skip;
+// Get __dirname equivalent for ES modules
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const MODELS = ['google/gemini-2.5-flash', 'anthropic/claude-haiku-4.5', 'openai/gpt-4o'];
+const VARIANTS = [
+    ...MODELS.map((model) => ({
+        ...BASELINE_VARIANT,
+        model,
+        name: `baseline-${model.replace('/', '-')}`
+    })),
+    ...MODELS.map((model) => ({
+        ...STREAMLINED_VARIANT,
+        model,
+        name: `streamlined-${model.replace('/', '-')}`
+    }))
+];
+describeWithApiKey('App Chat LLM Evaluation', () => {
+    const TEST_TIMEOUT = 120_000;
+    if (!OPENROUTER_API_KEY) {
+        console.warn('OPENROUTER_API_KEY is not set, skipping tests');
+    }
+    it('test1: creates a simple counter app', async () => {
+        const USER_PROMPT = `Create a counter app with increment/decrement buttons`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY);
+        // Write results to files
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    it('test2: modifies existing counter app to add reset button', async () => {
+        // Load initial app from fixture folder
+        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'test1_counter_app'));
+        const USER_PROMPT = `Add a reset button that sets the counter back to 0`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+            initialFrontend,
+            initialBackend
+        });
+        // Write results to files
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    // ==================== Shopping Cart Tests ====================
+    it('test3: shopping cart - add quantity selector', async () => {
+        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'shopping_cart'));
+        const USER_PROMPT = `Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+            initialFrontend,
+            initialBackend
+        });
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    it('test4: shopping cart - add discount code', async () => {
+        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'shopping_cart'));
+        const USER_PROMPT = `Add a discount code input field in the cart. When the code "SAVE10" is entered, apply a 10% discount to the total`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+            initialFrontend,
+            initialBackend
+        });
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    // ==================== File Manager Tests ====================
+    it('test5: file manager - add search bar', async () => {
+        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
+        const USER_PROMPT = `Add a search bar in the toolbar that filters files and folders by name as the user types`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+            initialFrontend,
+            initialBackend
+        });
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    it('test6: file manager - show file details', async () => {
+        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
+        const USER_PROMPT = `Show file size (formatted as KB/MB) and modified date in the file list for each item`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+            initialFrontend,
+            initialBackend
+        });
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    it('test7: file manager - add select all checkbox', async () => {
+        const { initialFrontend, initialBackend } = await loadAppFixtureForEval(join(__dirname, 'initial', 'file_manager'));
+        const USER_PROMPT = `Add a "Select All" checkbox in the file list header and individual checkboxes for each file. Add a "Delete Selected" button that appears when items are selected`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY, {
+            initialFrontend,
+            initialBackend
+        });
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    // ==================== From-Scratch Creation Tests ====================
+    it('test8: create quiz app from scratch', async () => {
+        const USER_PROMPT = `Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct.`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY);
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+    it('test9: create recipe book from scratch', async () => {
+        const USER_PROMPT = `Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes.`;
+        const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY);
+        const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results);
+        console.log(`\nResults written to: ${summaryPath}`);
+        console.log(`App files: ${appPaths.join(', ')}`);
+        expect(true).toBe(true);
+    }, TEST_TIMEOUT);
+});

package/package/components/copilot/chat/__tests__/app/appEvalComparison.d.ts

@@ -0,0 +1,26 @@
+import type { AppFiles, BackendRunnable } from '../../app/core';
+import type { EvaluationResult } from '../shared';
+/**
+ * Expected app structure for evaluation.
+ */
+export interface ExpectedApp {
+    frontend: Record<string, string>;
+    backend: Record<string, BackendRunnable>;
+}
+/**
+ * Initial app state for evaluation context.
+ */
+export interface InitialApp {
+    frontend: Record<string, string>;
+    backend: Record<string, BackendRunnable>;
+}
+/**
+ * Evaluates how well a generated app fulfills the user's request, considering any initial app state.
+ * This evaluator does not require an expected reference app - it evaluates based on the request alone.
+ *
+ * @param userPrompt The original user request
+ * @param generatedApp The app generated by the AI
+ * @param initialApp Optional initial app state (what the app looked like before AI changes)
+ * @returns Evaluation result with score, statement, and missing requirements
+ */
+export declare function evaluateAppGeneration(userPrompt: string, generatedApp: AppFiles, initialApp?: InitialApp): Promise<EvaluationResult>;

package/package/components/copilot/chat/__tests__/app/appEvalComparison.js

@@ -0,0 +1,131 @@
+import OpenAI from 'openai';
+import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared';
+/**
+ * System prompt for evaluating app generation without a reference expected app.
+ * Evaluates based on user request fulfillment and appropriate modifications to initial state.
+ */
+const APP_GENERATION_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill Raw App definitions. Your task is to evaluate a generated app based on:
+1. The original user request/prompt
+2. The initial app state (if any) - this is what the app looked like before the AI made changes
+
+## Windmill Raw App Context
+- Raw Apps consist of frontend files and backend runnables
+- Frontend files are TypeScript/JavaScript files bundled with esbuild (entrypoint: index.tsx)
+- Backend runnables can be: inline scripts (TypeScript/Python), workspace scripts, workspace flows, or hub scripts
+- Frontend calls backend using \`await backend.<runnable_key>(args...)\`
+- Each backend runnable has a key (identifier), name (description), type, and configuration
+
+## Backend Runnable Types
+- **inline**: Custom code with \`inlineScript.language\` and \`inlineScript.content\`
+- **script**: Workspace script reference with \`path\`
+- **flow**: Workspace flow reference with \`path\`
+- **hubscript**: Hub script reference with \`path\`
+
+## Evaluation Criteria
+1. **User Request Fulfillment**: Does the generated app address ALL requirements from the user's original prompt?
+   - Are all requested features implemented?
+   - Does the frontend UI match the requirements?
+   - Are the correct backend runnables created?
+2. **Appropriate Modifications** (if initial app was provided):
+   - Were the changes made relevant to the user's request?
+   - Was existing functionality preserved where appropriate?
+   - Were only necessary changes made (no unnecessary removals or additions)?
+3. **Frontend Structure**: Are the frontend files correctly organized and implemented?
+   - Is the code valid TypeScript/JavaScript?
+   - Are components properly structured?
+   - Are backend calls correctly made?
+4. **Backend Structure**: Are the backend runnables correctly configured?
+   - Do inline scripts have proper main functions?
+   - Are types and paths correct for non-inline runnables?
+5. **Integration**: Does the frontend correctly call the backend?
+   - Are the runnable keys correctly referenced?
+   - Are arguments passed correctly?
+6. **Code Quality**: Is the code functionally correct and well-structured?
+
+## Important Notes
+- Focus on whether the user's request was fulfilled, not on stylistic preferences
+- If an initial app was provided, evaluate the appropriateness of the changes made
+- For new apps (no initial state), evaluate completeness and correctness
+- Extra helper functions or slightly different approaches can still score high if they accomplish the goal
+
+${BASE_EVALUATOR_RESPONSE_FORMAT}`;
+/**
+ * Evaluates how well a generated app fulfills the user's request, considering any initial app state.
+ * This evaluator does not require an expected reference app - it evaluates based on the request alone.
+ *
+ * @param userPrompt The original user request
+ * @param generatedApp The app generated by the AI
+ * @param initialApp Optional initial app state (what the app looked like before AI changes)
+ * @returns Evaluation result with score, statement, and missing requirements
+ */
+export async function evaluateAppGeneration(userPrompt, generatedApp, initialApp) {
+    // @ts-ignore
+    const apiKey = process.env.OPENROUTER_API_KEY;
+    const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey });
+    let userMessage = `## User's Original Request
+${userPrompt}
+
+`;
+    if (initialApp) {
+        userMessage += `## Initial App State (before AI modifications)
+\`\`\`json
+${JSON.stringify(initialApp, null, 2)}
+\`\`\`
+
+`;
+    }
+    else {
+        userMessage += `## Initial App State
+No initial app was provided - this is a new app created from scratch.
+
+`;
+    }
+    userMessage += `## Generated App
+\`\`\`json
+${JSON.stringify(generatedApp, null, 2)}
+\`\`\`
+
+Please evaluate how well the generated app:
+1. Fulfills ALL requirements from the user's original request
+2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}`;
+    try {
+        const response = await client.chat.completions.create({
+            model: 'anthropic/claude-sonnet-4.5',
+            messages: [
+                { role: 'system', content: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT },
+                { role: 'user', content: userMessage }
+            ],
+            temperature: 0
+        });
+        const content = response.choices[0]?.message?.content;
+        if (!content) {
+            return {
+                success: false,
+                resemblanceScore: 0,
+                statement: 'No response from evaluator',
+                error: 'Empty response from LLM'
+            };
+        }
+        // Parse JSON response - handle potential markdown code blocks
+        let jsonContent = content.trim();
+        if (jsonContent.startsWith('```')) {
+            jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
+        }
+        const parsed = JSON.parse(jsonContent);
+        return {
+            success: true,
+            resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
+            statement: parsed.statement,
+            missingRequirements: parsed.missingRequirements ?? []
+        };
+    }
+    catch (err) {
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        return {
+            success: false,
+            resemblanceScore: 0,
+            statement: 'Evaluation failed',
+            error: errorMessage
+        };
+    }
+}

package/package/components/copilot/chat/__tests__/app/appEvalHelpers.d.ts

@@ -0,0 +1,15 @@
+import type { AppAIChatHelpers, AppFiles, BackendRunnable } from '../../app/core';
+/**
+ * Creates mock AppAIChatHelpers for eval testing.
+ * Tracks app files state in memory and allows tool functions to modify it.
+ */
+export declare function createAppEvalHelpers(initialFrontend?: Record<string, string>, initialBackend?: Record<string, BackendRunnable>): {
+    helpers: AppAIChatHelpers;
+    getFiles: () => AppFiles;
+    getFrontend: () => {
+        [x: string]: string;
+    };
+    getBackend: () => {
+        [x: string]: BackendRunnable;
+    };
+};

package/package/components/copilot/chat/__tests__/app/appEvalHelpers.js

@@ -0,0 +1,91 @@
+/**
+ * Creates an empty lint result (no errors or warnings).
+ */
+function createEmptyLintResult() {
+    return {
+        errorCount: 0,
+        warningCount: 0,
+        errors: { frontend: {}, backend: {} },
+        warnings: { frontend: {}, backend: {} }
+    };
+}
+/**
+ * Creates mock AppAIChatHelpers for eval testing.
+ * Tracks app files state in memory and allows tool functions to modify it.
+ */
+export function createAppEvalHelpers(initialFrontend = {}, initialBackend = {}) {
+    // In-memory state
+    let frontend = { ...initialFrontend };
+    let backend = { ...initialBackend };
+    let snapshotId = 0;
+    const snapshots = new Map();
+    const helpers = {
+        // Frontend file operations
+        listFrontendFiles: () => Object.keys(frontend),
+        getFrontendFile: (path) => frontend[path],
+        getFrontendFiles: () => ({ ...frontend }),
+        setFrontendFile: (path, content) => {
+            frontend[path] = content;
+            // Return mock lint result - in real usage this would validate the file
+            return createEmptyLintResult();
+        },
+        deleteFrontendFile: (path) => {
+            delete frontend[path];
+        },
+        // Backend runnable operations
+        listBackendRunnables: () => {
+            return Object.entries(backend).map(([key, runnable]) => ({
+                key,
+                name: runnable.name
+            }));
+        },
+        getBackendRunnable: (key) => backend[key],
+        getBackendRunnables: () => ({ ...backend }),
+        setBackendRunnable: async (key, runnable) => {
+            backend[key] = runnable;
+            // Return mock lint result - in real usage this would validate the runnable
+            return createEmptyLintResult();
+        },
+        deleteBackendRunnable: (key) => {
+            delete backend[key];
+        },
+        // Combined view
+        getFiles: () => ({
+            frontend: { ...frontend },
+            backend: { ...backend }
+        }),
+        getSelectedContext: () => ({
+            type: 'none'
+        }),
+        // Snapshot management
+        snapshot: () => {
+            const id = ++snapshotId;
+            snapshots.set(id, {
+                frontend: { ...frontend },
+                backend: { ...backend }
+            });
+            return id;
+        },
+        revertToSnapshot: (id) => {
+            const snap = snapshots.get(id);
+            if (snap) {
+                frontend = { ...snap.frontend };
+                backend = { ...snap.backend };
+            }
+        },
+        // Linting
+        lint: () => {
+            // Return mock lint result - no actual linting in eval
+            return createEmptyLintResult();
+        }
+    };
+    return {
+        helpers,
+        getFiles: () => ({
+            frontend: { ...frontend },
+            backend: { ...backend }
+        }),
+        getFrontend: () => ({ ...frontend }),
+        getBackend: () => ({ ...backend })
+    };
+}

package/package/components/copilot/chat/__tests__/app/appEvalRunner.d.ts

@@ -0,0 +1,41 @@
+import type { AppFiles, BackendRunnable } from '../../app/core';
+import { type VariantConfig, type BaseEvalResult } from '../shared';
+export type { InitialApp } from './appEvalComparison';
+/**
+ * App-specific evaluation result.
+ */
+export interface AppEvalResult extends BaseEvalResult<AppFiles> {
+    /** Alias for output to maintain API compatibility */
+    files: AppFiles;
+}
+/**
+ * Options for running an app evaluation.
+ */
+export interface AppEvalOptions {
+    initialFrontend?: Record<string, string>;
+    initialBackend?: Record<string, BackendRunnable>;
+    model?: string;
+    customSystemPrompt?: string;
+    maxIterations?: number;
+    variant?: VariantConfig;
+    /** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */
+    evaluateWithLLM?: boolean;
+}
+/**
+ * Runs an app chat evaluation with real OpenAI API calls.
+ * Executes tool calls using the actual app tools from core.ts or variant-configured tools.
+ */
+export declare function runAppEval(userPrompt: string, openaiApiKey: string, options?: AppEvalOptions): Promise<AppEvalResult>;
+/**
+ * Runs the same prompt against multiple variants sequentially for comparison.
+ * Returns results in the same order as the input variants.
+ */
+export declare function runVariantComparison(userPrompt: string, variants: VariantConfig[], openaiApiKey: string, baseOptions?: Omit<AppEvalOptions, 'variant'>): Promise<AppEvalResult[]>;
+/**
+ * Writes app comparison results to a folder-based structure.
+ * Each variant gets its own folder with frontend/, backend/, and details.json.
+ */
+export declare function writeAppComparisonResults(userPrompt: string, results: AppEvalResult[], outputDir?: string): Promise<{
+    summaryPath: string;
+    appPaths: string[];
+}>;