windmill-components 1.677.1 → 1.695.0

package/package/components/copilot/chat/__tests__/app/appEvalComparison.js

@@ -1,136 +0,0 @@
-import Anthropic from '@anthropic-ai/sdk';
-import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared';
-/**
- * System prompt for evaluating app generation without a reference expected app.
- * Evaluates based on user request fulfillment and appropriate modifications to initial state.
- */
-const APP_GENERATION_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill Raw App definitions. Your task is to evaluate a generated app based on:
-1. The original user request/prompt
-2. The initial app state (if any) - this is what the app looked like before the AI made changes
-
-## Windmill Raw App Context
-- Raw Apps consist of frontend files and backend runnables
-- Frontend files are TypeScript/JavaScript files bundled with esbuild (entrypoint: index.tsx)
-- Backend runnables can be: inline scripts (TypeScript/Python), workspace scripts, workspace flows, or hub scripts
-- Frontend calls backend using \`await backend.<runnable_key>(args...)\`
-- Each backend runnable has a key (identifier), name (description), type, and configuration
-
-## Backend Runnable Types
-- **inline**: Custom code with \`inlineScript.language\` and \`inlineScript.content\`
-- **script**: Workspace script reference with \`path\`
-- **flow**: Workspace flow reference with \`path\`
-- **hubscript**: Hub script reference with \`path\`
-
-## Evaluation Criteria
-1. **User Request Fulfillment**: Does the generated app address ALL requirements from the user's original prompt?
-   - Are all requested features implemented?
-   - Does the frontend UI match the requirements?
-   - Are the correct backend runnables created?
-2. **Appropriate Modifications** (if initial app was provided):
-   - Were the changes made relevant to the user's request?
-   - Was existing functionality preserved where appropriate?
-   - Were only necessary changes made (no unnecessary removals or additions)?
-3. **Frontend Structure**: Are the frontend files correctly organized and implemented?
-   - Is the code valid TypeScript/JavaScript?
-   - Are components properly structured?
-   - Are backend calls correctly made?
-4. **Backend Structure**: Are the backend runnables correctly configured?
-   - Do inline scripts have proper main functions?
-   - Are types and paths correct for non-inline runnables?
-5. **Integration**: Does the frontend correctly call the backend?
-   - Are the runnable keys correctly referenced?
-   - Are arguments passed correctly?
-6. **Code Quality**: Is the code functionally correct and well-structured?
-
-## Important Notes
-- Focus on whether the user's request was fulfilled, not on stylistic preferences
-- If an initial app was provided, evaluate the appropriateness of the changes made
-- For new apps (no initial state), evaluate completeness and correctness
-- Extra helper functions or slightly different approaches can still score high if they accomplish the goal
-
-${BASE_EVALUATOR_RESPONSE_FORMAT}`;
-/**
- * Evaluates how well a generated app fulfills the user's request, considering any initial app state.
- * Uses Anthropic API directly.
- */
-export async function evaluateAppGeneration(userPrompt, generatedApp, initialApp) {
-    // @ts-ignore
-    const apiKey = process.env.ANTHROPIC_API_KEY;
-    if (!apiKey) {
-        return {
-            success: false,
-            resemblanceScore: 0,
-            statement: 'No API key available for evaluation',
-            error: 'ANTHROPIC_API_KEY not set'
-        };
-    }
-    const client = new Anthropic({ apiKey });
-    let userMessage = `## User's Original Request
-${userPrompt}
-
-`;
-    if (initialApp) {
-        userMessage += `## Initial App State (before AI modifications)
-\`\`\`json
-${JSON.stringify(initialApp, null, 2)}
-\`\`\`
-
-`;
-    }
-    else {
-        userMessage += `## Initial App State
-No initial app was provided - this is a new app created from scratch.
-
-`;
-    }
-    userMessage += `## Generated App
-\`\`\`json
-${JSON.stringify(generatedApp, null, 2)}
-\`\`\`
-
-Please evaluate how well the generated app:
-1. Fulfills ALL requirements from the user's original request
-2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}`;
-    try {
-        const response = await client.messages.create({
-            model: 'claude-sonnet-4-5-20250514',
-            max_tokens: 2048,
-            system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT,
-            messages: [
-                { role: 'user', content: userMessage }
-            ],
-            temperature: 0
-        });
-        const textBlock = response.content.find((block) => block.type === 'text');
-        const content = textBlock?.text;
-        if (!content) {
-            return {
-                success: false,
-                resemblanceScore: 0,
-                statement: 'No response from evaluator',
-                error: 'Empty response from LLM'
-            };
-        }
-        // Parse JSON response - handle potential markdown code blocks
-        let jsonContent = content.trim();
-        if (jsonContent.startsWith('```')) {
-            jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
-        }
-        const parsed = JSON.parse(jsonContent);
-        return {
-            success: true,
-            resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
-            statement: parsed.statement,
-            missingRequirements: parsed.missingRequirements ?? []
-        };
-    }
-    catch (err) {
-        const errorMessage = err instanceof Error ? err.message : String(err);
-        return {
-            success: false,
-            resemblanceScore: 0,
-            statement: 'Evaluation failed',
-            error: errorMessage
-        };
-    }
-}

package/package/components/copilot/chat/__tests__/app/appEvalHelpers.d.ts

@@ -1,15 +0,0 @@
-import type { AppAIChatHelpers, AppFiles, BackendRunnable } from '../../app/core';
-/**
- * Creates mock AppAIChatHelpers for eval testing.
- * Tracks app files state in memory and allows tool functions to modify it.
- */
-export declare function createAppEvalHelpers(initialFrontend?: Record<string, string>, initialBackend?: Record<string, BackendRunnable>): {
-    helpers: AppAIChatHelpers;
-    getFiles: () => AppFiles;
-    getFrontend: () => {
-        [x: string]: string;
-    };
-    getBackend: () => {
-        [x: string]: BackendRunnable;
-    };
-};

package/package/components/copilot/chat/__tests__/app/appEvalHelpers.js

@@ -1,107 +0,0 @@
-/**
- * Creates an empty lint result (no errors or warnings).
- */
-function createEmptyLintResult() {
-    return {
-        errorCount: 0,
-        warningCount: 0,
-        errors: { frontend: {}, backend: {} },
-        warnings: { frontend: {}, backend: {} }
-    };
-}
-/**
- * Creates mock AppAIChatHelpers for eval testing.
- * Tracks app files state in memory and allows tool functions to modify it.
- */
-export function createAppEvalHelpers(initialFrontend = {}, initialBackend = {}) {
-    // In-memory state
-    let frontend = { ...initialFrontend };
-    let backend = { ...initialBackend };
-    let snapshotId = 0;
-    const snapshots = new Map();
-    const helpers = {
-        // Frontend file operations
-        listFrontendFiles: () => Object.keys(frontend),
-        getFrontendFile: (path) => frontend[path],
-        getFrontendFiles: () => ({ ...frontend }),
-        setFrontendFile: (path, content) => {
-            frontend[path] = content;
-            // Return mock lint result - in real usage this would validate the file
-            return createEmptyLintResult();
-        },
-        deleteFrontendFile: (path) => {
-            delete frontend[path];
-        },
-        // Backend runnable operations
-        listBackendRunnables: () => {
-            return Object.entries(backend).map(([key, runnable]) => ({
-                key,
-                name: runnable.name
-            }));
-        },
-        getBackendRunnable: (key) => backend[key],
-        getBackendRunnables: () => ({ ...backend }),
-        setBackendRunnable: async (key, runnable) => {
-            backend[key] = runnable;
-            // Return mock lint result - in real usage this would validate the runnable
-            return createEmptyLintResult();
-        },
-        deleteBackendRunnable: (key) => {
-            delete backend[key];
-        },
-        // Combined view
-        getFiles: () => ({
-            frontend: { ...frontend },
-            backend: { ...backend }
-        }),
-        getSelectedContext: () => ({
-            type: 'none'
-        }),
-        // Snapshot management
-        snapshot: () => {
-            const id = ++snapshotId;
-            snapshots.set(id, {
-                frontend: { ...frontend },
-                backend: { ...backend }
-            });
-            return id;
-        },
-        revertToSnapshot: (id) => {
-            const snap = snapshots.get(id);
-            if (snap) {
-                frontend = { ...snap.frontend };
-                backend = { ...snap.backend };
-            }
-        },
-        // Linting
-        lint: () => {
-            // Return mock lint result - no actual linting in eval
-            return createEmptyLintResult();
-        },
-        // Data table operations (mock implementation for testing)
-        getDatatables: async () => {
-            // Return empty array for eval testing - no real datatables in test context
-            return [];
-        },
-        getAvailableDatatableNames: () => {
-            // Return empty array for eval testing - no real datatables in test context
-            return [];
-        },
-        execDatatableSql: async (_datatableName, _sql, _newTable) => {
-            // Return success with empty result for eval testing
-            return { success: true, result: [] };
-        },
-        addTableToWhitelist: (_datatableName, _schemaName, _tableName) => {
-            // No-op for eval testing - tables are not tracked in test context
-        }
-    };
-    return {
-        helpers,
-        getFiles: () => ({
-            frontend: { ...frontend },
-            backend: { ...backend }
-        }),
-        getFrontend: () => ({ ...frontend }),
-        getBackend: () => ({ ...backend })
-    };
-}

package/package/components/copilot/chat/__tests__/app/appEvalRunner.d.ts

@@ -1,50 +0,0 @@
-import type { AppFiles, BackendRunnable } from '../../app/core';
-import { type VariantConfig, type BaseEvalResult } from '../shared';
-import type { AIProvider } from '../../../../../gen/types.gen';
-export type { InitialApp } from './appEvalComparison';
-/**
- * App-specific evaluation result.
- */
-export interface AppEvalResult extends BaseEvalResult<AppFiles> {
-    /** Alias for output to maintain API compatibility */
-    files: AppFiles;
-}
-/**
- * Options for running an app evaluation.
- */
-export interface AppEvalOptions {
-    initialFrontend?: Record<string, string>;
-    initialBackend?: Record<string, BackendRunnable>;
-    model?: string;
-    customSystemPrompt?: string;
-    maxIterations?: number;
-    variant?: VariantConfig;
-    /** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */
-    evaluateWithLLM?: boolean;
-    /** AI provider (inferred from model name if omitted) */
-    provider?: AIProvider;
-}
-/**
- * Runs an app chat evaluation using the shared chat loop (same code path as production).
- */
-export declare function runAppEval(userPrompt: string, apiKey: string, options?: AppEvalOptions): Promise<AppEvalResult>;
-/**
- * Per-variant provider override.
- */
-export interface VariantProviderOverride {
-    provider: AIProvider;
-    apiKey: string;
-}
-/**
- * Runs the same prompt against multiple variants sequentially for comparison.
- * Accepts optional per-variant provider/apiKey overrides.
- */
-export declare function runVariantComparison(userPrompt: string, variants: VariantConfig[], defaultApiKey: string, baseOptions?: Omit<AppEvalOptions, 'variant'>, providerOverrides?: VariantProviderOverride[]): Promise<AppEvalResult[]>;
-/**
- * Writes app comparison results to a folder-based structure.
- * Each variant gets its own folder with frontend/, backend/, and details.json.
- */
-export declare function writeAppComparisonResults(userPrompt: string, results: AppEvalResult[], outputDir?: string): Promise<{
-    summaryPath: string;
-    appPaths: string[];
-}>;

package/package/components/copilot/chat/__tests__/app/appEvalRunner.js

@@ -1,93 +0,0 @@
-import { getAppTools, prepareAppSystemMessage, prepareAppUserMessage } from '../../app/core';
-import { createAppEvalHelpers } from './appEvalHelpers';
-import { evaluateAppGeneration } from './appEvalComparison';
-import { runEval, resolveSystemPrompt, resolveTools, resolveModel } from '../shared';
-import { writeAppComparisonResultsToFolders } from './appResultsWriter';
-/**
- * App-specific variant defaults.
- */
-const appDefaults = {
-    prepareSystemMessage: prepareAppSystemMessage,
-    tools: getAppTools()
-};
-/**
- * Runs an app chat evaluation using the shared chat loop (same code path as production).
- */
-export async function runAppEval(userPrompt, apiKey, options) {
-    const { helpers, getFiles } = createAppEvalHelpers(options?.initialFrontend ?? {}, options?.initialBackend ?? {});
-    // Resolve variant configuration
-    const variantName = options?.variant?.name ?? 'baseline';
-    const systemMessage = resolveSystemPrompt(options?.variant, appDefaults, options?.customSystemPrompt);
-    const { tools } = resolveTools(options?.variant, appDefaults);
-    const model = resolveModel(options?.variant, options?.model);
-    // Build user message
-    const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext());
-    // Run the base evaluation
-    const rawResult = await runEval({
-        userPrompt,
-        systemMessage,
-        userMessage,
-        tools,
-        helpers,
-        apiKey,
-        getOutput: getFiles,
-        options: {
-            maxIterations: options?.maxIterations,
-            model,
-            workspace: 'test-workspace',
-            provider: options?.provider
-        }
-    });
-    // Run LLM evaluation unless explicitly disabled
-    let evaluationResult;
-    if (options?.evaluateWithLLM !== false) {
-        const generatedApp = getFiles();
-        const initialApp = options?.initialFrontend || options?.initialBackend
-            ? {
-                frontend: options.initialFrontend ?? {},
-                backend: options.initialBackend ?? {}
-            }
-            : undefined;
-        evaluationResult = await evaluateAppGeneration(userPrompt, generatedApp, initialApp);
-    }
-    return {
-        ...rawResult,
-        variantName,
-        files: rawResult.output,
-        evaluationResult
-    };
-}
-/**
- * Runs the same prompt against multiple variants sequentially for comparison.
- * Accepts optional per-variant provider/apiKey overrides.
- */
-export async function runVariantComparison(userPrompt, variants, defaultApiKey, baseOptions, providerOverrides) {
-    const results = await Promise.all(variants.map(async (variant, i) => {
-        const override = providerOverrides?.[i];
-        return await runAppEval(userPrompt, override?.apiKey ?? defaultApiKey, {
-            ...baseOptions,
-            variant,
-            provider: override?.provider ?? baseOptions?.provider
-        });
-    }));
-    return results;
-}
-/**
- * Writes app comparison results to a folder-based structure.
- * Each variant gets its own folder with frontend/, backend/, and details.json.
- */
-export async function writeAppComparisonResults(userPrompt, results, outputDir) {
-    // @ts-ignore
-    const { dirname, join } = await import('path');
-    // @ts-ignore
-    const { fileURLToPath } = await import('url');
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
-    const resultsDir = outputDir ?? join(__dirname, 'results');
-    const result = await writeAppComparisonResultsToFolders({
-        userPrompt,
-        results,
-        outputDir: resultsDir
-    });
-    return { summaryPath: result.summaryPath, appPaths: result.variantPaths };
-}

package/package/components/copilot/chat/__tests__/app/appFixtureLoader.d.ts

@@ -1,29 +0,0 @@
-import type { AppFiles, BackendRunnable } from '../../app/core';
-/**
- * Loads an app fixture from a directory structure.
- *
- * Expected structure:
- * ```
- * fixturePath/
- * ├── frontend/
- * │   └── index.tsx           # → frontend["/index.tsx"]
- * │   └── components/
- * │       └── Button.tsx      # → frontend["/components/Button.tsx"]
- * └── backend/
- *     └── incrementCounter/
- *         ├── main.ts         # The code content
- *         └── meta.json       # { "name": "...", "language": "bun" }
- * ```
- *
- * @param fixturePath - Path to the fixture directory
- * @returns AppFiles object with frontend and backend
- */
-export declare function loadAppFixture(fixturePath: string): Promise<AppFiles>;
-/**
- * Loads an app fixture and returns the separate frontend and backend objects.
- * Convenience function for use with runAppEval options.
- */
-export declare function loadAppFixtureForEval(fixturePath: string): Promise<{
-    initialFrontend: Record<string, string>;
-    initialBackend: Record<string, BackendRunnable>;
-}>;

package/package/components/copilot/chat/__tests__/app/appFixtureLoader.js

@@ -1,134 +0,0 @@
-/**
- * Recursively reads all files in a directory and returns them as a record.
- * File paths are relative to the base directory with a leading '/'.
- */
-async function readFilesRecursively(dir, basePath = '') {
-    // @ts-ignore - Node.js fs/promises
-    const { readdir, readFile } = await import('fs/promises');
-    // @ts-ignore - Node.js path
-    const { join } = await import('path');
-    const result = {};
-    const entries = await readdir(dir, { withFileTypes: true });
-    for (const entry of entries) {
-        const fullPath = join(dir, entry.name);
-        const relativePath = basePath ? `${basePath}/${entry.name}` : `/${entry.name}`;
-        if (entry.isDirectory()) {
-            const subFiles = await readFilesRecursively(fullPath, relativePath);
-            Object.assign(result, subFiles);
-        }
-        else {
-            const content = await readFile(fullPath, 'utf-8');
-            result[relativePath] = content;
-        }
-    }
-    return result;
-}
-/**
- * Loads frontend files from a directory.
- * All files are read recursively and paths become keys with leading '/'.
- */
-async function loadFrontend(frontendPath) {
-    // @ts-ignore - Node.js fs/promises
-    const { access } = await import('fs/promises');
-    try {
-        await access(frontendPath);
-    }
-    catch {
-        // Directory doesn't exist, return empty
-        return {};
-    }
-    return readFilesRecursively(frontendPath);
-}
-/**
- * Loads backend runnables from a directory.
- * Each subdirectory is a runnable with:
- * - main.ts or main.py: The code content
- * - meta.json: Metadata { name, language }
- */
-async function loadBackend(backendPath) {
-    // @ts-ignore - Node.js fs/promises
-    const { readdir, readFile, access } = await import('fs/promises');
-    // @ts-ignore - Node.js path
-    const { join } = await import('path');
-    try {
-        await access(backendPath);
-    }
-    catch {
-        // Directory doesn't exist, return empty
-        return {};
-    }
-    const result = {};
-    const entries = await readdir(backendPath, { withFileTypes: true });
-    for (const entry of entries) {
-        if (!entry.isDirectory())
-            continue;
-        const runnableKey = entry.name;
-        const runnablePath = join(backendPath, entry.name);
-        // Read meta.json
-        const metaPath = join(runnablePath, 'meta.json');
-        let meta;
-        try {
-            const metaContent = await readFile(metaPath, 'utf-8');
-            meta = JSON.parse(metaContent);
-        }
-        catch {
-            console.warn(`Missing or invalid meta.json for runnable '${runnableKey}', skipping`);
-            continue;
-        }
-        // Find and read the main file (main.ts or main.py)
-        const runnableFiles = await readdir(runnablePath);
-        const mainFile = runnableFiles.find((f) => f === 'main.ts' || f === 'main.py');
-        if (!mainFile) {
-            console.warn(`No main.ts or main.py found for runnable '${runnableKey}', skipping`);
-            continue;
-        }
-        const content = await readFile(join(runnablePath, mainFile), 'utf-8');
-        const inlineScript = {
-            language: meta.language,
-            content
-        };
-        result[runnableKey] = {
-            name: meta.name,
-            type: 'inline',
-            inlineScript
-        };
-    }
-    return result;
-}
-/**
- * Loads an app fixture from a directory structure.
- *
- * Expected structure:
- * ```
- * fixturePath/
- * ├── frontend/
- * │   └── index.tsx           # → frontend["/index.tsx"]
- * │   └── components/
- * │       └── Button.tsx      # → frontend["/components/Button.tsx"]
- * └── backend/
- *     └── incrementCounter/
- *         ├── main.ts         # The code content
- *         └── meta.json       # { "name": "...", "language": "bun" }
- * ```
- *
- * @param fixturePath - Path to the fixture directory
- * @returns AppFiles object with frontend and backend
- */
-export async function loadAppFixture(fixturePath) {
-    // @ts-ignore - Node.js path
-    const { join } = await import('path');
-    const frontend = await loadFrontend(join(fixturePath, 'frontend'));
-    const backend = await loadBackend(join(fixturePath, 'backend'));
-    return { frontend, backend };
-}
-/**
- * Loads an app fixture and returns the separate frontend and backend objects.
- * Convenience function for use with runAppEval options.
- */
-export async function loadAppFixtureForEval(fixturePath) {
-    const { frontend, backend } = await loadAppFixture(fixturePath);
-    return {
-        initialFrontend: frontend,
-        initialBackend: backend
-    };
-}

package/package/components/copilot/chat/__tests__/app/appResultsWriter.d.ts

@@ -1,30 +0,0 @@
-import type { AppEvalResult } from './appEvalRunner';
-/**
- * Parameters for writing app comparison results.
- */
-export interface WriteAppResultsParams {
-    userPrompt: string;
-    results: AppEvalResult[];
-    outputDir: string;
-}
-/**
- * Writes app comparison results to a folder-based structure.
- *
- * Creates:
- * ```
- * results/{timestamp}/
- * ├── summary.md
- * └── {variant_name}/
- *     ├── details.json    # Metadata (toolsCalled, evaluationResult, etc.)
- *     ├── frontend/       # Frontend files
- *     │   └── index.tsx
- *     └── backend/        # Backend runnables
- *         └── myFunction/
- *             ├── main.ts
- *             └── meta.json
- * ```
- */
-export declare function writeAppComparisonResultsToFolders(params: WriteAppResultsParams): Promise<{
-    summaryPath: string;
-    variantPaths: string[];
-}>;