windmill-components 1.558.3 → 1.596.0

package/package/components/copilot/chat/__tests__/shared/baseLLMEvaluator.js

@@ -0,0 +1,88 @@
+import OpenAI from 'openai';
+/**
+ * Base evaluator system prompt template.
+ * Domain-specific evaluators should build on this structure.
+ */
+export const BASE_EVALUATOR_RESPONSE_FORMAT = `
+## Response Format
+You MUST respond with valid JSON only, no additional text:
+{
+  "resemblanceScore": <0-100 integer>,
+  "statement": "<brief 1-2 sentence summary of how well the output matches the user request and expected output>",
+  "missingRequirements": ["<list any requirements from user prompt that are missing or incorrectly implemented>"]
+}
+
+Score guidelines:
+- 90-100: Fully addresses user request, functionally equivalent to expected output
+- 70-89: Addresses most user requirements, same overall structure with minor differences
+- 50-69: Partially addresses user request, achieves similar goal but different approach
+- 30-49: Missing significant requirements from user request
+- 0-29: Does not address user request or significantly incorrect`;
+/**
+ * Evaluates how well a generated output matches an expected output using an LLM.
+ * Returns a resemblance score (0-100), a qualitative statement, and any missing requirements.
+ *
+ * @param params Evaluation parameters including prompts, outputs, and API configuration
+ * @returns Evaluation result with score, statement, and missing requirements
+ */
+export async function evaluateWithLLM(params) {
+    const { userPrompt, generatedOutput, expectedOutput, evaluatorSystemPrompt, apiKey, model = 'anthropic/claude-sonnet-4.5' } = params;
+    const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey });
+    const userMessage = `## User's Original Request
+${userPrompt}
+
+## Expected Reference Output
+\`\`\`json
+${JSON.stringify(expectedOutput, null, 2)}
+\`\`\`
+
+## Generated Output
+\`\`\`json
+${JSON.stringify(generatedOutput, null, 2)}
+\`\`\`
+
+Please evaluate how well the generated output:
+1. Fulfills ALL requirements from the user's original request
+2. Matches the structure and logic of the expected reference output`;
+    try {
+        const response = await client.chat.completions.create({
+            model,
+            messages: [
+                { role: 'system', content: evaluatorSystemPrompt },
+                { role: 'user', content: userMessage }
+            ],
+            temperature: 0
+        });
+        const content = response.choices[0]?.message?.content;
+        if (!content) {
+            return {
+                success: false,
+                resemblanceScore: 0,
+                statement: 'No response from evaluator',
+                error: 'Empty response from LLM'
+            };
+        }
+        // Parse JSON response - handle potential markdown code blocks
+        let jsonContent = content.trim();
+        if (jsonContent.startsWith('```')) {
+            // Remove markdown code block wrapper
+            jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
+        }
+        const parsed = JSON.parse(jsonContent);
+        return {
+            success: true,
+            resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
+            statement: parsed.statement,
+            missingRequirements: parsed.missingRequirements ?? []
+        };
+    }
+    catch (err) {
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        return {
+            success: false,
+            resemblanceScore: 0,
+            statement: 'Evaluation failed',
+            error: errorMessage
+        };
+    }
+}

package/package/components/copilot/chat/__tests__/shared/baseResultsWriter.d.ts

@@ -0,0 +1,32 @@
+import type { BaseEvalResult } from './types';
+/**
+ * Generates a timestamp string suitable for filenames.
+ * Format: 2024-01-15T10-30-45-123Z (ISO but with dashes instead of colons)
+ */
+export declare function generateTimestamp(): string;
+/**
+ * Parameters for writing comparison results.
+ */
+export interface WriteResultsParams<TOutput> {
+    /** User prompt that was tested */
+    userPrompt: string;
+    /** Results from all variants */
+    results: BaseEvalResult<TOutput>[];
+    /** Directory to write results to */
+    outputDir: string;
+    /** Function to format domain-specific output for JSON files */
+    formatOutput: (output: TOutput) => unknown;
+    /** Label for the output type (e.g., 'flow', 'app') */
+    outputLabel?: string;
+}
+/**
+ * Writes comparison results to files in the results folder.
+ * Creates:
+ * - summary.md - Summary with prompt and results table
+ * - {variant_name}.json - Full result with metadata for each variant
+ * - {variant_name}_{outputLabel}.json - Clean output for each variant
+ */
+export declare function writeComparisonResults<TOutput>(params: WriteResultsParams<TOutput>): Promise<{
+    summaryPath: string;
+    outputPaths: string[];
+}>;

package/package/components/copilot/chat/__tests__/shared/baseResultsWriter.js

@@ -0,0 +1,130 @@
+// @ts-ignore
+import { writeFile, mkdir } from 'fs/promises';
+// @ts-ignore
+import { join, dirname } from 'path';
+// @ts-ignore
+import { fileURLToPath } from 'url';
+/**
+ * Generates a timestamp string suitable for filenames.
+ * Format: 2024-01-15T10-30-45-123Z (ISO but with dashes instead of colons)
+ */
+export function generateTimestamp() {
+    return new Date().toISOString().replace(/:/g, '-');
+}
+/**
+ * Writes comparison results to files in the results folder.
+ * Creates:
+ * - summary.md - Summary with prompt and results table
+ * - {variant_name}.json - Full result with metadata for each variant
+ * - {variant_name}_{outputLabel}.json - Clean output for each variant
+ */
+export async function writeComparisonResults(params) {
+    const { userPrompt, results, outputDir, formatOutput, outputLabel = 'output' } = params;
+    const timestamp = generateTimestamp();
+    // Ensure results directory exists
+    await mkdir(outputDir, { recursive: true });
+    const resultFolder = join(outputDir, timestamp);
+    await mkdir(resultFolder, { recursive: true });
+    // Check if any results have evaluation data
+    const hasEvaluation = results.some((r) => r.evaluationResult);
+    // Build summary markdown
+    const summaryLines = [
+        `# Eval Results - ${timestamp}`,
+        '',
+        '## User Prompt',
+        '```',
+        userPrompt.trim(),
+        '```',
+        '',
+        '## Results',
+        ''
+    ];
+    // Add results table header based on whether evaluation data exists
+    if (hasEvaluation) {
+        summaryLines.push('| Variant | Success | Total Tokens | Tool Calls | Iterations | Resemblance Score |');
+        summaryLines.push('|---------|---------|--------------|------------|------------|-------------------|');
+    }
+    else {
+        summaryLines.push('| Variant | Success | Total Tokens | Tool Calls | Iterations |');
+        summaryLines.push('|---------|---------|--------------|------------|------------|');
+    }
+    for (const result of results) {
+        const baseRow = `| ${result.variantName} | ${result.success} | ${result.tokenUsage.total} | ${result.toolsCalled.length} | ${result.iterations}`;
+        if (hasEvaluation) {
+            const score = result.evaluationResult?.resemblanceScore ?? 'N/A';
+            summaryLines.push(`${baseRow} | ${score} |`);
+        }
+        else {
+            summaryLines.push(`${baseRow} |`);
+        }
+    }
+    // Add evaluation details section if available
+    if (hasEvaluation) {
+        summaryLines.push('');
+        summaryLines.push('## Evaluation Details');
+        summaryLines.push('');
+        for (const result of results) {
+            if (result.evaluationResult) {
+                summaryLines.push(`### ${result.variantName}`);
+                summaryLines.push('');
+                summaryLines.push(`**Score:** ${result.evaluationResult.resemblanceScore}/100`);
+                summaryLines.push('');
+                summaryLines.push(`**Statement:** ${result.evaluationResult.statement}`);
+                summaryLines.push('');
+                if (result.evaluationResult.missingRequirements &&
+                    result.evaluationResult.missingRequirements.length > 0) {
+                    summaryLines.push('**Missing Requirements:**');
+                    for (const req of result.evaluationResult.missingRequirements) {
+                        summaryLines.push(`- ${req}`);
+                    }
+                    summaryLines.push('');
+                }
+                if (result.evaluationResult.error) {
+                    summaryLines.push(`**Error:** ${result.evaluationResult.error}`);
+                    summaryLines.push('');
+                }
+            }
+        }
+    }
+    // Add errors section for failed variants
+    const failedResults = results.filter((r) => !r.success && r.error);
+    if (failedResults.length > 0) {
+        summaryLines.push('');
+        summaryLines.push('## Errors');
+        summaryLines.push('');
+        for (const result of failedResults) {
+            summaryLines.push(`### ${result.variantName}`);
+            summaryLines.push('');
+            summaryLines.push('```');
+            summaryLines.push(result.error);
+            summaryLines.push('```');
+            summaryLines.push('');
+        }
+    }
+    const outputPaths = [];
+    for (const result of results) {
+        const resultFilename = `${result.variantName}.json`;
+        const resultPath = join(resultFolder, resultFilename);
+        outputPaths.push(resultPath);
+        const outputFilename = `${result.variantName}_${outputLabel}.json`;
+        const outputPath = join(resultFolder, outputFilename);
+        // Write result JSON file (with metadata)
+        const resultData = {
+            variantName: result.variantName,
+            success: result.success,
+            error: result.error,
+            evaluationResult: result.evaluationResult,
+            toolsCalled: result.toolsCalled,
+            toolCallDetails: result.toolCallDetails,
+            messages: result.messages
+        };
+        await writeFile(resultPath, JSON.stringify(resultData, null, 2));
+        // Write clean output JSON file (domain-specific format)
+        const outputData = formatOutput(result.output);
+        await writeFile(outputPath, JSON.stringify(outputData, null, 2));
+    }
+    // Write summary markdown file
+    const summaryPath = join(resultFolder, `summary.md`);
+    await writeFile(summaryPath, summaryLines.join('\n'));
+    return { summaryPath, outputPaths };
+}

package/package/components/copilot/chat/__tests__/shared/baseVariants.d.ts

@@ -0,0 +1,45 @@
+import type { ChatCompletionFunctionTool, ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs';
+import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs';
+import type { VariantConfig } from './types';
+/**
+ * Generic tool interface that matches the structure used across chat modules.
+ */
+export interface Tool<THelpers> {
+    def: ChatCompletionFunctionTool;
+    fn: (params: {
+        args: Record<string, unknown>;
+        workspace: string;
+        helpers: THelpers;
+        toolCallbacks: {
+            setToolStatus: (...args: unknown[]) => void;
+            removeToolStatus: (...args: unknown[]) => void;
+        };
+        toolId: string;
+    }) => Promise<string>;
+}
+/**
+ * Domain-specific defaults for variant resolution.
+ */
+export interface VariantDefaults<THelpers> {
+    /** Function to prepare system message, optionally with custom prompt */
+    prepareSystemMessage: (customPrompt?: string) => ChatCompletionSystemMessageParam;
+    /** Available tools for the domain */
+    tools: Tool<THelpers>[];
+}
+/**
+ * Resolves system prompt from variant config.
+ * Returns the appropriate ChatCompletionSystemMessageParam based on config.
+ */
+export declare function resolveSystemPrompt<THelpers>(variant: VariantConfig | undefined, defaults: VariantDefaults<THelpers>, fallbackCustomPrompt?: string): ChatCompletionSystemMessageParam;
+/**
+ * Resolves tools from variant config.
+ * Returns both the tool definitions (for API) and full tools (for execution).
+ */
+export declare function resolveTools<THelpers>(variant: VariantConfig | undefined, defaults: VariantDefaults<THelpers>): {
+    toolDefs: ChatCompletionTool[];
+    tools: Tool<THelpers>[];
+};
+/**
+ * Resolves model from variant config with fallback.
+ */
+export declare function resolveModel(variant?: VariantConfig, fallback?: string): string;

package/package/components/copilot/chat/__tests__/shared/baseVariants.js

@@ -0,0 +1,57 @@
+/**
+ * Resolves system prompt from variant config.
+ * Returns the appropriate ChatCompletionSystemMessageParam based on config.
+ */
+export function resolveSystemPrompt(variant, defaults, fallbackCustomPrompt) {
+    if (!variant?.systemPrompt || variant.systemPrompt.type === 'default') {
+        return defaults.prepareSystemMessage(fallbackCustomPrompt);
+    }
+    if (variant.systemPrompt.type === 'default-with-custom') {
+        return defaults.prepareSystemMessage(variant.systemPrompt.custom);
+    }
+    // type === 'custom'
+    return {
+        role: 'system',
+        content: variant.systemPrompt.content
+    };
+}
+/**
+ * Resolves tools from variant config.
+ * Returns both the tool definitions (for API) and full tools (for execution).
+ */
+export function resolveTools(variant, defaults) {
+    if (!variant?.tools || variant.tools.type === 'default') {
+        return {
+            toolDefs: defaults.tools.map((t) => t.def),
+            tools: defaults.tools
+        };
+    }
+    if (variant.tools.type === 'subset') {
+        const includeList = variant.tools.include;
+        const subset = defaults.tools.filter((t) => includeList.includes(t.def.function.name));
+        return {
+            toolDefs: subset.map((t) => t.def),
+            tools: subset
+        };
+    }
+    if (variant.tools.type === 'custom') {
+        // Custom tools are typed as unknown[] in base VariantConfig but domain-specific
+        // code should ensure they are the correct Tool<THelpers> type
+        const customTools = variant.tools.tools;
+        return {
+            toolDefs: customTools.map((t) => t.def),
+            tools: customTools
+        };
+    }
+    // Default fallback
+    return {
+        toolDefs: defaults.tools.map((t) => t.def),
+        tools: defaults.tools
+    };
+}
+/**
+ * Resolves model from variant config with fallback.
+ */
+export function resolveModel(variant, fallback) {
+    return variant?.model ?? fallback ?? 'gpt-4o';
+}

package/package/components/copilot/chat/__tests__/shared/index.d.ts

@@ -0,0 +1,10 @@
+export type { TokenUsage, ToolCallDetail, EvaluationResult, BaseEvalResult, VariantConfig, EvalRunnerOptions, ToolCallbacks } from './types';
+export { createNoOpToolCallbacks } from './types';
+export type { Tool, VariantDefaults } from './baseVariants';
+export { resolveSystemPrompt, resolveTools, resolveModel } from './baseVariants';
+export type { RawEvalResult, RunEvalParams } from './baseEvalRunner';
+export { runEval } from './baseEvalRunner';
+export type { EvaluateParams } from './baseLLMEvaluator';
+export { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from './baseLLMEvaluator';
+export type { WriteResultsParams } from './baseResultsWriter';
+export { writeComparisonResults, generateTimestamp } from './baseResultsWriter';

package/package/components/copilot/chat/__tests__/shared/index.js

@@ -0,0 +1,5 @@
+export { createNoOpToolCallbacks } from './types';
+export { resolveSystemPrompt, resolveTools, resolveModel } from './baseVariants';
+export { runEval } from './baseEvalRunner';
+export { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from './baseLLMEvaluator';
+export { writeComparisonResults, generateTimestamp } from './baseResultsWriter';

package/package/components/copilot/chat/__tests__/shared/types.d.ts

@@ -0,0 +1,102 @@
+import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs';
+/**
+ * Token usage tracking for LLM calls.
+ */
+export interface TokenUsage {
+    prompt: number;
+    completion: number;
+    total: number;
+}
+/**
+ * Details of a single tool call made during evaluation.
+ */
+export interface ToolCallDetail {
+    name: string;
+    arguments: Record<string, unknown>;
+}
+/**
+ * Result of LLM-based comparison/evaluation.
+ */
+export interface EvaluationResult {
+    success: boolean;
+    resemblanceScore: number;
+    statement: string;
+    missingRequirements?: string[];
+    error?: string;
+}
+/**
+ * Base evaluation result that can be extended for domain-specific outputs.
+ * @template TOutput The domain-specific output type (e.g., flow definition, app files)
+ */
+export interface BaseEvalResult<TOutput> {
+    success: boolean;
+    output: TOutput;
+    error?: string;
+    tokenUsage: TokenUsage;
+    toolCallsCount: number;
+    toolsCalled: string[];
+    toolCallDetails: ToolCallDetail[];
+    iterations: number;
+    variantName: string;
+    evaluationResult?: EvaluationResult;
+    messages: ChatCompletionMessageParam[];
+}
+/**
+ * Base configuration for a variant in eval testing.
+ * Allows customizing system prompt, tools, and model for comparison.
+ *
+ * Note: Domain-specific variants may extend this with custom tool configurations.
+ * See flow/flowEvalVariants.ts for an example with custom tools.
+ */
+export interface VariantConfig {
+    name: string;
+    description?: string;
+    /** System prompt configuration */
+    systemPrompt?: {
+        type: 'default';
+    } | {
+        type: 'default-with-custom';
+        custom: string;
+    } | {
+        type: 'custom';
+        content: string;
+    };
+    /** Tools configuration - basic types supported by shared code */
+    tools?: {
+        type: 'default';
+    } | {
+        type: 'subset';
+        include: string[];
+    } | {
+        type: 'custom';
+        tools: unknown[];
+    };
+    /** Model to use (default: 'gpt-4o') */
+    model?: string;
+}
+/**
+ * Options for running an evaluation.
+ */
+export interface EvalRunnerOptions {
+    /** Maximum iterations for tool call loop (default: 20) */
+    maxIterations?: number;
+    /** Model to use for LLM calls */
+    model?: string;
+    /** Workspace ID for tool calls */
+    workspace?: string;
+}
+/**
+ * No-op tool callbacks for eval testing.
+ */
+export interface ToolCallbacks {
+    setToolStatus: (id: string, status: {
+        content?: string;
+        result?: string;
+        error?: string;
+    }) => void;
+    removeToolStatus: (id: string) => void;
+}
+/**
+ * Creates no-op tool callbacks for eval testing.
+ */
+export declare function createNoOpToolCallbacks(): ToolCallbacks;

package/package/components/copilot/chat/__tests__/shared/types.js

@@ -0,0 +1,9 @@
+/**
+ * Creates no-op tool callbacks for eval testing.
+ */
+export function createNoOpToolCallbacks() {
+    return {
+        setToolStatus: () => { },
+        removeToolStatus: () => { }
+    };
+}

package/package/components/copilot/chat/anthropic.d.ts

@@ -7,7 +7,7 @@ export declare function getAnthropicCompletion(messages: ChatCompletionMessagePa
 export declare function parseAnthropicCompletion(completion: MessageStream, callbacks: ToolCallbacks & {
     onNewToken: (token: string) => void;
     onMessageEnd: () => void;
-}, messages: ChatCompletionMessageParam[], addedMessages: ChatCompletionMessageParam[], tools: Tool<any>[], helpers: any): Promise<boolean>;
+}, messages: ChatCompletionMessageParam[], addedMessages: ChatCompletionMessageParam[], tools: Tool<any>[], helpers: any, abortController?: AbortController): Promise<boolean>;
 export declare function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessageParam[]): {
     system: TextBlockParam[] | undefined;
     messages: MessageParam[];

package/package/components/copilot/chat/anthropic.js

@@ -24,9 +24,57 @@ export async function getAnthropicCompletion(messages, abortController, tools) {
     });
     return stream;
 }
-export async function parseAnthropicCompletion(completion, callbacks, messages, addedMessages, tools, helpers) {
+export async function parseAnthropicCompletion(completion, callbacks, messages, addedMessages, tools, helpers, abortController) {
     let toolCallsToProcess = [];
     let error = null;
+    let currentStreamingTool = undefined;
+    let accumulatedJson = '';
+    completion.on('streamEvent', (event) => {
+        if (event.type === 'content_block_start') {
+            const block = event.content_block;
+            if (block.type === 'tool_use') {
+                const toolName = block.name;
+                const toolId = block.id;
+                const tool = tools.find((t) => t.def.function.name === toolName);
+                const shouldStream = tool?.streamArguments ?? false;
+                callbacks.onMessageEnd();
+                // Reset accumulated JSON for new tool
+                accumulatedJson = '';
+                currentStreamingTool = { tempId: toolId, shouldStream, toolName };
+                callbacks.setToolStatus(toolId, {
+                    isLoading: true,
+                    content: `Calling ${toolName}...`,
+                    toolName,
+                    isStreamingArguments: shouldStream,
+                    showFade: tool?.showFade,
+                    showDetails: tool?.showDetails
+                });
+            }
+        }
+    });
+    completion.on('inputJson', (partialJson) => {
+        if (currentStreamingTool?.shouldStream && currentStreamingTool.tempId) {
+            // Accumulate the partial JSON
+            accumulatedJson += partialJson;
+            // Try to parse and display
+            try {
+                const parsed = JSON.parse(accumulatedJson);
+                callbacks.setToolStatus(currentStreamingTool.tempId, {
+                    parameters: parsed,
+                    isStreamingArguments: true,
+                    isLoading: true
+                });
+            }
+            catch {
+                // JSON incomplete, display as raw string
+                callbacks.setToolStatus(currentStreamingTool.tempId, {
+                    parameters: accumulatedJson,
+                    isStreamingArguments: true,
+                    isLoading: true
+                });
+            }
+        }
+    });
     // Handle text streaming
     completion.on('text', (textDelta, _textSnapshot) => {
         callbacks.onNewToken(textDelta);
@@ -57,10 +105,35 @@ export async function parseAnthropicCompletion(completion, callbacks, messages,
                 }
             }
         }
+        // Clear temp tracking after processing
+        currentStreamingTool = undefined;
+    });
+    // Handle abort
+    completion.on('abort', (e) => {
+        // Check the AbortController's signal for the reason
+        const abortReason = abortController?.signal.reason;
+        console.warn('Anthropic stream aborted:', {
+            name: e?.name,
+            message: e?.message,
+            abortReason,
+            wasAbortedByUser: abortReason === 'user_cancelled',
+            signalAborted: abortController?.signal.aborted,
+            cause: e?.cause,
+            stack: e?.stack
+        });
+        error = e;
     });
     // Handle errors
     completion.on('error', (e) => {
-        console.error('Anthropic stream error:', e);
+        console.error('Anthropic stream error:', {
+            name: e?.name,
+            message: e?.message,
+            status: e?.status,
+            headers: e?.headers,
+            error: e?.error,
+            cause: e?.cause,
+            stack: e?.stack
+        });
         error = e;
     });
     // Wait for completion

package/package/components/copilot/chat/app/core.d.ts

@@ -0,0 +1,69 @@
+import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/chat/completions.mjs';
+import { type Tool } from '../shared';
+export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'inline';
+export interface InlineScript {
+    language: 'bun' | 'python3';
+    content: string;
+}
+export interface BackendRunnable {
+    name: string;
+    type: BackendRunnableType;
+    staticInputs?: Record<string, any>;
+    inlineScript?: InlineScript;
+    path?: string;
+}
+export interface AppFiles {
+    frontend: Record<string, string>;
+    backend: Record<string, BackendRunnable>;
+}
+export interface LintMessages {
+    /** Error/warning messages from frontend files */
+    frontend: Record<string, string[]>;
+    /** Error/warning messages from backend runnables */
+    backend: Record<string, string[]>;
+}
+export interface LintResult {
+    /** Total count of errors across all files */
+    errorCount: number;
+    /** Total count of warnings across all files */
+    warningCount: number;
+    /** Error messages grouped by source type */
+    errors: LintMessages;
+    /** Warning messages grouped by source type */
+    warnings: LintMessages;
+}
+/** Context about the currently selected file or runnable in the app editor */
+export interface SelectedContext {
+    /** Type of selection: 'frontend' for frontend files, 'backend' for backend runnables, or 'none' if nothing is selected */
+    type: 'frontend' | 'backend' | 'none';
+    /** The path of the selected frontend file (when type is 'frontend') */
+    frontendPath?: string;
+    /** The key of the selected backend runnable (when type is 'backend') */
+    backendKey?: string;
+}
+export interface AppAIChatHelpers {
+    listFrontendFiles: () => string[];
+    getFrontendFile: (path: string) => string | undefined;
+    getFrontendFiles: () => Record<string, string>;
+    /** Sets a frontend file and returns lint results */
+    setFrontendFile: (path: string, content: string) => LintResult;
+    deleteFrontendFile: (path: string) => void;
+    listBackendRunnables: () => {
+        key: string;
+        name: string;
+    }[];
+    getBackendRunnable: (key: string) => BackendRunnable | undefined;
+    getBackendRunnables: () => Record<string, BackendRunnable>;
+    /** Sets a backend runnable, switches UI to it, waits for Monaco to analyze, and returns lint results */
+    setBackendRunnable: (key: string, runnable: BackendRunnable) => Promise<LintResult>;
+    deleteBackendRunnable: (key: string) => void;
+    getFiles: () => AppFiles;
+    getSelectedContext: () => SelectedContext;
+    snapshot: () => number;
+    revertToSnapshot: (id: number) => void;
+    /** Lint all frontend files and backend runnables, returns errors and warnings */
+    lint: () => LintResult;
+}
+export declare const getAppTools: () => Tool<AppAIChatHelpers>[];
+export declare function prepareAppSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam;
+export declare function prepareAppUserMessage(instructions: string, selectedContext?: SelectedContext): ChatCompletionUserMessageParam;