windmill-components 1.677.1 → 1.695.0

package/package/components/copilot/chat/__tests__/shared/baseResultsWriter.d.ts

@@ -1,32 +0,0 @@
-import type { BaseEvalResult } from './types';
-/**
- * Generates a timestamp string suitable for filenames.
- * Format: 2024-01-15T10-30-45-123Z (ISO but with dashes instead of colons)
- */
-export declare function generateTimestamp(): string;
-/**
- * Parameters for writing comparison results.
- */
-export interface WriteResultsParams<TOutput> {
-    /** User prompt that was tested */
-    userPrompt: string;
-    /** Results from all variants */
-    results: BaseEvalResult<TOutput>[];
-    /** Directory to write results to */
-    outputDir: string;
-    /** Function to format domain-specific output for JSON files */
-    formatOutput: (output: TOutput) => unknown;
-    /** Label for the output type (e.g., 'flow', 'app') */
-    outputLabel?: string;
-}
-/**
- * Writes comparison results to files in the results folder.
- * Creates:
- * - summary.md - Summary with prompt and results table
- * - {variant_name}.json - Full result with metadata for each variant
- * - {variant_name}_{outputLabel}.json - Clean output for each variant
- */
-export declare function writeComparisonResults<TOutput>(params: WriteResultsParams<TOutput>): Promise<{
-    summaryPath: string;
-    outputPaths: string[];
-}>;

package/package/components/copilot/chat/__tests__/shared/baseResultsWriter.js

@@ -1,130 +0,0 @@
-// @ts-ignore
-import { writeFile, mkdir } from 'fs/promises';
-// @ts-ignore
-import { join, dirname } from 'path';
-// @ts-ignore
-import { fileURLToPath } from 'url';
-/**
- * Generates a timestamp string suitable for filenames.
- * Format: 2024-01-15T10-30-45-123Z (ISO but with dashes instead of colons)
- */
-export function generateTimestamp() {
-    return new Date().toISOString().replace(/:/g, '-');
-}
-/**
- * Writes comparison results to files in the results folder.
- * Creates:
- * - summary.md - Summary with prompt and results table
- * - {variant_name}.json - Full result with metadata for each variant
- * - {variant_name}_{outputLabel}.json - Clean output for each variant
- */
-export async function writeComparisonResults(params) {
-    const { userPrompt, results, outputDir, formatOutput, outputLabel = 'output' } = params;
-    const timestamp = generateTimestamp();
-    // Ensure results directory exists
-    await mkdir(outputDir, { recursive: true });
-    const resultFolder = join(outputDir, timestamp);
-    await mkdir(resultFolder, { recursive: true });
-    // Check if any results have evaluation data
-    const hasEvaluation = results.some((r) => r.evaluationResult);
-    // Build summary markdown
-    const summaryLines = [
-        `# Eval Results - ${timestamp}`,
-        '',
-        '## User Prompt',
-        '```',
-        userPrompt.trim(),
-        '```',
-        '',
-        '## Results',
-        ''
-    ];
-    // Add results table header based on whether evaluation data exists
-    if (hasEvaluation) {
-        summaryLines.push('| Variant | Success | Total Tokens | Tool Calls | Iterations | Resemblance Score |');
-        summaryLines.push('|---------|---------|--------------|------------|------------|-------------------|');
-    }
-    else {
-        summaryLines.push('| Variant | Success | Total Tokens | Tool Calls | Iterations |');
-        summaryLines.push('|---------|---------|--------------|------------|------------|');
-    }
-    for (const result of results) {
-        const baseRow = `| ${result.variantName} | ${result.success} | ${result.tokenUsage.total} | ${result.toolsCalled.length} | ${result.iterations}`;
-        if (hasEvaluation) {
-            const score = result.evaluationResult?.resemblanceScore ?? 'N/A';
-            summaryLines.push(`${baseRow} | ${score} |`);
-        }
-        else {
-            summaryLines.push(`${baseRow} |`);
-        }
-    }
-    // Add evaluation details section if available
-    if (hasEvaluation) {
-        summaryLines.push('');
-        summaryLines.push('## Evaluation Details');
-        summaryLines.push('');
-        for (const result of results) {
-            if (result.evaluationResult) {
-                summaryLines.push(`### ${result.variantName}`);
-                summaryLines.push('');
-                summaryLines.push(`**Score:** ${result.evaluationResult.resemblanceScore}/100`);
-                summaryLines.push('');
-                summaryLines.push(`**Statement:** ${result.evaluationResult.statement}`);
-                summaryLines.push('');
-                if (result.evaluationResult.missingRequirements &&
-                    result.evaluationResult.missingRequirements.length > 0) {
-                    summaryLines.push('**Missing Requirements:**');
-                    for (const req of result.evaluationResult.missingRequirements) {
-                        summaryLines.push(`- ${req}`);
-                    }
-                    summaryLines.push('');
-                }
-                if (result.evaluationResult.error) {
-                    summaryLines.push(`**Error:** ${result.evaluationResult.error}`);
-                    summaryLines.push('');
-                }
-            }
-        }
-    }
-    // Add errors section for failed variants
-    const failedResults = results.filter((r) => !r.success && r.error);
-    if (failedResults.length > 0) {
-        summaryLines.push('');
-        summaryLines.push('## Errors');
-        summaryLines.push('');
-        for (const result of failedResults) {
-            summaryLines.push(`### ${result.variantName}`);
-            summaryLines.push('');
-            summaryLines.push('```');
-            summaryLines.push(result.error);
-            summaryLines.push('```');
-            summaryLines.push('');
-        }
-    }
-    const outputPaths = [];
-    for (const result of results) {
-        const resultFilename = `${result.variantName}.json`;
-        const resultPath = join(resultFolder, resultFilename);
-        outputPaths.push(resultPath);
-        const outputFilename = `${result.variantName}_${outputLabel}.json`;
-        const outputPath = join(resultFolder, outputFilename);
-        // Write result JSON file (with metadata)
-        const resultData = {
-            variantName: result.variantName,
-            success: result.success,
-            error: result.error,
-            evaluationResult: result.evaluationResult,
-            toolsCalled: result.toolsCalled,
-            toolCallDetails: result.toolCallDetails,
-            messages: result.messages
-        };
-        await writeFile(resultPath, JSON.stringify(resultData, null, 2));
-        // Write clean output JSON file (domain-specific format)
-        const outputData = formatOutput(result.output);
-        await writeFile(outputPath, JSON.stringify(outputData, null, 2));
-    }
-    // Write summary markdown file
-    const summaryPath = join(resultFolder, `summary.md`);
-    await writeFile(summaryPath, summaryLines.join('\n'));
-    return { summaryPath, outputPaths };
-}

package/package/components/copilot/chat/__tests__/shared/baseVariants.d.ts

@@ -1,45 +0,0 @@
-import type { ChatCompletionFunctionTool, ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs';
-import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs';
-import type { VariantConfig } from './types';
-/**
- * Generic tool interface that matches the structure used across chat modules.
- */
-export interface Tool<THelpers> {
-    def: ChatCompletionFunctionTool;
-    fn: (params: {
-        args: Record<string, unknown>;
-        workspace: string;
-        helpers: THelpers;
-        toolCallbacks: {
-            setToolStatus: (...args: unknown[]) => void;
-            removeToolStatus: (...args: unknown[]) => void;
-        };
-        toolId: string;
-    }) => Promise<string>;
-}
-/**
- * Domain-specific defaults for variant resolution.
- */
-export interface VariantDefaults<THelpers> {
-    /** Function to prepare system message, optionally with custom prompt */
-    prepareSystemMessage: (customPrompt?: string) => ChatCompletionSystemMessageParam;
-    /** Available tools for the domain */
-    tools: Tool<THelpers>[];
-}
-/**
- * Resolves system prompt from variant config.
- * Returns the appropriate ChatCompletionSystemMessageParam based on config.
- */
-export declare function resolveSystemPrompt<THelpers>(variant: VariantConfig | undefined, defaults: VariantDefaults<THelpers>, fallbackCustomPrompt?: string): ChatCompletionSystemMessageParam;
-/**
- * Resolves tools from variant config.
- * Returns both the tool definitions (for API) and full tools (for execution).
- */
-export declare function resolveTools<THelpers>(variant: VariantConfig | undefined, defaults: VariantDefaults<THelpers>): {
-    toolDefs: ChatCompletionTool[];
-    tools: Tool<THelpers>[];
-};
-/**
- * Resolves model from variant config with fallback.
- */
-export declare function resolveModel(variant?: VariantConfig, fallback?: string): string;

package/package/components/copilot/chat/__tests__/shared/baseVariants.js

@@ -1,57 +0,0 @@
-/**
- * Resolves system prompt from variant config.
- * Returns the appropriate ChatCompletionSystemMessageParam based on config.
- */
-export function resolveSystemPrompt(variant, defaults, fallbackCustomPrompt) {
-    if (!variant?.systemPrompt || variant.systemPrompt.type === 'default') {
-        return defaults.prepareSystemMessage(fallbackCustomPrompt);
-    }
-    if (variant.systemPrompt.type === 'default-with-custom') {
-        return defaults.prepareSystemMessage(variant.systemPrompt.custom);
-    }
-    // type === 'custom'
-    return {
-        role: 'system',
-        content: variant.systemPrompt.content
-    };
-}
-/**
- * Resolves tools from variant config.
- * Returns both the tool definitions (for API) and full tools (for execution).
- */
-export function resolveTools(variant, defaults) {
-    if (!variant?.tools || variant.tools.type === 'default') {
-        return {
-            toolDefs: defaults.tools.map((t) => t.def),
-            tools: defaults.tools
-        };
-    }
-    if (variant.tools.type === 'subset') {
-        const includeList = variant.tools.include;
-        const subset = defaults.tools.filter((t) => includeList.includes(t.def.function.name));
-        return {
-            toolDefs: subset.map((t) => t.def),
-            tools: subset
-        };
-    }
-    if (variant.tools.type === 'custom') {
-        // Custom tools are typed as unknown[] in base VariantConfig but domain-specific
-        // code should ensure they are the correct Tool<THelpers> type
-        const customTools = variant.tools.tools;
-        return {
-            toolDefs: customTools.map((t) => t.def),
-            tools: customTools
-        };
-    }
-    // Default fallback
-    return {
-        toolDefs: defaults.tools.map((t) => t.def),
-        tools: defaults.tools
-    };
-}
-/**
- * Resolves model from variant config with fallback.
- */
-export function resolveModel(variant, fallback) {
-    return variant?.model ?? fallback ?? 'gpt-4o';
-}

package/package/components/copilot/chat/__tests__/shared/index.d.ts

@@ -1,10 +0,0 @@
-export type { TokenUsage, ToolCallDetail, EvaluationResult, BaseEvalResult, VariantConfig, EvalRunnerOptions, ToolCallbacks } from './types';
-export { createNoOpToolCallbacks } from './types';
-export type { Tool, VariantDefaults } from './baseVariants';
-export { resolveSystemPrompt, resolveTools, resolveModel } from './baseVariants';
-export type { RawEvalResult, RunEvalParams } from './baseEvalRunner';
-export { runEval } from './baseEvalRunner';
-export type { EvaluateParams } from './baseLLMEvaluator';
-export { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from './baseLLMEvaluator';
-export type { WriteResultsParams } from './baseResultsWriter';
-export { writeComparisonResults, generateTimestamp } from './baseResultsWriter';

package/package/components/copilot/chat/__tests__/shared/index.js

@@ -1,5 +0,0 @@
-export { createNoOpToolCallbacks } from './types';
-export { resolveSystemPrompt, resolveTools, resolveModel } from './baseVariants';
-export { runEval } from './baseEvalRunner';
-export { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from './baseLLMEvaluator';
-export { writeComparisonResults, generateTimestamp } from './baseResultsWriter';

package/package/components/copilot/chat/__tests__/shared/types.d.ts

@@ -1,105 +0,0 @@
-import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs';
-import type { AIProvider } from '../../../../../gen/types.gen';
-/**
- * Token usage tracking for LLM calls.
- */
-export interface TokenUsage {
-    prompt: number;
-    completion: number;
-    total: number;
-}
-/**
- * Details of a single tool call made during evaluation.
- */
-export interface ToolCallDetail {
-    name: string;
-    arguments: Record<string, unknown>;
-}
-/**
- * Result of LLM-based comparison/evaluation.
- */
-export interface EvaluationResult {
-    success: boolean;
-    resemblanceScore: number;
-    statement: string;
-    missingRequirements?: string[];
-    error?: string;
-}
-/**
- * Base evaluation result that can be extended for domain-specific outputs.
- * @template TOutput The domain-specific output type (e.g., flow definition, app files)
- */
-export interface BaseEvalResult<TOutput> {
-    success: boolean;
-    output: TOutput;
-    error?: string;
-    tokenUsage: TokenUsage;
-    toolCallsCount: number;
-    toolsCalled: string[];
-    toolCallDetails: ToolCallDetail[];
-    iterations: number;
-    variantName: string;
-    evaluationResult?: EvaluationResult;
-    messages: ChatCompletionMessageParam[];
-}
-/**
- * Base configuration for a variant in eval testing.
- * Allows customizing system prompt, tools, and model for comparison.
- *
- * Note: Domain-specific variants may extend this with custom tool configurations.
- * See flow/flowEvalVariants.ts for an example with custom tools.
- */
-export interface VariantConfig {
-    name: string;
-    description?: string;
-    /** System prompt configuration */
-    systemPrompt?: {
-        type: 'default';
-    } | {
-        type: 'default-with-custom';
-        custom: string;
-    } | {
-        type: 'custom';
-        content: string;
-    };
-    /** Tools configuration - basic types supported by shared code */
-    tools?: {
-        type: 'default';
-    } | {
-        type: 'subset';
-        include: string[];
-    } | {
-        type: 'custom';
-        tools: unknown[];
-    };
-    /** Model to use (default: 'gpt-4o') */
-    model?: string;
-}
-/**
- * Options for running an evaluation.
- */
-export interface EvalRunnerOptions {
-    /** Maximum iterations for tool call loop (default: 20) */
-    maxIterations?: number;
-    /** Model to use for LLM calls */
-    model?: string;
-    /** Workspace ID for tool calls */
-    workspace?: string;
-    /** AI provider (inferred from model name if omitted) */
-    provider?: AIProvider;
-}
-/**
- * No-op tool callbacks for eval testing.
- */
-export interface ToolCallbacks {
-    setToolStatus: (id: string, status: {
-        content?: string;
-        result?: string;
-        error?: string;
-    }) => void;
-    removeToolStatus: (id: string) => void;
-}
-/**
- * Creates no-op tool callbacks for eval testing.
- */
-export declare function createNoOpToolCallbacks(): ToolCallbacks;

package/package/components/copilot/chat/__tests__/shared/types.js

@@ -1,9 +0,0 @@
-/**
- * Creates no-op tool callbacks for eval testing.
- */
-export function createNoOpToolCallbacks() {
-    return {
-        setToolStatus: () => { },
-        removeToolStatus: () => { }
-    };
-}