windmill-components 1.677.1 → 1.687.0

package/package/components/copilot/chat/__tests__/flow/flowEvalRunner.js

@@ -1,102 +0,0 @@
-import { flowTools, prepareFlowSystemMessage, prepareFlowUserMessage } from '../../flow/core';
-import { createFlowEvalHelpers } from './flowEvalHelpers';
-import { evaluateFlowComparison } from './flowEvalComparison';
-import { runEval, resolveSystemPrompt, resolveTools, resolveModel, writeComparisonResults } from '../shared';
-/**
- * Flow-specific variant defaults.
- */
-const flowDefaults = {
-    prepareSystemMessage: prepareFlowSystemMessage,
-    tools: flowTools
-};
-/**
- * Runs a flow chat evaluation using the shared chat loop (same code path as production).
- */
-export async function runFlowEval(userPrompt, apiKey, options) {
-    const { helpers, getFlow } = createFlowEvalHelpers(options?.initialModules ?? [], options?.initialSchema);
-    // Resolve variant configuration
-    const variantName = options?.variant?.name ?? 'baseline';
-    const systemMessage = resolveSystemPrompt(options?.variant, flowDefaults, options?.customSystemPrompt);
-    const { tools } = resolveTools(options?.variant, flowDefaults);
-    const model = resolveModel(options?.variant, options?.model);
-    // Build user message
-    const userMessage = prepareFlowUserMessage(userPrompt, helpers.getFlowAndSelectedId(), []);
-    // Run the base evaluation
-    const rawResult = await runEval({
-        userPrompt,
-        systemMessage,
-        userMessage,
-        tools,
-        helpers,
-        apiKey,
-        getOutput: getFlow,
-        options: {
-            maxIterations: options?.maxIterations,
-            model,
-            workspace: 'test-workspace',
-            provider: options?.provider
-        }
-    });
-    // Run evaluation if expected flow is provided
-    let evaluationResult;
-    if (options?.expectedFlow) {
-        const generatedFlow = getFlow();
-        evaluationResult = await evaluateFlowComparison({
-            summary: generatedFlow.summary,
-            value: { modules: generatedFlow.value.modules },
-            schema: generatedFlow.schema
-        }, options.expectedFlow, userPrompt);
-    }
-    return {
-        ...rawResult,
-        variantName,
-        flow: rawResult.output,
-        evaluationResult
-    };
-}
-/**
- * Runs the same prompt against multiple variants sequentially for comparison.
- * Accepts optional per-variant provider/apiKey overrides.
- */
-export async function runVariantComparison(userPrompt, variants, defaultApiKey, baseOptions, providerOverrides) {
-    const results = await Promise.all(variants.map(async (variant, i) => {
-        const override = providerOverrides?.[i];
-        return await runFlowEval(userPrompt, override?.apiKey ?? defaultApiKey, {
-            ...baseOptions,
-            variant,
-            provider: override?.provider ?? baseOptions?.provider
-        });
-    }));
-    return results;
-}
-/**
- * Writes flow comparison results to files.
- */
-export async function writeFlowComparisonResults(userPrompt, results, outputDir) {
-    // @ts-ignore
-    const { dirname, join } = await import('path');
-    // @ts-ignore
-    const { fileURLToPath } = await import('url');
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
-    const resultsDir = outputDir ?? join(__dirname, 'results');
-    const result = await writeComparisonResults({
-        userPrompt,
-        results,
-        outputDir: resultsDir,
-        formatOutput: (flow) => ({
-            summary: flow.summary ?? '',
-            value: {
-                modules: flow.value.modules
-            },
-            schema: flow.schema ?? {
-                $schema: 'https://json-schema.org/draft/2020-12/schema',
-                properties: {},
-                required: [],
-                type: 'object'
-            }
-        }),
-        outputLabel: 'flow'
-    });
-    return { summaryPath: result.summaryPath, flowPaths: result.outputPaths };
-}

package/package/components/copilot/chat/__tests__/flow/initial/test5_initial.json

@@ -1,53 +0,0 @@
-{
-    "summary": "Simple data pipeline",
-    "value": {
-        "modules": [
-            {
-                "id": "fetch_data",
-                "summary": "Fetch data from API",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main() {\n  // Mock API call\n  return [\n    { id: 1, name: \"Item 1\", value: 100 },\n    { id: 2, name: \"Item 2\", value: 200 },\n    { id: 3, name: \"Item 3\", value: 300 }\n  ];\n}",
-                    "input_transforms": {}
-                }
-            },
-            {
-                "id": "process_data",
-                "summary": "Process the fetched data",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(data: any[]) {\n  // Apply some transformation\n  return data.map(item => ({\n    ...item,\n    value: item.value * 1.1,\n    processed: true\n  }));\n}",
-                    "input_transforms": {
-                        "data": {
-                            "type": "javascript",
-                            "expr": "results.fetch_data"
-                        }
-                    }
-                }
-            },
-            {
-                "id": "save_results",
-                "summary": "Save results to database",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(data: any[]) {\n  // Mock database save\n  console.log(`Saving ${data.length} items to database`);\n  return { saved: data.length, status: \"success\" };\n}",
-                    "input_transforms": {
-                        "data": {
-                            "type": "javascript",
-                            "expr": "results.process_data"
-                        }
-                    }
-                }
-            }
-        ]
-    },
-    "schema": {
-        "$schema": "https://json-schema.org/draft/2020-12/schema",
-        "properties": {},
-        "required": [],
-        "type": "object"
-    }
-}

package/package/components/copilot/chat/__tests__/flow/initial/test6_initial.json

@@ -1,68 +0,0 @@
-{
-    "summary": "Order processing flow",
-    "value": {
-        "modules": [
-            {
-                "id": "get_orders",
-                "summary": "Fetch list of orders",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main() {\n  // Mock orders from database\n  return [\n    { id: \"ORD-001\", type: \"express\", items: 3, total: 150 },\n    { id: \"ORD-002\", type: \"standard\", items: 5, total: 280 },\n    { id: \"ORD-003\", type: \"pickup\", items: 2, total: 75 },\n    { id: \"ORD-004\", type: \"express\", items: 1, total: 50 }\n  ];\n}",
-                    "input_transforms": {}
-                }
-            },
-            {
-                "id": "loop_orders",
-                "summary": "Process each order",
-                "value": {
-                    "type": "forloopflow",
-                    "iterator": {
-                        "type": "javascript",
-                        "expr": "results.get_orders"
-                    },
-                    "skip_failures": false,
-                    "parallel": false,
-                    "modules": [
-                        {
-                            "id": "process_order",
-                            "summary": "Process individual order",
-                            "value": {
-                                "type": "rawscript",
-                                "language": "bun",
-                                "content": "export async function main(order: any) {\n  console.log(`Processing order ${order.id}`);\n  return {\n    orderId: order.id,\n    processed: true,\n    timestamp: new Date().toISOString()\n  };\n}",
-                                "input_transforms": {
-                                    "order": {
-                                        "type": "javascript",
-                                        "expr": "flow_input.iter.value"
-                                    }
-                                }
-                            }
-                        }
-                    ]
-                }
-            },
-            {
-                "id": "summarize",
-                "summary": "Return processing summary",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(results: any[]) {\n  return {\n    totalProcessed: results.length,\n    processedAt: new Date().toISOString()\n  };\n}",
-                    "input_transforms": {
-                        "results": {
-                            "type": "javascript",
-                            "expr": "results.loop_orders"
-                        }
-                    }
-                }
-            }
-        ]
-    },
-    "schema": {
-        "$schema": "https://json-schema.org/draft/2020-12/schema",
-        "properties": {},
-        "required": [],
-        "type": "object"
-    }
-}

package/package/components/copilot/chat/__tests__/flow/initial/test7_initial.json

@@ -1,120 +0,0 @@
-{
-    "summary": "Data enrichment flow",
-    "value": {
-        "modules": [
-            {
-                "id": "get_item",
-                "summary": "Get item from input",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(item_id: string) {\n  // Mock item lookup\n  return {\n    id: item_id,\n    name: \"Product \" + item_id,\n    sku: \"SKU-\" + item_id\n  };\n}",
-                    "input_transforms": {
-                        "item_id": {
-                            "type": "javascript",
-                            "expr": "flow_input.item_id"
-                        }
-                    }
-                }
-            },
-            {
-                "id": "enrich_price",
-                "summary": "Call pricing API",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(item: any) {\n  // Mock pricing API call\n  return {\n    itemId: item.id,\n    price: 99.99,\n    currency: \"USD\",\n    discount: 10\n  };\n}",
-                    "input_transforms": {
-                        "item": {
-                            "type": "javascript",
-                            "expr": "results.get_item"
-                        }
-                    }
-                }
-            },
-            {
-                "id": "enrich_inventory",
-                "summary": "Call inventory API",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(item: any) {\n  // Mock inventory API call\n  return {\n    itemId: item.id,\n    inStock: true,\n    quantity: 150,\n    warehouse: \"WH-001\"\n  };\n}",
-                    "input_transforms": {
-                        "item": {
-                            "type": "javascript",
-                            "expr": "results.get_item"
-                        }
-                    }
-                }
-            },
-            {
-                "id": "enrich_reviews",
-                "summary": "Call reviews API",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(item: any) {\n  // Mock reviews API call\n  return {\n    itemId: item.id,\n    averageRating: 4.5,\n    reviewCount: 127,\n    topReview: \"Great product!\"\n  };\n}",
-                    "input_transforms": {
-                        "item": {
-                            "type": "javascript",
-                            "expr": "results.get_item"
-                        }
-                    }
-                }
-            },
-            {
-                "id": "combine_data",
-                "summary": "Combine all enrichment data",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(item: any, price: any, inventory: any, reviews: any) {\n  return {\n    ...item,\n    pricing: price,\n    inventory: inventory,\n    reviews: reviews\n  };\n}",
-                    "input_transforms": {
-                        "item": {
-                            "type": "javascript",
-                            "expr": "results.get_item"
-                        },
-                        "price": {
-                            "type": "javascript",
-                            "expr": "results.enrich_price"
-                        },
-                        "inventory": {
-                            "type": "javascript",
-                            "expr": "results.enrich_inventory"
-                        },
-                        "reviews": {
-                            "type": "javascript",
-                            "expr": "results.enrich_reviews"
-                        }
-                    }
-                }
-            },
-            {
-                "id": "return_result",
-                "summary": "Return final result",
-                "value": {
-                    "type": "rawscript",
-                    "language": "bun",
-                    "content": "export async function main(enriched_item: any) {\n  return {\n    success: true,\n    data: enriched_item,\n    enrichedAt: new Date().toISOString()\n  };\n}",
-                    "input_transforms": {
-                        "enriched_item": {
-                            "type": "javascript",
-                            "expr": "results.combine_data"
-                        }
-                    }
-                }
-            }
-        ]
-    },
-    "schema": {
-        "$schema": "https://json-schema.org/draft/2020-12/schema",
-        "properties": {
-            "item_id": {
-                "type": "string",
-                "description": "The ID of the item to enrich"
-            }
-        },
-        "required": ["item_id"],
-        "type": "object"
-    }
-}

package/package/components/copilot/chat/__tests__/flow/variants/baseline.d.ts

@@ -1,6 +0,0 @@
-import type { VariantConfig } from '../../shared';
-/**
- * Baseline variant - uses the production system prompt and all tools.
- * This is the default configuration that matches the actual flow chat implementation.
- */
-export declare const BASELINE_VARIANT: VariantConfig;

package/package/components/copilot/chat/__tests__/flow/variants/baseline.js

@@ -1,10 +0,0 @@
-/**
- * Baseline variant - uses the production system prompt and all tools.
- * This is the default configuration that matches the actual flow chat implementation.
- */
-export const BASELINE_VARIANT = {
-    name: 'baseline',
-    description: 'Production configuration with default system prompt and all tools',
-    systemPrompt: { type: 'default' },
-    tools: { type: 'default' }
-};

package/package/components/copilot/chat/__tests__/flow/variants/index.d.ts

@@ -1,3 +0,0 @@
-export { BASELINE_VARIANT } from './baseline';
-export { MINIMAL_SINGLE_TOOL_VARIANT, setFlowJsonTool } from './minimal-single-tool';
-export type { VariantConfig } from '../../shared';

package/package/components/copilot/chat/__tests__/flow/variants/index.js

@@ -1,3 +0,0 @@
-// Re-export all variant configurations
-export { BASELINE_VARIANT } from './baseline';
-export { MINIMAL_SINGLE_TOOL_VARIANT, setFlowJsonTool } from './minimal-single-tool';

package/package/components/copilot/chat/__tests__/flow/variants/minimal-single-tool.d.ts

@@ -1,15 +0,0 @@
-import type { VariantConfig } from '../../shared';
-import type { Tool } from '../../shared';
-import type { FlowAIChatHelpers } from '../../../flow/core';
-/**
- * A single tool that sets the entire flow JSON at once.
- * This replaces the granular flow editing tools (add_module, remove_module, modify_module, etc.)
- */
-export declare const setFlowJsonTool: Tool<FlowAIChatHelpers>;
-/**
- * Minimal single-tool variant.
- * Replaces granular flow editing tools (add_module, remove_module, modify_module, etc.)
- * with a single set_flow_json tool, while keeping all other utility tools.
- * Uses the default system prompt.
- */
-export declare const MINIMAL_SINGLE_TOOL_VARIANT: VariantConfig;