windmill-components 1.677.1 → 1.695.0

package/package/components/copilot/chat/__tests__/flow/variants/minimal-single-tool.js

@@ -1,388 +0,0 @@
-import { flowTools } from '../../../flow/core';
-import openFlowSchema from '../../../flow/openFlow.json';
-/**
- * IDs of the granular flow editing tools that should be replaced by set_flow_json.
- */
-const FLOW_EDITING_TOOL_NAMES = [
-    'add_module',
-    'remove_module',
-    'remove_branch',
-    'modify_module',
-    'set_flow_schema'
-];
-/**
- * A single tool that sets the entire flow JSON at once.
- * This replaces the granular flow editing tools (add_module, remove_module, modify_module, etc.)
- */
-export const setFlowJsonTool = {
-    def: {
-        type: 'function',
-        function: {
-            name: 'set_flow_json',
-            description: 'Set the entire flow by providing the complete flow object. This replaces all existing modules and schema.',
-            strict: false,
-            parameters: {
-                type: 'object',
-                properties: {
-                    modules: {
-                        type: 'array',
-                        description: 'Array of flow modules',
-                        items: {
-                            type: 'object'
-                        }
-                    },
-                    schema: {
-                        type: 'object',
-                        description: 'Flow input schema (JSON Schema format) defining parameters the flow accepts'
-                    }
-                },
-                required: ['modules']
-            }
-        }
-    },
-    fn: async ({ args, helpers }) => {
-        const { modules, schema } = args;
-        await helpers.setFlowJson(modules, schema);
-        return `Flow updated with ${modules.length} module(s): [${modules.map((m) => m.id).join(', ')}]`;
-    }
-};
-/**
- * Build the tools array for the minimal-single-tool variant.
- * Keeps all utility tools (search, resource type, test run, db schema, code generation instructions)
- * but replaces all flow editing tools with a single set_flow_json tool.
- */
-function buildMinimalSingleToolTools() {
-    // Get all production tools except flow editing tools
-    const utilityTools = flowTools.filter((t) => !FLOW_EDITING_TOOL_NAMES.includes(t.def.function.name));
-    return [...utilityTools, setFlowJsonTool];
-}
-const MINIMAL_SINGLE_TOOL_SYSTEM_PROMPT = `You are a helpful assistant that creates and edits workflows on the Windmill platform.
-
-## IMPORTANT RULES
-
-**Reserved IDs - Do NOT use these module IDs:**
-- \`failure\` - Reserved for failure handler module
-- \`preprocessor\` - Reserved for preprocessor module
-- \`Input\` - Reserved for flow input reference
-
-## Tool Selection Guide
-
-**Flow Modification:**
-- **Create or modify the entire flow** → \`set_flow_json\` (provide complete modules array and optional schema)
-
-**Code & Scripts:**
-- **Get language-specific coding instructions** → \`get_instructions_for_code_generation\` (call BEFORE writing code)
-- **Find workspace scripts and flows** → \`search_workspace\`
-- **Get details of a specific script or flow** → \`get_runnable_details\`
-- **Find Windmill Hub scripts** → \`search_hub_scripts\`
-
-**Testing:**
-- **Test entire flow** → \`test_run_flow\`
-- **Test single step** → \`test_run_step\`
-
-**Resources & Schema:**
-- **Search resource types** → \`resource_type\`
-- **Get database schema** → \`get_db_schema\`
-
-## Common Mistakes to Avoid
-
-- **Don't forget \`input_transforms\`** - Rawscript parameters won't receive values without them
-- **Don't use spaces in module IDs** - Use underscores (e.g., \`fetch_data\` not \`fetch data\`)
-- **Don't reference future steps** - \`results.step_id\` only works for steps that execute before the current one
-- **Don't create duplicate IDs** - Each module ID must be unique in the flow
-
-## Flow Modification with set_flow_json
-
-Use the \`set_flow_json\` tool to set the entire flow structure at once. Provide the complete modules array and optionally the flow input schema.
-
-**Parameters:**
-- \`modules\`: Array of flow modules (required)
-- \`schema\`: Flow input schema in JSON Schema format (optional)
-
-**Example - Simple flow:**
-\`\`\`javascript
-set_flow_json({
-  modules: [
-    {
-      id: "fetch_data",
-      summary: "Fetch user data from API",
-      value: {
-        type: "rawscript",
-        language: "bun",
-        content: "export async function main(userId: string) { return { id: userId, name: 'John' }; }",
-        input_transforms: {
-          userId: { type: "javascript", expr: "flow_input.user_id" }
-        }
-      }
-    },
-    {
-      id: "process_data",
-      summary: "Process the fetched data",
-      value: {
-        type: "rawscript",
-        language: "bun",
-        content: "export async function main(data: any) { return { processed: true, ...data }; }",
-        input_transforms: {
-          data: { type: "javascript", expr: "results.fetch_data" }
-        }
-      }
-    }
-  ],
-  schema: {
-    type: "object",
-    properties: {
-      user_id: { type: "string", description: "User ID to fetch" }
-    },
-    required: ["user_id"]
-  }
-})
-\`\`\`
-
-**Example - Flow with for loop:**
-\`\`\`javascript
-set_flow_json({
-  modules: [
-    {
-      id: "get_items",
-      summary: "Get list of items",
-      value: {
-        type: "rawscript",
-        language: "bun",
-        content: "export async function main() { return [1, 2, 3]; }",
-        input_transforms: {}
-      }
-    },
-    {
-      id: "loop_items",
-      summary: "Process each item",
-      value: {
-        type: "forloopflow",
-        iterator: { type: "javascript", expr: "results.get_items" },
-        skip_failures: false,
-        parallel: true,
-        modules: [
-          {
-            id: "process_item",
-            summary: "Process single item",
-            value: {
-              type: "rawscript",
-              language: "bun",
-              content: "export async function main(item: number) { return item * 2; }",
-              input_transforms: {
-                item: { type: "javascript", expr: "flow_input.iter.value" }
-              }
-            }
-          }
-        ]
-      }
-    }
-  ]
-})
-\`\`\`
-
-**Example - Flow with branches (branchone):**
-\`\`\`javascript
-set_flow_json({
-  modules: [
-    {
-      id: "get_value",
-      summary: "Get a value to branch on",
-      value: {
-        type: "rawscript",
-        language: "bun",
-        content: "export async function main() { return 50; }",
-        input_transforms: {}
-      }
-    },
-    {
-      id: "branch_on_value",
-      summary: "Branch based on value",
-      value: {
-        type: "branchone",
-        branches: [
-          {
-            summary: "High value",
-            expr: "results.get_value > 75",
-            modules: [
-              {
-                id: "high_handler",
-                value: {
-                  type: "rawscript",
-                  language: "bun",
-                  content: "export async function main() { return 'high'; }",
-                  input_transforms: {}
-                }
-              }
-            ]
-          },
-          {
-            summary: "Medium value",
-            expr: "results.get_value > 25",
-            modules: [
-              {
-                id: "medium_handler",
-                value: {
-                  type: "rawscript",
-                  language: "bun",
-                  content: "export async function main() { return 'medium'; }",
-                  input_transforms: {}
-                }
-              }
-            ]
-          }
-        ],
-        default: [
-          {
-            id: "low_handler",
-            value: {
-              type: "rawscript",
-              language: "bun",
-              content: "export async function main() { return 'low'; }",
-              input_transforms: {}
-            }
-          }
-        ]
-      }
-    }
-  ]
-})
-\`\`\`
-
-Follow the user instructions carefully.
-At the end of your changes, explain precisely what you did and what the flow does now.
-ALWAYS test your modifications using the \`test_run_flow\` tool. If the user cancels the test run, do not try again and wait for the next user instruction.
-When testing steps that are sql scripts, the arguments to be passed are { database: $res:<db_resource> }.
-
-### Input Transforms for Rawscripts
-
-Rawscript modules use \`input_transforms\` to map function parameters to values. Each key in \`input_transforms\` corresponds to a parameter name in your script's \`main\` function.
-
-**Transform Types:**
-- \`static\`: Fixed value passed directly
-- \`javascript\`: Dynamic expression evaluated at runtime
-
-**Available Variables in JavaScript Expressions:**
-- \`flow_input.{property}\` - Access flow input parameters
-- \`results.{step_id}\` - Access output from a previous step
-- \`flow_input.iter.value\` - Current item when inside a for-loop
-- \`flow_input.iter.index\` - Current index when inside a for-loop
-
-**Example - Rawscript using flow input and previous step result:**
-\`\`\`json
-{
-  "id": "step_b",
-  "value": {
-    "type": "rawscript",
-    "language": "bun",
-    "content": "export async function main(userId: string, data: any[]) { return 'Hello, world!'; }",
-    "input_transforms": {
-      "userId": { "type": "javascript", "expr": "flow_input.user_id" },
-      "data": { "type": "javascript", "expr": "results.step_a" }
-    }
-  }
-}
-\`\`\`
-
-**Important:** The parameter names in \`input_transforms\` must match the function parameter names in your script.
-
-### Other Key Concepts
-- **Resources**: For flow inputs, use type "object" with format "resource-<type>". For step inputs, use "$res:path/to/resource"
-- **Module IDs**: Must be unique and valid identifiers. Used to reference results via \`results.step_id\`
-- **Module types**: Use 'bun' as default language for rawscript if unspecified
-
-### Writing Code for Modules
-
-**IMPORTANT: Before writing any code for a rawscript module, you MUST call the \`get_instructions_for_code_generation\` tool with the target language.** This tool provides essential language-specific instructions.
-
-Example: Before writing TypeScript/Bun code, call \`get_instructions_for_code_generation({ language: "bun" })\`
-
-### Creating Flows
-
-1. **Search for existing scripts first** (unless user explicitly asks to write from scratch):
-   - First: \`search_workspace\` to find workspace scripts and flows
-   - Use \`get_runnable_details\` to inspect a specific script or flow (inputs, description, code)
-   - Then: \`search_hub_scripts\` (only consider highly relevant results)
-   - Only create raw scripts if no suitable script is found
-
-2. **Build the complete flow using \`set_flow_json\`:**
-   - If using existing script: use \`type: "script"\` with \`path\`
-   - If creating rawscript: use \`type: "rawscript"\` with \`language\` and \`content\`
-   - **First call \`get_instructions_for_code_generation\` to get the correct code format**
-   - Always define \`input_transforms\` to connect parameters to flow inputs or previous step results
-
-### AI Agent Modules
-
-AI agents can use tools to accomplish tasks. When creating an AI agent module:
-
-\`\`\`javascript
-{
-  id: "support_agent",
-  summary: "AI agent for customer support",
-  value: {
-    type: "aiagent",
-    input_transforms: {
-      provider: { type: "static", value: "$res:f/ai_providers/openai" },
-      output_type: { type: "static", value: "text" },
-      user_message: { type: "javascript", expr: "flow_input.query" },
-      system_prompt: { type: "static", value: "You are a helpful assistant." }
-    },
-    tools: [
-      {
-        id: "search_docs",
-        summary: "Search_documentation",
-        value: {
-          tool_type: "flowmodule",
-          type: "rawscript",
-          language: "bun",
-          content: "export async function main(query: string) { return ['doc1', 'doc2']; }",
-          input_transforms: { query: { type: "static", value: "" } }
-        }
-      }
-    ]
-  }
-}
-\`\`\`
-
-- **Tool IDs**: Cannot contain spaces - use underscores
-- **Tool summaries**: Cannot contain spaces - use underscores
-- **Tool types**: \`flowmodule\` for scripts/flows, \`mcp\` for MCP server tools
-
-## Resource Types
-On Windmill, credentials and configuration are stored in resources. Resource types define the format of the resource.
-- Use the \`resource_type\` tool to search for available resource types (e.g. stripe, google, postgresql, etc.)
-- If the user needs a resource as flow input, set the property type in the schema to "object" and add a key called "format" set to "resource-nameofresourcetype" (e.g. "resource-stripe")
-- If the user wants a specific resource as step input, set the step value to a static string in the format: "$res:path/to/resource"
-
-### OpenFlow Schema Reference
-Below is the complete OpenAPI schema for OpenFlow. All field descriptions and behaviors are defined here. Refer to this as the authoritative reference when generating flow JSON:
-
-\`\`\`json
-${JSON.stringify(openFlowSchema, null, 2)}
-\`\`\`
-
-The schema includes detailed descriptions for:
-- **FlowModuleValue types**: rawscript, script, flow, forloopflow, whileloopflow, branchone, branchall, identity, aiagent
-- **Module configuration**: stop_after_if, skip_if, suspend, sleep, cache_ttl, retry, mock, timeout
-- **InputTransform**: static vs javascript, available variables (results, flow_input, flow_input.iter)
-- **Special modules**: preprocessor_module, failure_module
-- **Loop options**: iterator, parallel, parallelism, skip_failures
-- **Branch types**: BranchOne (first match), BranchAll (all execute)
-`;
-/**
- * Minimal single-tool variant.
- * Replaces granular flow editing tools (add_module, remove_module, modify_module, etc.)
- * with a single set_flow_json tool, while keeping all other utility tools.
- * Uses the default system prompt.
- */
-export const MINIMAL_SINGLE_TOOL_VARIANT = {
-    name: 'minimal-single-tool',
-    description: 'Default prompt with set_flow_json instead of granular flow editing tools, keeps all utility tools',
-    systemPrompt: {
-        type: 'custom',
-        content: MINIMAL_SINGLE_TOOL_SYSTEM_PROMPT
-    },
-    tools: {
-        type: 'custom',
-        tools: buildMinimalSingleToolTools()
-    }
-};

package/package/components/copilot/chat/__tests__/shared/baseEvalRunner.d.ts

@@ -1,45 +0,0 @@
-import type { ChatCompletionMessageParam, ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs';
-import type { TokenUsage, ToolCallDetail, EvalRunnerOptions } from './types';
-import type { Tool } from './baseVariants';
-/**
- * Result from a single eval run (before domain-specific evaluation).
- */
-export interface RawEvalResult<TOutput> {
-    success: boolean;
-    output: TOutput;
-    error?: string;
-    tokenUsage: TokenUsage;
-    toolCallsCount: number;
-    toolsCalled: string[];
-    toolCallDetails: ToolCallDetail[];
-    iterations: number;
-    messages: ChatCompletionMessageParam[];
-}
-/**
- * Parameters for running a base evaluation.
- */
-export interface RunEvalParams<THelpers, TOutput> {
-    /** The user's prompt/instruction */
-    userPrompt: string;
-    /** System message for the LLM */
-    systemMessage: ChatCompletionSystemMessageParam;
-    /** User message for the LLM */
-    userMessage: ChatCompletionMessageParam;
-    /** Tool definitions for the LLM API (unused — derived from tools) */
-    toolDefs?: unknown;
-    /** Full tool implementations for execution */
-    tools: Tool<THelpers>[];
-    /** Domain-specific helpers for tool execution */
-    helpers: THelpers;
-    /** API key for the provider */
-    apiKey: string;
-    /** Function to get the current output state */
-    getOutput: () => TOutput;
-    /** Optional configuration */
-    options?: EvalRunnerOptions;
-}
-/**
- * Runs a generic evaluation using the shared chat loop (same code path as production).
- * Uses streaming via real provider SDKs instead of OpenRouter non-streaming.
- */
-export declare function runEval<THelpers, TOutput>(params: RunEvalParams<THelpers, TOutput>): Promise<RawEvalResult<TOutput>>;

package/package/components/copilot/chat/__tests__/shared/baseEvalRunner.js

@@ -1,121 +0,0 @@
-import OpenAI from 'openai';
-import Anthropic from '@anthropic-ai/sdk';
-import { runChatLoop } from '../../chatLoop';
-/**
- * Creates SDK clients for the given provider.
- */
-function createEvalClients(provider, apiKey) {
-    if (provider === 'anthropic') {
-        return {
-            openai: new OpenAI({ apiKey: 'unused' }),
-            anthropic: new Anthropic({ apiKey })
-        };
-    }
-    return {
-        openai: new OpenAI({ apiKey }),
-        anthropic: new Anthropic({ apiKey: 'unused' })
-    };
-}
-/**
- * Resolves model string to AIProviderModel.
- */
-function resolveModelProvider(model, provider) {
-    if (provider)
-        return { provider, model };
-    if (model.startsWith('claude'))
-        return { provider: 'anthropic', model };
-    if (model.startsWith('gpt') || model.startsWith('o'))
-        return { provider: 'openai', model };
-    return { provider: 'openai', model };
-}
-/**
- * Runs a generic evaluation using the shared chat loop (same code path as production).
- * Uses streaming via real provider SDKs instead of OpenRouter non-streaming.
- */
-export async function runEval(params) {
-    const { systemMessage, userMessage, tools, helpers, apiKey, getOutput, options } = params;
-    const model = options?.model ?? 'gpt-4o';
-    const maxIterations = options?.maxIterations ?? 20;
-    const workspace = options?.workspace ?? 'test-workspace';
-    const provider = options?.provider;
-    const modelProvider = resolveModelProvider(model, provider);
-    const clients = createEvalClients(modelProvider.provider, apiKey);
-    const messages = [userMessage];
-    let toolCallsCount = 0;
-    const toolsCalled = [];
-    const toolCallDetails = [];
-    // Wrap tools to intercept fn calls for tracking.
-    // Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type
-    // but the actual callbacks passed at runtime will satisfy both interfaces.
-    const wrappedTools = tools.map((tool) => ({
-        ...tool,
-        fn: async (p) => {
-            toolCallsCount++;
-            toolsCalled.push(tool.def.function.name);
-            try {
-                const args = typeof p.args === 'string' ? JSON.parse(p.args) : p.args;
-                toolCallDetails.push({ name: tool.def.function.name, arguments: args });
-            }
-            catch {
-                toolCallDetails.push({
-                    name: tool.def.function.name,
-                    arguments: p.args
-                });
-            }
-            return tool.fn(p);
-        }
-    }));
-    // No-op callbacks for eval
-    const callbacks = {
-        setToolStatus: () => { },
-        removeToolStatus: () => { },
-        onNewToken: () => { },
-        onMessageEnd: () => { }
-    };
-    const abortController = new AbortController();
-    try {
-        const result = await runChatLoop({
-            messages,
-            systemMessage,
-            tools: wrappedTools,
-            helpers,
-            abortController,
-            callbacks,
-            modelProvider,
-            clients,
-            workspace,
-            maxIterations,
-            skipResponsesApi: modelProvider.provider !== 'openai' && modelProvider.provider !== 'azure_openai'
-        });
-        return {
-            success: true,
-            output: getOutput(),
-            tokenUsage: { prompt: 0, completion: 0, total: 0 },
-            toolCallsCount,
-            toolsCalled,
-            toolCallDetails,
-            iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length),
-            messages
-        };
-    }
-    catch (err) {
-        let errorMessage;
-        if (err instanceof Error) {
-            errorMessage = err.stack ?? err.message;
-        }
-        else {
-            errorMessage = String(err);
-        }
-        return {
-            success: false,
-            output: getOutput(),
-            error: errorMessage,
-            tokenUsage: { prompt: 0, completion: 0, total: 0 },
-            toolCallsCount,
-            toolsCalled,
-            toolCallDetails,
-            iterations: 0,
-            messages
-        };
-    }
-}

package/package/components/copilot/chat/__tests__/shared/baseLLMEvaluator.d.ts

@@ -1,28 +0,0 @@
-import type { EvaluationResult } from './types';
-/**
- * Parameters for LLM-based evaluation.
- */
-export interface EvaluateParams {
-    /** The user's original request/prompt */
-    userPrompt: string;
-    /** The generated output to evaluate */
-    generatedOutput: unknown;
-    /** The expected/reference output */
-    expectedOutput: unknown;
-    /** Domain-specific system prompt for the evaluator */
-    evaluatorSystemPrompt: string;
-    /** Anthropic API key for evaluation */
-    apiKey?: string;
-    /** Model to use for evaluation (default: 'claude-sonnet-4-5-20250514') */
-    model?: string;
-}
-/**
- * Base evaluator system prompt template.
- * Domain-specific evaluators should build on this structure.
- */
-export declare const BASE_EVALUATOR_RESPONSE_FORMAT = "\n## Response Format\nYou MUST respond with valid JSON only, no additional text:\n{\n  \"resemblanceScore\": <0-100 integer>,\n  \"statement\": \"<brief 1-2 sentence summary of how well the output matches the user request and expected output>\",\n  \"missingRequirements\": [\"<list any requirements from user prompt that are missing or incorrectly implemented>\"]\n}\n\nScore guidelines:\n- 90-100: Fully addresses user request, functionally equivalent to expected output\n- 70-89: Addresses most user requirements, same overall structure with minor differences\n- 50-69: Partially addresses user request, achieves similar goal but different approach\n- 30-49: Missing significant requirements from user request\n- 0-29: Does not address user request or significantly incorrect";
-/**
- * Evaluates how well a generated output matches an expected output using an LLM.
- * Uses Anthropic API directly instead of OpenRouter.
- */
-export declare function evaluateWithLLM(params: EvaluateParams): Promise<EvaluationResult>;

package/package/components/copilot/chat/__tests__/shared/baseLLMEvaluator.js

@@ -1,96 +0,0 @@
-import Anthropic from '@anthropic-ai/sdk';
-/**
- * Base evaluator system prompt template.
- * Domain-specific evaluators should build on this structure.
- */
-export const BASE_EVALUATOR_RESPONSE_FORMAT = `
-## Response Format
-You MUST respond with valid JSON only, no additional text:
-{
-  "resemblanceScore": <0-100 integer>,
-  "statement": "<brief 1-2 sentence summary of how well the output matches the user request and expected output>",
-  "missingRequirements": ["<list any requirements from user prompt that are missing or incorrectly implemented>"]
-}
-
-Score guidelines:
-- 90-100: Fully addresses user request, functionally equivalent to expected output
-- 70-89: Addresses most user requirements, same overall structure with minor differences
-- 50-69: Partially addresses user request, achieves similar goal but different approach
-- 30-49: Missing significant requirements from user request
-- 0-29: Does not address user request or significantly incorrect`;
-/**
- * Evaluates how well a generated output matches an expected output using an LLM.
- * Uses Anthropic API directly instead of OpenRouter.
- */
-export async function evaluateWithLLM(params) {
-    const { userPrompt, generatedOutput, expectedOutput, evaluatorSystemPrompt, apiKey, model = 'claude-sonnet-4-5-20250514' } = params;
-    // @ts-ignore - process.env
-    const anthropicKey = apiKey ?? process.env.ANTHROPIC_API_KEY;
-    if (!anthropicKey) {
-        return {
-            success: false,
-            resemblanceScore: 0,
-            statement: 'No API key available for evaluation',
-            error: 'ANTHROPIC_API_KEY not set and no apiKey provided'
-        };
-    }
-    const client = new Anthropic({ apiKey: anthropicKey });
-    const userMessage = `## User's Original Request
-${userPrompt}
-
-## Expected Reference Output
-\`\`\`json
-${JSON.stringify(expectedOutput, null, 2)}
-\`\`\`
-
-## Generated Output
-\`\`\`json
-${JSON.stringify(generatedOutput, null, 2)}
-\`\`\`
-
-Please evaluate how well the generated output:
-1. Fulfills ALL requirements from the user's original request
-2. Matches the structure and logic of the expected reference output`;
-    try {
-        const response = await client.messages.create({
-            model,
-            max_tokens: 2048,
-            system: evaluatorSystemPrompt,
-            messages: [
-                { role: 'user', content: userMessage }
-            ],
-            temperature: 0
-        });
-        const textBlock = response.content.find((block) => block.type === 'text');
-        const content = textBlock?.text;
-        if (!content) {
-            return {
-                success: false,
-                resemblanceScore: 0,
-                statement: 'No response from evaluator',
-                error: 'Empty response from LLM'
-            };
-        }
-        // Parse JSON response - handle potential markdown code blocks
-        let jsonContent = content.trim();
-        if (jsonContent.startsWith('```')) {
-            jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
-        }
-        const parsed = JSON.parse(jsonContent);
-        return {
-            success: true,
-            resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
-            statement: parsed.statement,
-            missingRequirements: parsed.missingRequirements ?? []
-        };
-    }
-    catch (err) {
-        const errorMessage = err instanceof Error ? err.message : String(err);
-        return {
-            success: false,
-            resemblanceScore: 0,
-            statement: 'Evaluation failed',
-            error: errorMessage
-        };
-    }
-}