windmill-components 1.558.4 → 1.596.0

package/package/components/copilot/chat/__tests__/flow/variants/minimal-single-tool.js

@@ -0,0 +1,386 @@
+import { flowTools } from '../../../flow/core';
+import openFlowSchema from '../../../flow/openFlow.json';
+/**
+ * IDs of the granular flow editing tools that should be replaced by set_flow_json.
+ */
+const FLOW_EDITING_TOOL_NAMES = [
+    'add_module',
+    'remove_module',
+    'remove_branch',
+    'modify_module',
+    'set_flow_schema'
+];
+/**
+ * A single tool that sets the entire flow JSON at once.
+ * This replaces the granular flow editing tools (add_module, remove_module, modify_module, etc.)
+ */
+export const setFlowJsonTool = {
+    def: {
+        type: 'function',
+        function: {
+            name: 'set_flow_json',
+            description: 'Set the entire flow by providing the complete flow object. This replaces all existing modules and schema.',
+            strict: false,
+            parameters: {
+                type: 'object',
+                properties: {
+                    modules: {
+                        type: 'array',
+                        description: 'Array of flow modules',
+                        items: {
+                            type: 'object'
+                        }
+                    },
+                    schema: {
+                        type: 'object',
+                        description: 'Flow input schema (JSON Schema format) defining parameters the flow accepts'
+                    }
+                },
+                required: ['modules']
+            }
+        }
+    },
+    fn: async ({ args, helpers }) => {
+        const { modules, schema } = args;
+        await helpers.setFlowJson(modules, schema);
+        return `Flow updated with ${modules.length} module(s): [${modules.map((m) => m.id).join(', ')}]`;
+    }
+};
+/**
+ * Build the tools array for the minimal-single-tool variant.
+ * Keeps all utility tools (search, resource type, test run, db schema, code generation instructions)
+ * but replaces all flow editing tools with a single set_flow_json tool.
+ */
+function buildMinimalSingleToolTools() {
+    // Get all production tools except flow editing tools
+    const utilityTools = flowTools.filter((t) => !FLOW_EDITING_TOOL_NAMES.includes(t.def.function.name));
+    return [...utilityTools, setFlowJsonTool];
+}
+const MINIMAL_SINGLE_TOOL_SYSTEM_PROMPT = `You are a helpful assistant that creates and edits workflows on the Windmill platform.
+
+## IMPORTANT RULES
+
+**Reserved IDs - Do NOT use these module IDs:**
+- \`failure\` - Reserved for failure handler module
+- \`preprocessor\` - Reserved for preprocessor module
+- \`Input\` - Reserved for flow input reference
+
+## Tool Selection Guide
+
+**Flow Modification:**
+- **Create or modify the entire flow** → \`set_flow_json\` (provide complete modules array and optional schema)
+
+**Code & Scripts:**
+- **Get language-specific coding instructions** → \`get_instructions_for_code_generation\` (call BEFORE writing code)
+- **Find workspace scripts** → \`search_scripts\`
+- **Find Windmill Hub scripts** → \`search_hub_scripts\`
+
+**Testing:**
+- **Test entire flow** → \`test_run_flow\`
+- **Test single step** → \`test_run_step\`
+
+**Resources & Schema:**
+- **Search resource types** → \`resource_type\`
+- **Get database schema** → \`get_db_schema\`
+
+## Common Mistakes to Avoid
+
+- **Don't forget \`input_transforms\`** - Rawscript parameters won't receive values without them
+- **Don't use spaces in module IDs** - Use underscores (e.g., \`fetch_data\` not \`fetch data\`)
+- **Don't reference future steps** - \`results.step_id\` only works for steps that execute before the current one
+- **Don't create duplicate IDs** - Each module ID must be unique in the flow
+
+## Flow Modification with set_flow_json
+
+Use the \`set_flow_json\` tool to set the entire flow structure at once. Provide the complete modules array and optionally the flow input schema.
+
+**Parameters:**
+- \`modules\`: Array of flow modules (required)
+- \`schema\`: Flow input schema in JSON Schema format (optional)
+
+**Example - Simple flow:**
+\`\`\`javascript
+set_flow_json({
+  modules: [
+    {
+      id: "fetch_data",
+      summary: "Fetch user data from API",
+      value: {
+        type: "rawscript",
+        language: "bun",
+        content: "export async function main(userId: string) { return { id: userId, name: 'John' }; }",
+        input_transforms: {
+          userId: { type: "javascript", expr: "flow_input.user_id" }
+        }
+      }
+    },
+    {
+      id: "process_data",
+      summary: "Process the fetched data",
+      value: {
+        type: "rawscript",
+        language: "bun",
+        content: "export async function main(data: any) { return { processed: true, ...data }; }",
+        input_transforms: {
+          data: { type: "javascript", expr: "results.fetch_data" }
+        }
+      }
+    }
+  ],
+  schema: {
+    type: "object",
+    properties: {
+      user_id: { type: "string", description: "User ID to fetch" }
+    },
+    required: ["user_id"]
+  }
+})
+\`\`\`
+
+**Example - Flow with for loop:**
+\`\`\`javascript
+set_flow_json({
+  modules: [
+    {
+      id: "get_items",
+      summary: "Get list of items",
+      value: {
+        type: "rawscript",
+        language: "bun",
+        content: "export async function main() { return [1, 2, 3]; }",
+        input_transforms: {}
+      }
+    },
+    {
+      id: "loop_items",
+      summary: "Process each item",
+      value: {
+        type: "forloopflow",
+        iterator: { type: "javascript", expr: "results.get_items" },
+        skip_failures: false,
+        parallel: true,
+        modules: [
+          {
+            id: "process_item",
+            summary: "Process single item",
+            value: {
+              type: "rawscript",
+              language: "bun",
+              content: "export async function main(item: number) { return item * 2; }",
+              input_transforms: {
+                item: { type: "javascript", expr: "flow_input.iter.value" }
+              }
+            }
+          }
+        ]
+      }
+    }
+  ]
+})
+\`\`\`
+
+**Example - Flow with branches (branchone):**
+\`\`\`javascript
+set_flow_json({
+  modules: [
+    {
+      id: "get_value",
+      summary: "Get a value to branch on",
+      value: {
+        type: "rawscript",
+        language: "bun",
+        content: "export async function main() { return 50; }",
+        input_transforms: {}
+      }
+    },
+    {
+      id: "branch_on_value",
+      summary: "Branch based on value",
+      value: {
+        type: "branchone",
+        branches: [
+          {
+            summary: "High value",
+            expr: "results.get_value > 75",
+            modules: [
+              {
+                id: "high_handler",
+                value: {
+                  type: "rawscript",
+                  language: "bun",
+                  content: "export async function main() { return 'high'; }",
+                  input_transforms: {}
+                }
+              }
+            ]
+          },
+          {
+            summary: "Medium value",
+            expr: "results.get_value > 25",
+            modules: [
+              {
+                id: "medium_handler",
+                value: {
+                  type: "rawscript",
+                  language: "bun",
+                  content: "export async function main() { return 'medium'; }",
+                  input_transforms: {}
+                }
+              }
+            ]
+          }
+        ],
+        default: [
+          {
+            id: "low_handler",
+            value: {
+              type: "rawscript",
+              language: "bun",
+              content: "export async function main() { return 'low'; }",
+              input_transforms: {}
+            }
+          }
+        ]
+      }
+    }
+  ]
+})
+\`\`\`
+
+Follow the user instructions carefully.
+At the end of your changes, explain precisely what you did and what the flow does now.
+ALWAYS test your modifications using the \`test_run_flow\` tool. If the user cancels the test run, do not try again and wait for the next user instruction.
+When testing steps that are sql scripts, the arguments to be passed are { database: $res:<db_resource> }.
+
+### Input Transforms for Rawscripts
+
+Rawscript modules use \`input_transforms\` to map function parameters to values. Each key in \`input_transforms\` corresponds to a parameter name in your script's \`main\` function.
+
+**Transform Types:**
+- \`static\`: Fixed value passed directly
+- \`javascript\`: Dynamic expression evaluated at runtime
+
+**Available Variables in JavaScript Expressions:**
+- \`flow_input.{property}\` - Access flow input parameters
+- \`results.{step_id}\` - Access output from a previous step
+- \`flow_input.iter.value\` - Current item when inside a for-loop
+- \`flow_input.iter.index\` - Current index when inside a for-loop
+
+**Example - Rawscript using flow input and previous step result:**
+\`\`\`json
+{
+  "id": "step_b",
+  "value": {
+    "type": "rawscript",
+    "language": "bun",
+    "content": "export async function main(userId: string, data: any[]) { return 'Hello, world!'; }",
+    "input_transforms": {
+      "userId": { "type": "javascript", "expr": "flow_input.user_id" },
+      "data": { "type": "javascript", "expr": "results.step_a" }
+    }
+  }
+}
+\`\`\`
+
+**Important:** The parameter names in \`input_transforms\` must match the function parameter names in your script.
+
+### Other Key Concepts
+- **Resources**: For flow inputs, use type "object" with format "resource-<type>". For step inputs, use "$res:path/to/resource"
+- **Module IDs**: Must be unique and valid identifiers. Used to reference results via \`results.step_id\`
+- **Module types**: Use 'bun' as default language for rawscript if unspecified
+
+### Writing Code for Modules
+
+**IMPORTANT: Before writing any code for a rawscript module, you MUST call the \`get_instructions_for_code_generation\` tool with the target language.** This tool provides essential language-specific instructions.
+
+Example: Before writing TypeScript/Bun code, call \`get_instructions_for_code_generation({ language: "bun" })\`
+
+### Creating Flows
+
+1. **Search for existing scripts first** (unless user explicitly asks to write from scratch):
+   - First: \`search_scripts\` to find workspace scripts
+   - Then: \`search_hub_scripts\` (only consider highly relevant results)
+   - Only create raw scripts if no suitable script is found
+
+2. **Build the complete flow using \`set_flow_json\`:**
+   - If using existing script: use \`type: "script"\` with \`path\`
+   - If creating rawscript: use \`type: "rawscript"\` with \`language\` and \`content\`
+   - **First call \`get_instructions_for_code_generation\` to get the correct code format**
+   - Always define \`input_transforms\` to connect parameters to flow inputs or previous step results
+
+### AI Agent Modules
+
+AI agents can use tools to accomplish tasks. When creating an AI agent module:
+
+\`\`\`javascript
+{
+  id: "support_agent",
+  summary: "AI agent for customer support",
+  value: {
+    type: "aiagent",
+    input_transforms: {
+      provider: { type: "static", value: "$res:f/ai_providers/openai" },
+      output_type: { type: "static", value: "text" },
+      user_message: { type: "javascript", expr: "flow_input.query" },
+      system_prompt: { type: "static", value: "You are a helpful assistant." }
+    },
+    tools: [
+      {
+        id: "search_docs",
+        summary: "Search_documentation",
+        value: {
+          tool_type: "flowmodule",
+          type: "rawscript",
+          language: "bun",
+          content: "export async function main(query: string) { return ['doc1', 'doc2']; }",
+          input_transforms: { query: { type: "static", value: "" } }
+        }
+      }
+    ]
+  }
+}
+\`\`\`
+
+- **Tool IDs**: Cannot contain spaces - use underscores
+- **Tool summaries**: Cannot contain spaces - use underscores
+- **Tool types**: \`flowmodule\` for scripts/flows, \`mcp\` for MCP server tools
+
+## Resource Types
+On Windmill, credentials and configuration are stored in resources. Resource types define the format of the resource.
+- Use the \`resource_type\` tool to search for available resource types (e.g. stripe, google, postgresql, etc.)
+- If the user needs a resource as flow input, set the property type in the schema to "object" and add a key called "format" set to "resource-nameofresourcetype" (e.g. "resource-stripe")
+- If the user wants a specific resource as step input, set the step value to a static string in the format: "$res:path/to/resource"
+
+### OpenFlow Schema Reference
+Below is the complete OpenAPI schema for OpenFlow. All field descriptions and behaviors are defined here. Refer to this as the authoritative reference when generating flow JSON:
+
+\`\`\`json
+${JSON.stringify(openFlowSchema, null, 2)}
+\`\`\`
+
+The schema includes detailed descriptions for:
+- **FlowModuleValue types**: rawscript, script, flow, forloopflow, whileloopflow, branchone, branchall, identity, aiagent
+- **Module configuration**: stop_after_if, skip_if, suspend, sleep, cache_ttl, retry, mock, timeout
+- **InputTransform**: static vs javascript, available variables (results, flow_input, flow_input.iter)
+- **Special modules**: preprocessor_module, failure_module
+- **Loop options**: iterator, parallel, parallelism, skip_failures
+- **Branch types**: BranchOne (first match), BranchAll (all execute)
+`;
+/**
+ * Minimal single-tool variant.
+ * Replaces granular flow editing tools (add_module, remove_module, modify_module, etc.)
+ * with a single set_flow_json tool, while keeping all other utility tools.
+ * Uses the default system prompt.
+ */
+export const MINIMAL_SINGLE_TOOL_VARIANT = {
+    name: 'minimal-single-tool',
+    description: 'Default prompt with set_flow_json instead of granular flow editing tools, keeps all utility tools',
+    systemPrompt: {
+        type: 'custom',
+        content: MINIMAL_SINGLE_TOOL_SYSTEM_PROMPT
+    },
+    tools: {
+        type: 'custom',
+        tools: buildMinimalSingleToolTools()
+    }
+};

package/package/components/copilot/chat/__tests__/shared/baseEvalRunner.d.ts

@@ -0,0 +1,48 @@
+import type { ChatCompletionMessageParam, ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs';
+import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs';
+import type { TokenUsage, ToolCallDetail, EvalRunnerOptions } from './types';
+import type { Tool } from './baseVariants';
+/**
+ * Result from a single eval run (before domain-specific evaluation).
+ */
+export interface RawEvalResult<TOutput> {
+    success: boolean;
+    output: TOutput;
+    error?: string;
+    tokenUsage: TokenUsage;
+    toolCallsCount: number;
+    toolsCalled: string[];
+    toolCallDetails: ToolCallDetail[];
+    iterations: number;
+    messages: ChatCompletionMessageParam[];
+}
+/**
+ * Parameters for running a base evaluation.
+ */
+export interface RunEvalParams<THelpers, TOutput> {
+    /** The user's prompt/instruction */
+    userPrompt: string;
+    /** System message for the LLM */
+    systemMessage: ChatCompletionSystemMessageParam;
+    /** User message for the LLM */
+    userMessage: ChatCompletionMessageParam;
+    /** Tool definitions for the LLM API */
+    toolDefs: ChatCompletionTool[];
+    /** Full tool implementations for execution */
+    tools: Tool<THelpers>[];
+    /** Domain-specific helpers for tool execution */
+    helpers: THelpers;
+    /** API key for OpenRouter */
+    apiKey: string;
+    /** Function to get the current output state */
+    getOutput: () => TOutput;
+    /** Optional configuration */
+    options?: EvalRunnerOptions;
+}
+/**
+ * Runs a generic evaluation with real LLM API calls.
+ * Executes tool calls in a loop until the LLM stops calling tools.
+ *
+ * This is the core execution loop shared across all chat eval tests.
+ */
+export declare function runEval<THelpers, TOutput>(params: RunEvalParams<THelpers, TOutput>): Promise<RawEvalResult<TOutput>>;

package/package/components/copilot/chat/__tests__/shared/baseEvalRunner.js

@@ -0,0 +1,147 @@
+import OpenAI, { APIError } from 'openai';
+/**
+ * Runs a generic evaluation with real LLM API calls.
+ * Executes tool calls in a loop until the LLM stops calling tools.
+ *
+ * This is the core execution loop shared across all chat eval tests.
+ */
+export async function runEval(params) {
+    const { systemMessage, userMessage, toolDefs, tools, helpers, apiKey, getOutput, options } = params;
+    const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey });
+    const model = options?.model ?? 'gpt-4o';
+    const maxIterations = options?.maxIterations ?? 20;
+    const workspace = options?.workspace ?? 'test-workspace';
+    const messages = [systemMessage, userMessage];
+    const totalTokens = { prompt: 0, completion: 0, total: 0 };
+    let toolCallsCount = 0;
+    const toolsCalled = [];
+    const toolCallDetails = [];
+    let iterations = 0;
+    // No-op tool callbacks for eval
+    const toolCallbacks = {
+        setToolStatus: () => { },
+        removeToolStatus: () => { }
+    };
+    try {
+        // Tool resolution loop
+        while (iterations < maxIterations) {
+            iterations++;
+            const response = await client.chat.completions.create({
+                model,
+                messages,
+                tools: toolDefs,
+                temperature: 0
+            });
+            // Track token usage
+            if (response.usage) {
+                totalTokens.prompt += response.usage.prompt_tokens;
+                totalTokens.completion += response.usage.completion_tokens;
+                totalTokens.total += response.usage.total_tokens;
+            }
+            if (!response.choices.length) {
+                throw new Error('No response from API');
+            }
+            const choice = response.choices[0];
+            const assistantMessage = choice.message;
+            // Add assistant message to history
+            messages.push(assistantMessage);
+            // If no tool calls, we're done
+            if (!assistantMessage.tool_calls?.length) {
+                break;
+            }
+            // Execute each tool call
+            for (const toolCall of assistantMessage.tool_calls) {
+                toolCallsCount++;
+                // Type guard: only handle function tool calls
+                if (toolCall.type !== 'function') {
+                    messages.push({
+                        role: 'tool',
+                        tool_call_id: toolCall.id,
+                        content: `Unsupported tool type: ${toolCall.type}`
+                    });
+                    continue;
+                }
+                toolsCalled.push(toolCall.function.name);
+                const tool = tools.find((t) => t.def.function.name === toolCall.function.name);
+                if (!tool) {
+                    messages.push({
+                        role: 'tool',
+                        tool_call_id: toolCall.id,
+                        content: `Unknown tool: ${toolCall.function.name}`
+                    });
+                    continue;
+                }
+                try {
+                    const args = JSON.parse(toolCall.function.arguments);
+                    toolCallDetails.push({ name: toolCall.function.name, arguments: args });
+                    const result = await tool.fn({
+                        args,
+                        workspace,
+                        helpers,
+                        toolCallbacks,
+                        toolId: toolCall.id
+                    });
+                    messages.push({
+                        role: 'tool',
+                        tool_call_id: toolCall.id,
+                        content: result
+                    });
+                }
+                catch (err) {
+                    const errorMessage = err instanceof Error ? err.message : String(err);
+                    messages.push({
+                        role: 'tool',
+                        tool_call_id: toolCall.id,
+                        content: `Error: ${errorMessage}`
+                    });
+                }
+            }
+        }
+        return {
+            success: true,
+            output: getOutput(),
+            tokenUsage: totalTokens,
+            toolCallsCount,
+            toolsCalled,
+            toolCallDetails,
+            iterations,
+            messages
+        };
+    }
+    catch (err) {
+        // Build detailed error message
+        let errorMessage;
+        if (err instanceof APIError) {
+            const details = [`${err.status} ${err.message}`];
+            if (err.code)
+                details.push(`Code: ${err.code}`);
+            if (err.type)
+                details.push(`Type: ${err.type}`);
+            if (err.param)
+                details.push(`Param: ${err.param}`);
+            if (err.requestID)
+                details.push(`Request ID: ${err.requestID}`);
+            if (err.error && typeof err.error === 'object') {
+                details.push(`Response: ${JSON.stringify(err.error, null, 2)}`);
+            }
+            errorMessage = details.join('\n');
+        }
+        else if (err instanceof Error) {
+            errorMessage = err.stack ?? err.message;
+        }
+        else {
+            errorMessage = String(err);
+        }
+        return {
+            success: false,
+            output: getOutput(),
+            error: errorMessage,
+            tokenUsage: totalTokens,
+            toolCallsCount,
+            toolsCalled,
+            toolCallDetails,
+            iterations,
+            messages
+        };
+    }
+}

package/package/components/copilot/chat/__tests__/shared/baseLLMEvaluator.d.ts

@@ -0,0 +1,31 @@
+import type { EvaluationResult } from './types';
+/**
+ * Parameters for LLM-based evaluation.
+ */
+export interface EvaluateParams {
+    /** The user's original request/prompt */
+    userPrompt: string;
+    /** The generated output to evaluate */
+    generatedOutput: unknown;
+    /** The expected/reference output */
+    expectedOutput: unknown;
+    /** Domain-specific system prompt for the evaluator */
+    evaluatorSystemPrompt: string;
+    /** API key for OpenRouter */
+    apiKey: string;
+    /** Model to use for evaluation (default: 'anthropic/claude-sonnet-4.5') */
+    model?: string;
+}
+/**
+ * Base evaluator system prompt template.
+ * Domain-specific evaluators should build on this structure.
+ */
+export declare const BASE_EVALUATOR_RESPONSE_FORMAT = "\n## Response Format\nYou MUST respond with valid JSON only, no additional text:\n{\n  \"resemblanceScore\": <0-100 integer>,\n  \"statement\": \"<brief 1-2 sentence summary of how well the output matches the user request and expected output>\",\n  \"missingRequirements\": [\"<list any requirements from user prompt that are missing or incorrectly implemented>\"]\n}\n\nScore guidelines:\n- 90-100: Fully addresses user request, functionally equivalent to expected output\n- 70-89: Addresses most user requirements, same overall structure with minor differences\n- 50-69: Partially addresses user request, achieves similar goal but different approach\n- 30-49: Missing significant requirements from user request\n- 0-29: Does not address user request or significantly incorrect";
+/**
+ * Evaluates how well a generated output matches an expected output using an LLM.
+ * Returns a resemblance score (0-100), a qualitative statement, and any missing requirements.
+ *
+ * @param params Evaluation parameters including prompts, outputs, and API configuration
+ * @returns Evaluation result with score, statement, and missing requirements
+ */
+export declare function evaluateWithLLM(params: EvaluateParams): Promise<EvaluationResult>;