rl-simulator-core 1.0.0

package/dist/index.mjs

@@ -0,0 +1,476 @@
+// src/index.js
+import fs2 from "fs";
+import path3 from "path";
+
+// src/runner.js
+import { chromium } from "playwright";
+import fs from "fs";
+import path2 from "path";
+
+// src/ai.js
+import { createOpenAI } from "@ai-sdk/openai";
+import { generateText } from "ai";
+
+// src/config.js
+import dotenv from "dotenv";
+dotenv.config();
+var API_KEY = process.env.ARK_API_KEY;
+var ENDPOINT = process.env.EP;
+var BASE_URL = process.env.BASE_URL;
+if (!API_KEY || !ENDPOINT || !BASE_URL) {
+  console.warn("Missing environment variables (ARK_API_KEY, EP, BASE_URL). Agent may fail if not provided later.");
+}
+var SYSTEM_PROMPT = `
+Intro to Think Mode
+To explore the performance of our model with different styles of thinking, we introduce four modes of thinking: budgted, efficient, no think, and unrestricted. The model is default to unrestricted think mode.
+- Budgeted think mode: Limit model's internal reasoning and reflections to stay within the specified token budget.
+- Efficient think mode: Model provides a concise internal thinking process with efficient reasoning and reflections. It doesn't have a strict token budget, but should be less verbose and more direct in your thinking.
+- No think mode: Model responds directly to the question without any internal reasoning process or extra thinking tokens. Still follows the template with the minimum required thinking tokens to justify the answer.
+- Unrestricted think mode: Model engages in an internal thinking process with thorough reasoning and reflections. There's an unlimited budget for thinking tokens and can continue thinking until you fully solve the problem.
+Think Mode Assignment
+As the "think mode" sp is not encapsulated yet, you need to manually sew in different sp for different modes of thinking. Here are the sp's: 
+NOTE: You should disable "thinking" first: (I know it's a bit counterintuitive.)
+"thinking": {
+"type": "disabled"
+}
+- Unrestricted think mode
+You should begin by detailing the internal reasoning process, and then present the answer to the user. The reasoning process should be enclosed within <think_never_used_51bce0c785ca2f68081bfa7d91973934> and </think_never_used_51bce0c785ca2f68081bfa7d91973934> tags, as follows:
+<think_never_used_51bce0c785ca2f68081bfa7d91973934> reasoning process here </think_never_used_51bce0c785ca2f68081bfa7d91973934> answer here.
+You have different modes of thinking:
+Unrestricted think mode: Engage in an internal thinking process with thorough reasoning and reflections. You have an unlimited budget for thinking tokens and can continue thinking until you fully solve the problem.
+Efficient think mode: Provide a concise internal thinking process with efficient reasoning and reflections. You don't have a strict token budget but be less verbose and more direct in your thinking.
+No think mode: Respond directly to the question without any internal reasoning process or extra thinking tokens. Still follow the template with the minimum required thinking tokens to justify the answer.
+Budgeted think mode: Limit your internal reasoning and reflections to stay within the specified token budget
+Based on the complexity of the problem, select the appropriate mode for reasoning among the provided options listed below.
+Provided Mode(s):
+Unrestricted think
+Action Space
+"""
+## Function Definition
+- You have access to the following functions:
+{"type": "function", "name": "click", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse left single click action."}
+{"type": "function", "name": "left_double", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse left double click action."}
+{"type": "function", "name": "left_triple", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse left triple click action."}
+{"type": "function", "name": "right_single", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse right single click action."}
+{"type": "function", "name": "drag", "parameters": {"type": "object", "properties": {"start_point": {"type": "string", "description": "Drag start point. The format is: x y"}, "end_point": {"type": "string", "description": "Drag end point. The format is: x y"}}, "required": ["start_point", "end_point"]}, "description": "Mouse left button drag action."}
+{"type": "function", "name": "scroll", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Scroll start position. If not specified, default to execute on the current mouse position. The format is: x y"}, "direction": {"type": "string", "description": "Scroll direction.", "enum": ["up", "down", "left", "right"]}}, "required": ["direction"]}, "description": "Scroll action."}
+{"type": "function", "name": "move_to", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Target coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse move action."}
+{"type": "function", "name": "mouse_down", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Mouse down position. If not specified, default to execute on the current mouse position. The format is: x y"}, "button": {"type": "string", "description": "Down button. Default to left.", "enum": ["left", "right"]}}, "required": []}, "description": "Mouse down action."}
+{"type": "function", "name": "mouse_up", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Mouse up position. If not specified, default to execute on the current mouse position. The format is: x y"}, "button": {"type": "string", "description": "Up button. Default to left.", "enum": ["left", "right"]}}, "required": []}, "description": "Mouse up action."}
+{"type": "function", "name": "type", "parameters": {"type": "object", "properties": {"content": {"type": "string", "description": "Type content. If you want to submit your input, use \\n at the end of content."}}, "required": ["content"]}, "description": "Type content."}
+{"type": "function", "name": "hotkey", "parameters": {"type": "object", "properties": {"key": {"type": "string", "description": "Hotkeys you want to press. Split keys with a space and use lowercase."}}, "required": ["key"]}, "description": "Press hotkey."}
+{"type": "function", "name": "press", "parameters": {"type": "object", "properties": {"key": {"type": "string", "description": "Key you want to press. Only one key can be pressed at one time."}}, "required": ["key"]}, "description": "Press key."}
+{"type": "function", "name": "call_user", "parameters": {"type": "object", "properties": {"content": {"type": "string", "description": "Message or information displayed to the user to request their input, feedback, or guidance."}}, "required": []}, "description": "This function is used to interact with the user by displaying a message and requesting their input, feedback, or guidance."}
+{"type": "function", "name": "wait", "parameters": {"type": "object", "properties": {"time": {"type": "string", "description": "Wait time in seconds."}}, "required": []}, "description": "Wait for a while."}
+{"type": "function", "name": "finished", "parameters": {"type": "object", "properties": {"content": {"type": "string", "description": "Provide the final answer or response to complete the task."}}, "required": []}, "description": "This function is used to indicate the completion of a task by providing the final answer or response."}
+
+- To call a function, use the following structure without any suffix:
+
+<gui_think> reasoning process </gui_think>
+<seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934><function_never_used_51bce0c785ca2f68081bfa7d91973934=example_function_name><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_1>value_1</parameter_never_used_51bce0c785ca2f68081bfa7d91973934><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_2>
+This is the value for the second parameter
+that can span
+multiple lines
+</parameter_never_used_51bce0c785ca2f68081bfa7d91973934></function_never_used_51bce0c785ca2f68081bfa7d91973934></seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934>
+
+## Important Notes
+- Function calls must begin with <function_never_used_51bce0c785ca2f68081bfa7d91973934= and end with </function_never_used_51bce0c785ca2f68081bfa7d91973934>.
+- All required parameters must be explicitly provided.
+
+## Additional Notes
+- You can execute multiple actions within a single tool call. For example:
+<seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934><function_never_used_51bce0c785ca2f68081bfa7d91973934=example_function_1><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_1>value_1</parameter_never_used_51bce0c785ca2f68081bfa7d91973934><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_2>
+This is the value for the second parameter
+that can span
+multiple lines
+</parameter_never_used_51bce0c785ca2f68081bfa7d91973934></function_never_used_51bce0c785ca2f68081bfa7d91973934><function_never_used_51bce0c785ca2f68081bfa7d91973934=example_function_2><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_3>value_4</parameter_never_used_51bce0c785ca2f68081bfa7d91973934></function_never_used_51bce0c785ca2f68081bfa7d91973934></seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934>
+"""`;
+
+// src/ai.js
+import path from "path";
+var logFile = path.join(process.cwd(), "ai_traffic.log");
+var openai = createOpenAI({
+  baseURL: BASE_URL,
+  apiKey: API_KEY,
+  compatibility: "compatible"
+});
+async function queryAI(messages) {
+  console.log("Querying AI with history length:", messages.length);
+  try {
+    console.log("\u23F3 \u53D1\u9001\u8BF7\u6C42\u4E2D...");
+    const startTime = Date.now();
+    const { text } = await generateText({
+      model: openai.chat(ENDPOINT),
+      system: SYSTEM_PROMPT,
+      messages,
+      temperature: 0.1,
+      topP: 0.7
+    });
+    const endTime = Date.now();
+    console.log("\u2705 \u8BF7\u6C42\u6210\u529F! \u8017\u65F6:", (endTime - startTime) / 1e3, "\u79D2");
+    console.log("\u{1F4E5} \u54CD\u5E94\u5185\u5BB9:", text.substring(0, 500) + "...");
+    return text;
+  } catch (error) {
+    console.error("\u274C AI \u8BF7\u6C42\u5931\u8D25!", error);
+    return null;
+  }
+}
+function parseActions(aiContent) {
+  const actions = [];
+  const functionRegex = /<function[^>]*=([^>]+)>([\s\S]*?)<\/function[^>]*>/g;
+  let functionMatch;
+  while ((functionMatch = functionRegex.exec(aiContent)) !== null) {
+    const functionName = functionMatch[1];
+    const functionBody = functionMatch[2];
+    const params = {};
+    const paramRegex = /<parameter[^>]*=([^>]+)>([\s\S]*?)<\/parameter[^>]*>/g;
+    let paramMatch;
+    while ((paramMatch = paramRegex.exec(functionBody)) !== null) {
+      const paramName = paramMatch[1];
+      let paramValue = paramMatch[2].trim();
+      const cleanedValue = paramValue.replace(/<[^>]+>/g, "").trim();
+      params[paramName] = cleanedValue;
+    }
+    actions.push({
+      name: functionName,
+      params
+    });
+  }
+  return actions;
+}
+
+// src/actions.js
+async function executeAction(page, action) {
+  console.log(`Executing action: ${action.name}`, action.params);
+  const { name, params } = action;
+  const viewport = page.viewportSize();
+  const transformPoint = (pointStr) => {
+    if (!pointStr) return null;
+    const [x, y] = pointStr.split(" ").map(Number);
+    if (!viewport) return { x, y };
+    const actualX = x / 1e3 * viewport.width;
+    const actualY = y / 1e3 * viewport.height;
+    return { x: actualX, y: actualY };
+  };
+  try {
+    switch (name) {
+      case "click":
+        if (params.point) {
+          const { x, y } = transformPoint(params.point);
+          console.log(`Executing click action: (${x}, ${y}) [Original: ${params.point}]`);
+          await page.mouse.click(x, y);
+        }
+        break;
+      case "left_double":
+        if (params.point) {
+          const { x, y } = transformPoint(params.point);
+          await page.mouse.dblclick(x, y);
+        }
+        break;
+      case "left_triple":
+        if (params.point) {
+          const { x, y } = transformPoint(params.point);
+          await page.mouse.click(x, y, { clickCount: 3 });
+        }
+        break;
+      case "right_single":
+        if (params.point) {
+          const { x, y } = transformPoint(params.point);
+          await page.mouse.click(x, y, { button: "right" });
+        }
+        break;
+      case "drag":
+        if (params.start_point && params.end_point) {
+          const start = transformPoint(params.start_point);
+          const end = transformPoint(params.end_point);
+          await page.mouse.move(start.x, start.y);
+          await page.mouse.down();
+          await page.mouse.move(end.x, end.y);
+          await page.mouse.up();
+        }
+        break;
+      case "move_to":
+        if (params.point) {
+          const { x, y } = transformPoint(params.point);
+          await page.mouse.move(x, y);
+        }
+        break;
+      case "mouse_down":
+        if (params.point) {
+          const { x, y } = transformPoint(params.point);
+          await page.mouse.move(x, y);
+          await page.mouse.down({ button: params.button || "left" });
+        } else {
+          await page.mouse.down({ button: params.button || "left" });
+        }
+        break;
+      case "mouse_up":
+        if (params.point) {
+          const { x, y } = transformPoint(params.point);
+          await page.mouse.move(x, y);
+          await page.mouse.up({ button: params.button || "left" });
+        } else {
+          await page.mouse.up({ button: params.button || "left" });
+        }
+        break;
+      case "type":
+        if (params.content) {
+          const content = params.content;
+          await page.keyboard.type(content);
+        }
+        break;
+      case "hotkey":
+        if (params.key) {
+          const keys = params.key.split(" ").join("+");
+          await page.keyboard.press(keys);
+        }
+        break;
+      case "press":
+        if (params.key) {
+          await page.keyboard.press(params.key);
+        }
+        break;
+      case "scroll":
+        const direction = params.direction;
+        if (direction === "down") {
+          await page.evaluate(() => window.scrollBy(0, 500));
+        } else if (direction === "up") {
+          await page.evaluate(() => window.scrollBy(0, -500));
+        }
+        break;
+      case "wait":
+        if (params.time) {
+          const ms = parseFloat(params.time) * 1e3;
+          await page.waitForTimeout(ms);
+        }
+        break;
+      case "finished":
+        console.log("AI Task Finished:", params.content);
+        return "FINISHED";
+      case "call_user":
+        console.log("AI requests user input:", params.content);
+        return "FINISHED";
+      default:
+        console.warn(`Unknown action: ${name}`);
+    }
+  } catch (e) {
+    console.error(`Failed to execute action ${name}:`, e);
+  }
+  return "CONTINUE";
+}
+
+// src/runner.js
+async function runAgent({ targetUrl, taskInput, sessionId, simulatedUserKnownInfo }) {
+  console.log(`[Agent] Starting task: ${taskInput} on ${targetUrl} (Session: ${sessionId})`);
+  const screenshotsDir = path2.join(process.cwd(), "screenshots");
+  if (!fs.existsSync(screenshotsDir)) {
+    fs.mkdirSync(screenshotsDir, { recursive: true });
+    console.log(`\u{1F4C1} \u521B\u5EFA\u622A\u56FE\u76EE\u5F55: ${screenshotsDir}`);
+  }
+  const browser = await chromium.launch({
+    headless: true,
+    // Visible for demo/debug
+    args: ["--start-maximized"]
+    // Attempt to maximize
+  });
+  const context = await browser.newContext({
+    viewport: { width: 430, height: 800 }
+    // Set a reasonable fixed viewport
+  });
+  const page = await context.newPage();
+  let conversationHistory = [];
+  try {
+    let finalUrl = targetUrl;
+    if (!finalUrl.startsWith("http://") && !finalUrl.startsWith("https://")) {
+      finalUrl = "http://" + finalUrl;
+    }
+    const urlObj = new URL(finalUrl);
+    urlObj.searchParams.append("sessionId", sessionId);
+    finalUrl = urlObj.toString();
+    console.log(`Navigating to ${finalUrl}...`);
+    await page.goto(finalUrl, { waitUntil: "load" });
+    await page.waitForTimeout(2e3);
+    let isRunning = true;
+    let loopCount = 0;
+    const MAX_LOOPS = 20;
+    while (isRunning && loopCount < MAX_LOOPS) {
+      loopCount++;
+      console.log(`
+--- Cycle ${loopCount} ---`);
+      const screenshotBuffer = await page.screenshot({ format: "png" });
+      const screenshotBase64 = screenshotBuffer.toString("base64");
+      const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+      const screenshotPath = path2.join(screenshotsDir, `screenshot_${sessionId}_${loopCount}_${timestamp}.png`);
+      fs.writeFileSync(screenshotPath, screenshotBuffer);
+      console.log(`\u{1F4F8} \u622A\u56FE\u5DF2\u4FDD\u5B58: ${screenshotPath}`);
+      const viewport = page.viewportSize();
+      console.log(`\u{1F4D0} Viewport\u5C3A\u5BF8: ${viewport.width}x${viewport.height}`);
+      console.log(`\u{1F4E6} \u622A\u56FE\u5927\u5C0F: ${screenshotBuffer.length} bytes`);
+      let promptText = `Task: ${taskInput}
+`;
+      if (simulatedUserKnownInfo) {
+        promptText += `
+Known User Info: ${JSON.stringify(simulatedUserKnownInfo, null, 2)}
+`;
+      }
+      promptText += `
+Please perform the next action based on the screenshot.`;
+      const userContent = [
+        { type: "text", text: promptText },
+        { type: "image", image: screenshotBase64 }
+      ];
+      conversationHistory.push({
+        role: "user",
+        content: userContent
+      });
+      console.log("Querying AI...");
+      const aiContent = await queryAI(conversationHistory);
+      if (!aiContent) {
+        console.error("Invalid AI response. Retrying...");
+        await page.waitForTimeout(2e3);
+        continue;
+      }
+      conversationHistory.push({
+        role: "assistant",
+        content: aiContent
+      });
+      console.log("Raw AI Content (Excerpt):", aiContent);
+      const actions = parseActions(aiContent);
+      if (actions.length === 0) {
+        console.log("No actions parsed from AI response.");
+      }
+      console.log("Actions:", actions);
+      for (const action of actions) {
+        const result = await executeAction(page, action);
+        console.log("Action result:", result);
+        if (result === "FINISHED") {
+          isRunning = false;
+          break;
+        }
+      }
+      if (isRunning) {
+        await page.waitForTimeout(3e3);
+      }
+    }
+    return { status: "success", message: "Task completed", messages: conversationHistory };
+  } catch (error) {
+    console.error("Runtime Custom Error:", error);
+    return { status: "error", message: error.message };
+  } finally {
+    console.log("Closing browser...");
+    await browser.close();
+  }
+}
+
+// src/index.js
+import AdmZip from "adm-zip";
+import dotenv2 from "dotenv";
+dotenv2.config();
+function getFormattedDate() {
+  const now = /* @__PURE__ */ new Date();
+  const pad = (n) => String(n).padStart(2, "0");
+  return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+}
+async function runTaskLoop(targetWeb, targetServer, taskId) {
+  const results = [];
+  const datetime = getFormattedDate();
+  const folderName = `${taskId}_${datetime}`;
+  const outputDir = path3.join(process.cwd(), folderName);
+  if (!fs2.existsSync(outputDir)) {
+    fs2.mkdirSync(outputDir, { recursive: true });
+    console.log(`Created output directory: ${outputDir}`);
+  }
+  if (!targetWeb.startsWith("http")) targetWeb = "http://" + targetWeb;
+  if (!targetServer.startsWith("http")) targetServer = "http://" + targetServer;
+  console.log(`Starting Task Loop for ${taskId}`);
+  for (let i = 1; i <= 2; i++) {
+    console.log(`
+=== Starting Iteration ${i}/5 ===`);
+    try {
+      const data = await executeSingleCycle(targetWeb, targetServer, taskId, i);
+      const { sessionId, agentMessages, ...rest } = data;
+      const sessionDir = path3.join(outputDir, sessionId);
+      if (!fs2.existsSync(sessionDir)) {
+        fs2.mkdirSync(sessionDir, { recursive: true });
+      }
+      fs2.writeFileSync(
+        path3.join(sessionDir, "messages.json"),
+        JSON.stringify(agentMessages, null, 2)
+      );
+      fs2.writeFileSync(
+        path3.join(sessionDir, "result.json"),
+        JSON.stringify(rest, null, 2)
+      );
+      results.push({ sessionId, ...rest });
+      console.log(`Iteration ${i} completed. Saved to ${sessionDir}`);
+    } catch (e) {
+      console.error(`Iteration ${i} failed:`, e.stack || e.message);
+      results.push({ iteration: i, error: e.message });
+    }
+  }
+  const finalResultPath = path3.join(outputDir, "final-results.json");
+  fs2.writeFileSync(finalResultPath, JSON.stringify(results, null, 2));
+  console.log(`Saved consolidated results to ${finalResultPath}`);
+  console.log("Packaging results into ZIP...");
+  const zipName = `${folderName}.zip`;
+  const zipPath = path3.join(process.cwd(), zipName);
+  try {
+    const zip = new AdmZip();
+    zip.addLocalFolder(outputDir, folderName);
+    zip.writeZip(zipPath);
+    console.log(`
+\u2705 All done! Zip saved to: ${zipPath}`);
+  } catch (zipError) {
+    console.error("Failed to create zip:", zipError);
+  }
+  return zipPath;
+}
+async function executeSingleCycle(targetWeb, targetServer, taskId, iteration) {
+  const fetchJson = async (url, opts) => {
+    const res = await fetch(url, opts);
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`API ${url} failed: ${res.status} ${text}`);
+    }
+    return await res.json();
+  };
+  console.log("Calling Start API...");
+  const startUrl = `${targetServer}/api/tasks/${taskId}/start`;
+  const startData = await fetchJson(startUrl, { method: "POST" });
+  if (startData.code !== 200) {
+    throw new Error(`Start API failed: ${JSON.stringify(startData)}`);
+  }
+  const { session_id: sessionId, TaskJson } = startData.data;
+  console.log(`Setup for session ${sessionId}...`);
+  const setupUrl = `${targetServer}/api/tasks/setup`;
+  await fetchJson(setupUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ task_id: taskId, session_id: sessionId })
+  });
+  console.log("Running Agent...");
+  const agentResult = await runAgent({
+    taskInput: TaskJson.task.instruction,
+    targetUrl: targetWeb,
+    sessionId,
+    simulatedUserKnownInfo: TaskJson.task.simulated_user_known_info
+  });
+  console.log("Verifying result...");
+  const verifyUrl = `${targetServer}/api/tasks/verify`;
+  const verifyData = await fetchJson(verifyUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ task_id: taskId, session_id: sessionId })
+  });
+  return {
+    iteration,
+    sessionId,
+    agentStatus: agentResult.status,
+    agentMessages: agentResult.messages || [],
+    verifyResult: verifyData.data
+  };
+}
+export {
+  runAgent,
+  runTaskLoop
+};

package/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "rl-simulator-core",
+  "version": "1.0.0",
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "tsup src/index.js --format cjs,esm --dts --clean"
+  },
+  "dependencies": {
+    "@ai-sdk/openai": "^3.0.14",
+    "adm-zip": "^0.5.10",
+    "ai": "^6.0.44",
+    "dotenv": "^17.2.3",
+    "playwright": "^1.57.0"
+  },
+  "devDependencies": {
+    "tsup": "^8.0.0",
+    "typescript": "^5.9.3"
+  }
+}

package/src/actions.js

@@ -0,0 +1,145 @@
+
+async function executeAction(page, action) {
+    console.log(`Executing action: ${action.name}`, action.params);
+    
+    const { name, params } = action;
+    const viewport = page.viewportSize();
+
+    // Helper to transform 0-1000 coordinates to actual viewport size
+    const transformPoint = (pointStr) => {
+        if (!pointStr) return null;
+        const [x, y] = pointStr.split(' ').map(Number);
+        
+        if (!viewport) return { x, y };
+
+        const actualX = (x / 1000) * viewport.width;
+        const actualY = (y / 1000) * viewport.height;
+        return { x: actualX, y: actualY };
+    };
+
+    try {
+        switch (name) {
+            case 'click':
+                if (params.point) {
+                    const { x, y } = transformPoint(params.point);
+                    console.log(`Executing click action: (${x}, ${y}) [Original: ${params.point}]`);
+                    await page.mouse.click(x, y);
+                }
+                break;
+                
+            case 'left_double':
+                if (params.point) {
+                    const { x, y } = transformPoint(params.point);
+                    await page.mouse.dblclick(x, y);
+                }
+                break;
+            
+            case 'left_triple':
+                 if (params.point) {
+                    const { x, y } = transformPoint(params.point);
+                    await page.mouse.click(x, y, { clickCount: 3 });
+                }
+                break;
+
+            case 'right_single':
+                if (params.point) {
+                    const { x, y } = transformPoint(params.point);
+                    await page.mouse.click(x, y, { button: 'right' });
+                }
+                break;
+
+            case 'drag':
+                if (params.start_point && params.end_point) {
+                    const start = transformPoint(params.start_point);
+                    const end = transformPoint(params.end_point);
+                    await page.mouse.move(start.x, start.y);
+                    await page.mouse.down();
+                    await page.mouse.move(end.x, end.y);
+                    await page.mouse.up();
+                }
+                break;
+
+            case 'move_to':
+                if (params.point) {
+                     const { x, y } = transformPoint(params.point);
+                     await page.mouse.move(x, y);
+                }
+                break;
+
+            case 'mouse_down':
+                 if (params.point) {
+                    const { x, y } = transformPoint(params.point);
+                    await page.mouse.move(x, y);
+                    await page.mouse.down({ button: params.button || 'left' });
+                 } else {
+                    await page.mouse.down({ button: params.button || 'left' });
+                 }
+                 break;
+                 
+            case 'mouse_up':
+                 if (params.point) {
+                    const { x, y } = transformPoint(params.point);
+                    await page.mouse.move(x, y);
+                    await page.mouse.up({ button: params.button || 'left' });
+                 } else {
+                     await page.mouse.up({ button: params.button || 'left' });
+                 }
+                 break;
+
+            case 'type':
+                if (params.content) {
+                    const content = params.content;
+                    await page.keyboard.type(content);
+                }
+                break;
+            
+            case 'hotkey':
+                if (params.key) {
+                    const keys = params.key.split(' ').join('+');
+                    await page.keyboard.press(keys);
+                }
+                break;
+                
+            case 'press':
+                if (params.key) {
+                    await page.keyboard.press(params.key);
+                }
+                break;
+                
+            case 'scroll':
+                const direction = params.direction;
+                if (direction === 'down') {
+                    await page.evaluate(() => window.scrollBy(0, 500));
+                } else if (direction === 'up') {
+                    await page.evaluate(() => window.scrollBy(0, -500));
+                }
+                break;
+                
+            case 'wait':
+                if (params.time) {
+                    const ms = parseFloat(params.time) * 1000;
+                    await page.waitForTimeout(ms);
+                }
+                break;
+
+            case 'finished':
+                console.log("AI Task Finished:", params.content);
+                return 'FINISHED';
+            
+            case 'call_user':
+                console.log("AI requests user input:", params.content);
+                 return 'FINISHED';
+
+            default:
+                console.warn(`Unknown action: ${name}`);
+        }
+    } catch (e) {
+        console.error(`Failed to execute action ${name}:`, e);
+    }
+    
+    return 'CONTINUE';
+}
+
+export {
+    executeAction
+};