rl-simulator-core 1.0.0

package/src/ai.js

@@ -0,0 +1,86 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+import { API_KEY, ENDPOINT, BASE_URL, SYSTEM_PROMPT } from './config.js';
+import fs from 'fs';
+import path from 'path';
+
+const logFile = path.join(process.cwd(), 'ai_traffic.log');
+
+const log = (message) => {
+    const timestamp = new Date().toISOString();
+    const logEntry = `[${timestamp}] ${message}\n`;
+    fs.appendFileSync(logFile, logEntry);
+};
+
+// Custom Fetch for Logging (Kept logic, same structure)
+const customFetch = async (url, options) => {
+    // ... (same logging logic as before) ...
+    // Simplified for brevity in rewrite, but keeping the core idea if needed.
+    return fetch(url, options);
+};
+
+const openai = createOpenAI({
+  baseURL: BASE_URL,
+  apiKey: API_KEY,
+  compatibility: 'compatible', 
+});
+
+async function queryAI(messages) {
+    console.log("Querying AI with history length:", messages.length);
+    
+    try {
+        console.log("⏳ 发送请求中...");
+        const startTime = Date.now();
+
+        const { text } = await generateText({
+            model: openai.chat(ENDPOINT), 
+            system: SYSTEM_PROMPT,
+            messages: messages, 
+            temperature: 0.1,
+            topP: 0.7,
+        });
+
+        const endTime = Date.now();
+        console.log("✅ 请求成功! 耗时:", (endTime - startTime) / 1000, "秒");
+        console.log("📥 响应内容:", text.substring(0, 500) + "...");
+        
+        return text;
+
+    } catch (error) {
+        console.error("❌ AI 请求失败!", error);
+        return null; 
+    }
+}
+
+function parseActions(aiContent) {
+    const actions = [];
+    const functionRegex = /<function[^>]*=([^>]+)>([\s\S]*?)<\/function[^>]*>/g;
+    
+    let functionMatch;
+    while ((functionMatch = functionRegex.exec(aiContent)) !== null) {
+        const functionName = functionMatch[1];
+        const functionBody = functionMatch[2];
+        const params = {};
+
+        const paramRegex = /<parameter[^>]*=([^>]+)>([\s\S]*?)<\/parameter[^>]*>/g;
+        let paramMatch;
+        while ((paramMatch = paramRegex.exec(functionBody)) !== null) {
+            const paramName = paramMatch[1];
+            let paramValue = paramMatch[2].trim();
+            const cleanedValue = paramValue.replace(/<[^>]+>/g, '').trim();
+            params[paramName] = cleanedValue;
+        }
+
+        actions.push({
+            name: functionName,
+            params: params
+        });
+    }
+
+    return actions;
+}
+
+export {
+    queryAI,
+    parseActions
+};

package/src/config.js

@@ -0,0 +1,83 @@
+import dotenv from 'dotenv';
+dotenv.config();
+
+const API_KEY = process.env.ARK_API_KEY;
+const ENDPOINT = process.env.EP;
+const BASE_URL = process.env.BASE_URL;
+
+if (!API_KEY || !ENDPOINT || !BASE_URL) {
+    console.warn("Missing environment variables (ARK_API_KEY, EP, BASE_URL). Agent may fail if not provided later.");
+}
+
+const SYSTEM_PROMPT = `
+Intro to Think Mode
+To explore the performance of our model with different styles of thinking, we introduce four modes of thinking: budgted, efficient, no think, and unrestricted. The model is default to unrestricted think mode.
+- Budgeted think mode: Limit model's internal reasoning and reflections to stay within the specified token budget.
+- Efficient think mode: Model provides a concise internal thinking process with efficient reasoning and reflections. It doesn't have a strict token budget, but should be less verbose and more direct in your thinking.
+- No think mode: Model responds directly to the question without any internal reasoning process or extra thinking tokens. Still follows the template with the minimum required thinking tokens to justify the answer.
+- Unrestricted think mode: Model engages in an internal thinking process with thorough reasoning and reflections. There's an unlimited budget for thinking tokens and can continue thinking until you fully solve the problem.
+Think Mode Assignment
+As the "think mode" sp is not encapsulated yet, you need to manually sew in different sp for different modes of thinking. Here are the sp's: 
+NOTE: You should disable "thinking" first: (I know it's a bit counterintuitive.)
+"thinking": {
+"type": "disabled"
+}
+- Unrestricted think mode
+You should begin by detailing the internal reasoning process, and then present the answer to the user. The reasoning process should be enclosed within <think_never_used_51bce0c785ca2f68081bfa7d91973934> and </think_never_used_51bce0c785ca2f68081bfa7d91973934> tags, as follows:
+<think_never_used_51bce0c785ca2f68081bfa7d91973934> reasoning process here </think_never_used_51bce0c785ca2f68081bfa7d91973934> answer here.
+You have different modes of thinking:
+Unrestricted think mode: Engage in an internal thinking process with thorough reasoning and reflections. You have an unlimited budget for thinking tokens and can continue thinking until you fully solve the problem.
+Efficient think mode: Provide a concise internal thinking process with efficient reasoning and reflections. You don't have a strict token budget but be less verbose and more direct in your thinking.
+No think mode: Respond directly to the question without any internal reasoning process or extra thinking tokens. Still follow the template with the minimum required thinking tokens to justify the answer.
+Budgeted think mode: Limit your internal reasoning and reflections to stay within the specified token budget
+Based on the complexity of the problem, select the appropriate mode for reasoning among the provided options listed below.
+Provided Mode(s):
+Unrestricted think
+Action Space
+"""
+## Function Definition
+- You have access to the following functions:
+{"type": "function", "name": "click", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse left single click action."}
+{"type": "function", "name": "left_double", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse left double click action."}
+{"type": "function", "name": "left_triple", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse left triple click action."}
+{"type": "function", "name": "right_single", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse right single click action."}
+{"type": "function", "name": "drag", "parameters": {"type": "object", "properties": {"start_point": {"type": "string", "description": "Drag start point. The format is: x y"}, "end_point": {"type": "string", "description": "Drag end point. The format is: x y"}}, "required": ["start_point", "end_point"]}, "description": "Mouse left button drag action."}
+{"type": "function", "name": "scroll", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Scroll start position. If not specified, default to execute on the current mouse position. The format is: x y"}, "direction": {"type": "string", "description": "Scroll direction.", "enum": ["up", "down", "left", "right"]}}, "required": ["direction"]}, "description": "Scroll action."}
+{"type": "function", "name": "move_to", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Target coordinates. The format is: x y"}}, "required": ["point"]}, "description": "Mouse move action."}
+{"type": "function", "name": "mouse_down", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Mouse down position. If not specified, default to execute on the current mouse position. The format is: x y"}, "button": {"type": "string", "description": "Down button. Default to left.", "enum": ["left", "right"]}}, "required": []}, "description": "Mouse down action."}
+{"type": "function", "name": "mouse_up", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Mouse up position. If not specified, default to execute on the current mouse position. The format is: x y"}, "button": {"type": "string", "description": "Up button. Default to left.", "enum": ["left", "right"]}}, "required": []}, "description": "Mouse up action."}
+{"type": "function", "name": "type", "parameters": {"type": "object", "properties": {"content": {"type": "string", "description": "Type content. If you want to submit your input, use \\n at the end of content."}}, "required": ["content"]}, "description": "Type content."}
+{"type": "function", "name": "hotkey", "parameters": {"type": "object", "properties": {"key": {"type": "string", "description": "Hotkeys you want to press. Split keys with a space and use lowercase."}}, "required": ["key"]}, "description": "Press hotkey."}
+{"type": "function", "name": "press", "parameters": {"type": "object", "properties": {"key": {"type": "string", "description": "Key you want to press. Only one key can be pressed at one time."}}, "required": ["key"]}, "description": "Press key."}
+{"type": "function", "name": "call_user", "parameters": {"type": "object", "properties": {"content": {"type": "string", "description": "Message or information displayed to the user to request their input, feedback, or guidance."}}, "required": []}, "description": "This function is used to interact with the user by displaying a message and requesting their input, feedback, or guidance."}
+{"type": "function", "name": "wait", "parameters": {"type": "object", "properties": {"time": {"type": "string", "description": "Wait time in seconds."}}, "required": []}, "description": "Wait for a while."}
+{"type": "function", "name": "finished", "parameters": {"type": "object", "properties": {"content": {"type": "string", "description": "Provide the final answer or response to complete the task."}}, "required": []}, "description": "This function is used to indicate the completion of a task by providing the final answer or response."}
+
+- To call a function, use the following structure without any suffix:
+
+<gui_think> reasoning process </gui_think>
+<seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934><function_never_used_51bce0c785ca2f68081bfa7d91973934=example_function_name><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_1>value_1</parameter_never_used_51bce0c785ca2f68081bfa7d91973934><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_2>
+This is the value for the second parameter
+that can span
+multiple lines
+</parameter_never_used_51bce0c785ca2f68081bfa7d91973934></function_never_used_51bce0c785ca2f68081bfa7d91973934></seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934>
+
+## Important Notes
+- Function calls must begin with <function_never_used_51bce0c785ca2f68081bfa7d91973934= and end with </function_never_used_51bce0c785ca2f68081bfa7d91973934>.
+- All required parameters must be explicitly provided.
+
+## Additional Notes
+- You can execute multiple actions within a single tool call. For example:
+<seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934><function_never_used_51bce0c785ca2f68081bfa7d91973934=example_function_1><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_1>value_1</parameter_never_used_51bce0c785ca2f68081bfa7d91973934><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_2>
+This is the value for the second parameter
+that can span
+multiple lines
+</parameter_never_used_51bce0c785ca2f68081bfa7d91973934></function_never_used_51bce0c785ca2f68081bfa7d91973934><function_never_used_51bce0c785ca2f68081bfa7d91973934=example_function_2><parameter_never_used_51bce0c785ca2f68081bfa7d91973934=example_parameter_3>value_4</parameter_never_used_51bce0c785ca2f68081bfa7d91973934></function_never_used_51bce0c785ca2f68081bfa7d91973934></seed:tool_call_never_used_51bce0c785ca2f68081bfa7d91973934>
+"""`;
+
+export {
+  API_KEY,
+  ENDPOINT,
+  BASE_URL,
+  SYSTEM_PROMPT
+};

package/src/index.js

@@ -0,0 +1,155 @@
+
+import fs from 'fs';
+import path from 'path';
+import { runAgent } from './runner.js';
+import AdmZip from 'adm-zip';
+import dotenv from 'dotenv';
+
+export { runAgent };
+
+// Load env vars
+dotenv.config();
+
+/**
+ * Format date to YYYYMMDD_HHMMSS
+ */
+function getFormattedDate() {
+    const now = new Date();
+    const pad = (n) => String(n).padStart(2, '0');
+    return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+}
+
+/**
+ * Execute the task flow 5 times and save results in optimized structure.
+ * @param {string} targetWeb - The URL of the web app to test
+ * @param {string} targetServer - The URL of the backend task service
+ * @param {string} taskId - The ID of the task to run
+ */
+export async function runTaskLoop(targetWeb, targetServer, taskId) {
+    const results = [];
+    const datetime = getFormattedDate();
+    const folderName = `${taskId}_${datetime}`;
+    const outputDir = path.join(process.cwd(), folderName);
+    
+    // Create root output directory
+    if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+        console.log(`Created output directory: ${outputDir}`);
+    }
+
+    // Ensure URLs have protocols
+    if (!targetWeb.startsWith('http')) targetWeb = 'http://' + targetWeb;
+    if (!targetServer.startsWith('http')) targetServer = 'http://' + targetServer;
+
+    console.log(`Starting Task Loop for ${taskId}`);
+
+    for (let i = 1; i <= 2; i++) {
+        console.log(`\n=== Starting Iteration ${i}/5 ===`);
+        try {
+            const data = await executeSingleCycle(targetWeb, targetServer, taskId, i);
+            const { sessionId, agentMessages, ...rest } = data;
+            
+            // Create session subdirectory
+            const sessionDir = path.join(outputDir, sessionId);
+            if (!fs.existsSync(sessionDir)) {
+                fs.mkdirSync(sessionDir, { recursive: true });
+            }
+
+            // Save messages.json
+            fs.writeFileSync(
+                path.join(sessionDir, 'messages.json'), 
+                JSON.stringify(agentMessages, null, 2)
+            );
+
+            // Save result.json
+            fs.writeFileSync(
+                path.join(sessionDir, 'result.json'), 
+                JSON.stringify(rest, null, 2)
+            );
+
+            results.push({ sessionId, ...rest });
+            console.log(`Iteration ${i} completed. Saved to ${sessionDir}`);
+
+        } catch (e) {
+            console.error(`Iteration ${i} failed:`, e.stack || e.message);
+            results.push({ iteration: i, error: e.message });
+        }
+    }
+
+    // Save final-results.json (Consolidated)
+    const finalResultPath = path.join(outputDir, 'final-results.json');
+    fs.writeFileSync(finalResultPath, JSON.stringify(results, null, 2));
+    console.log(`Saved consolidated results to ${finalResultPath}`);
+
+    // Zip the results
+    console.log("Packaging results into ZIP...");
+    const zipName = `${folderName}.zip`;
+    const zipPath = path.join(process.cwd(), zipName);
+    
+    try {
+        const zip = new AdmZip();
+        zip.addLocalFolder(outputDir, folderName);
+        zip.writeZip(zipPath);
+        console.log(`\n✅ All done! Zip saved to: ${zipPath}`);
+    } catch (zipError) {
+        console.error("Failed to create zip:", zipError);
+    }
+    
+    return zipPath;
+}
+
+async function executeSingleCycle(targetWeb, targetServer, taskId, iteration) {
+    const fetchJson = async (url, opts) => {
+        const res = await fetch(url, opts);
+        if (!res.ok) {
+            const text = await res.text();
+            throw new Error(`API ${url} failed: ${res.status} ${text}`);
+        }
+        return await res.json();
+    };
+
+    // 1. Start Task
+    console.log("Calling Start API...");
+    const startUrl = `${targetServer}/api/tasks/${taskId}/start`;
+    const startData = await fetchJson(startUrl, { method: 'POST' });
+    
+    if (startData.code !== 200) {
+        throw new Error(`Start API failed: ${JSON.stringify(startData)}`);
+    }
+    const { session_id: sessionId, TaskJson } = startData.data;
+
+    // 2. Setup Task
+    console.log(`Setup for session ${sessionId}...`);
+    const setupUrl = `${targetServer}/api/tasks/setup`;
+    await fetchJson(setupUrl, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ task_id: taskId, session_id: sessionId })
+    });
+
+    // 3. Run Agent
+    console.log("Running Agent...");
+    const agentResult = await runAgent({
+        taskInput: TaskJson.task.instruction,
+        targetUrl: targetWeb,
+        sessionId: sessionId,
+        simulatedUserKnownInfo: TaskJson.task.simulated_user_known_info
+    });
+
+    // 4. Verify Task
+    console.log("Verifying result...");
+    const verifyUrl = `${targetServer}/api/tasks/verify`;
+    const verifyData = await fetchJson(verifyUrl, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ task_id: taskId, session_id: sessionId })
+    });
+
+    return {
+        iteration,
+        sessionId,
+        agentStatus: agentResult.status,
+        agentMessages: agentResult.messages || [],
+        verifyResult: verifyData.data
+    };
+}

package/src/runner.js

@@ -0,0 +1,138 @@
+import { chromium } from 'playwright';
+import fs from 'fs';
+import path from 'path';
+import { queryAI, parseActions } from './ai.js';
+import { executeAction } from './actions.js';
+
+async function runAgent({ targetUrl, taskInput, sessionId, simulatedUserKnownInfo }) {
+    console.log(`[Agent] Starting task: ${taskInput} on ${targetUrl} (Session: ${sessionId})`);
+
+    const screenshotsDir = path.join(process.cwd(), 'screenshots');
+    if (!fs.existsSync(screenshotsDir)) {
+        fs.mkdirSync(screenshotsDir, { recursive: true });
+        console.log(`📁 创建截图目录: ${screenshotsDir}`);
+    }
+
+    // 2. Launch Browser
+    const browser = await chromium.launch({ 
+        headless: true, // Visible for demo/debug
+        args: ['--start-maximized'] // Attempt to maximize
+    });
+
+    const context = await browser.newContext({
+        viewport: { width: 430, height: 800 } // Set a reasonable fixed viewport
+    });
+    const page = await context.newPage();
+
+    let conversationHistory = [];
+
+    try {
+        // Ensure protocol exists
+        let finalUrl = targetUrl;
+        if (!finalUrl.startsWith('http://') && !finalUrl.startsWith('https://')) {
+            finalUrl = 'http://' + finalUrl;
+        }
+
+        // Append sessionId
+        const urlObj = new URL(finalUrl);
+        urlObj.searchParams.append('sessionId', sessionId);
+        finalUrl = urlObj.toString();
+
+        console.log(`Navigating to ${finalUrl}...`);
+        await page.goto(finalUrl, { waitUntil: 'load' });
+        await page.waitForTimeout(2000); // Wait for initial render
+
+        let isRunning = true;
+        let loopCount = 0;
+        const MAX_LOOPS = 20; // Safety break
+
+        while (isRunning && loopCount < MAX_LOOPS) {
+            loopCount++;
+            console.log(`\n--- Cycle ${loopCount} ---`);
+
+            // 1. Screenshot
+            const screenshotBuffer = await page.screenshot({ format: 'png' });
+            const screenshotBase64 = screenshotBuffer.toString('base64');
+
+            // 保存截图到本地
+            const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+            const screenshotPath = path.join(screenshotsDir, `screenshot_${sessionId}_${loopCount}_${timestamp}.png`);
+            fs.writeFileSync(screenshotPath, screenshotBuffer);
+            console.log(`📸 截图已保存: ${screenshotPath}`);
+
+            // 获取并记录viewport尺寸
+            const viewport = page.viewportSize();
+            console.log(`📐 Viewport尺寸: ${viewport.width}x${viewport.height}`);
+            console.log(`📦 截图大小: ${screenshotBuffer.length} bytes`);
+
+            // 2. Prepare Message for AI
+            let promptText = `Task: ${taskInput}\n`;
+            if (simulatedUserKnownInfo) {
+                promptText += `\nKnown User Info: ${JSON.stringify(simulatedUserKnownInfo, null, 2)}\n`;
+            }
+            promptText += `\nPlease perform the next action based on the screenshot.`;
+
+            const userContent = [
+                { type: 'text', text: promptText },
+                { type: 'image', image: screenshotBase64 }
+            ];
+            
+            conversationHistory.push({
+                role: 'user',
+                content: userContent
+            });
+
+            // 3. Query AI
+            console.log("Querying AI...");
+            const aiContent = await queryAI(conversationHistory);
+            
+            if (!aiContent) {
+                console.error("Invalid AI response. Retrying...");
+                await page.waitForTimeout(2000);
+                continue;
+            }
+
+            // Add assistant response to history
+            conversationHistory.push({
+                role: 'assistant',
+                content: aiContent
+            });
+
+            console.log("Raw AI Content (Excerpt):", aiContent);
+
+            // 4. Parse Actions
+            const actions = parseActions(aiContent);
+            if (actions.length === 0) {
+                console.log("No actions parsed from AI response.");
+            }
+
+            console.log("Actions:", actions);
+            // 5. Execute Actions
+            for (const action of actions) {
+                const result = await executeAction(page, action);
+                console.log("Action result:", result);
+                if (result === 'FINISHED') {
+                    isRunning = false;
+                    break;
+                }
+            }
+            
+            // 6. Short wait between cycles
+            if (isRunning) {
+                await page.waitForTimeout(3000);
+            }
+        }
+        return { status: 'success', message: 'Task completed', messages: conversationHistory };
+
+    } catch (error) {
+        console.error("Runtime Custom Error:", error);
+        return { status: 'error', message: error.message };
+    } finally {
+        console.log("Closing browser...");
+        await browser.close();
+    }
+}
+
+export {
+    runAgent
+};