camel-ai 0.2.73a11__py3-none-any.whl → 0.2.74__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts

@@ -1,4 +1,4 @@
-import { Page, Browser, BrowserContext, chromium } from 'playwright';
+import { Page, Browser, BrowserContext, chromium, ConsoleMessage } from 'playwright';
 import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming } from './types';
 import { ConfigLoader, StealthConfig } from './config-loader';
 
@@ -6,18 +6,43 @@ export class HybridBrowserSession {
   private browser: Browser | null = null;
   private context: BrowserContext | null = null;
   private pages: Map<string, Page> = new Map();
+  private consoleLogs: Map<string, ConsoleMessage[]> = new Map();
   private currentTabId: string | null = null;
   private tabCounter = 0;
   private configLoader: ConfigLoader;
   private scrollPosition: { x: number; y: number } = {x: 0, y: 0};
   private hasNavigatedBefore = false; //  Track if we've navigated before
+  private logLimit: number; 
 
   constructor(config: BrowserToolkitConfig = {}) {
     // Use ConfigLoader's fromPythonConfig to handle conversion properly
     this.configLoader = ConfigLoader.fromPythonConfig(config);
+    // Load browser configuration for console log limit, default to 1000
+    this.logLimit = this.configLoader.getBrowserConfig().consoleLogLimit || 1000;
   }
 
-  async ensureBrowser(): Promise<void> {
+  private registerNewPage(tabId: string, page: Page): void {
+    // Register page and logs with tabId
+    this.pages.set(tabId, page);
+    this.consoleLogs.set(tabId, []);
+    // Set up console log listener for the page
+    page.on('console', (msg: ConsoleMessage) => {
+      const logs = this.consoleLogs.get(tabId);
+      if (logs) {
+        logs.push(msg);
+        if (logs.length >  this.logLimit) {
+          logs.shift();
+        }
+      }
+    });
+
+    // Clean logs on page close
+    page.on('close', () => {
+      this.consoleLogs.delete(tabId);
+    });
+  }
+
+  async ensureBrowser(): Promise<void> {  
     if (this.browser) {
       return;
     }
@@ -57,7 +82,7 @@ export class HybridBrowserSession {
           // In CDP mode, only consider pages with about:blank as available
           if (pageUrl === 'about:blank') {
             const tabId = this.generateTabId();
-            this.pages.set(tabId, page);
+            this.registerNewPage(tabId, page);
             if (!this.currentTabId) {
               this.currentTabId = tabId;
               availablePageFound = true;
@@ -97,7 +122,7 @@ export class HybridBrowserSession {
         const pages = this.context.pages();
         if (pages.length > 0) {
           const initialTabId = this.generateTabId();
-          this.pages.set(initialTabId, pages[0]);
+          this.registerNewPage(initialTabId, pages[0]);
           this.currentTabId = initialTabId;
         }
       } else {
@@ -115,7 +140,7 @@ export class HybridBrowserSession {
         
         const initialPage = await this.context.newPage();
         const initialTabId = this.generateTabId();
-        this.pages.set(initialTabId, initialPage);
+        this.registerNewPage(initialTabId, initialPage);
         this.currentTabId = initialTabId;
       }
     }
@@ -139,6 +164,13 @@ export class HybridBrowserSession {
     return this.pages.get(this.currentTabId)!;
   }
 
+  async getCurrentLogs(): Promise<ConsoleMessage[]> {
+    if (!this.currentTabId || !this.consoleLogs.has(this.currentTabId)) {
+      return [];
+    }
+    return this.consoleLogs.get(this.currentTabId) || [];
+  }
+
   /**
    * Get current scroll position from the page
    */
@@ -343,7 +375,7 @@ export class HybridBrowserSession {
           
           // Generate tab ID for the new page
           const newTabId = this.generateTabId();
-          this.pages.set(newTabId, newPage);
+          this.registerNewPage(newTabId, newPage);
           
           // Set up page properties
           const browserConfig = this.configLoader.getBrowserConfig();
@@ -434,7 +466,97 @@ export class HybridBrowserSession {
     }
   }
 
+  /**
+   *  Simplified mouse control implementation
+   */
+  private async performMouseControl(page: Page, control: string, x: number, y: number): Promise<{ success: boolean; error?: string }> {
+    try {
+      const viewport = page.viewportSize();
+      if (!viewport) {
+        return { success: false, error: 'Viewport size not available from page.' };
+      }
+      if (x < 0 || y < 0 || x > viewport.width || y > viewport.height) {
+        return { success: false, error: `Invalid coordinates, outside viewport bounds: (${x}, ${y})` };
+      }
+      switch (control) {
+        case 'click': {
+          await page.mouse.click(x, y);
+          break;
+        }
+        case 'right_click': {
+          await page.mouse.click(x, y, { button: 'right' });
+          break;
+        }
+        case 'dblclick': {
+          await page.mouse.dblclick(x, y);
+          break;
+        }
+        default:
+          return { success: false, error: `Invalid control action: ${control}` };
+      }
+      
+      return { success: true };
+    } catch (error) {
+      return { success: false, error: `Mouse action failed: ${error}` };
+    }
+  }
 
+  /**
+   *  Enhanced mouse drag and drop implementation using ref IDs
+   */
+  private async performMouseDrag(page: Page, fromRef: string, toRef: string): Promise<{ success: boolean; error?: string }> {
+    try {
+      // Ensure we have the latest snapshot
+      await (page as any)._snapshotForAI();
+      
+      // Get elements using Playwright's aria-ref selector
+      const fromSelector = `aria-ref=${fromRef}`;
+      const toSelector = `aria-ref=${toRef}`;
+      
+      const fromElement = await page.locator(fromSelector).first();
+      const toElement = await page.locator(toSelector).first();
+      
+      // Check if elements exist
+      const fromExists = await fromElement.count() > 0;
+      const toExists = await toElement.count() > 0;
+      
+      if (!fromExists) {
+        return { success: false, error: `Source element with ref ${fromRef} not found` };
+      }
+      
+      if (!toExists) {
+        return { success: false, error: `Target element with ref ${toRef} not found` };
+      }
+      
+      // Get the center coordinates of both elements
+      const fromBox = await fromElement.boundingBox();
+      const toBox = await toElement.boundingBox();
+      
+      if (!fromBox) {
+        return { success: false, error: `Could not get bounding box for source element with ref ${fromRef}` };
+      }
+      
+      if (!toBox) {
+        return { success: false, error: `Could not get bounding box for target element with ref ${toRef}` };
+      }
+      
+      const fromX = fromBox.x + fromBox.width / 2;
+      const fromY = fromBox.y + fromBox.height / 2;
+      const toX = toBox.x + toBox.width / 2;
+      const toY = toBox.y + toBox.height / 2;
+
+      // Perform the drag operation
+      await page.mouse.move(fromX, fromY);
+      await page.mouse.down();
+      // Destination coordinates
+      await page.mouse.move(toX, toY);
+      await page.mouse.up();
+
+      return { success: true };
+    } catch (error) {
+      return { success: false, error: `Mouse drag action failed: ${error}` };
+    }
+  }
 
   async executeAction(action: BrowserAction): Promise<ActionResult> {
     const startTime = Date.now();
@@ -519,6 +641,40 @@ export class HybridBrowserSession {
           actionExecutionTime = Date.now() - enterStart;
           break;
         }
+
+        case 'mouse_control': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const mouseControlStart = Date.now();
+          const mouseControlResult = await this.performMouseControl(page, action.control, action.x, action.y);
+
+          if (!mouseControlResult.success) {
+            throw new Error(`Action failed: ${mouseControlResult.error}`);
+          }
+          actionExecutionTime = Date.now() - mouseControlStart;
+          break;
+        }
+
+        case 'mouse_drag': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const mouseDragStart = Date.now();
+          const mouseDragResult = await this.performMouseDrag(page, action.from_ref, action.to_ref);
+
+          if (!mouseDragResult.success) {
+            throw new Error(`Action failed: ${mouseDragResult.error}`);
+          }
+          actionExecutionTime = Date.now() - mouseDragStart;
+          break;
+        }
+
+        case 'press_key': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const keyPressStart = Date.now();
+          // concatenate keys with '+' for key combinations
+          const keys = action.keys.join('+');
+          await page.keyboard.press(keys);
+          actionExecutionTime = Date.now() - keyPressStart;
+          break;
+        }
           
         default:
           throw new Error(`Unknown action type: ${(action as any).type}`);
@@ -651,7 +807,7 @@ export class HybridBrowserSession {
             if (!isTracked && pageUrl === 'about:blank') {
               newPage = page;
               newTabId = this.generateTabId();
-              this.pages.set(newTabId, newPage);
+              this.registerNewPage(newTabId, newPage);
               break;
             }
           }
@@ -663,7 +819,7 @@ export class HybridBrowserSession {
           // Non-CDP mode: create new page as usual
           newPage = await this.context.newPage();
           newTabId = this.generateTabId();
-          this.pages.set(newTabId, newPage);
+          this.registerNewPage(newTabId, newPage);
         }
         
         // Set up page properties

camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts

@@ -30,6 +30,7 @@ export interface BrowserConfig {
   // Tab management
   tabIdPrefix: string;
   tabCounterPadding: number;
+  consoleLogLimit: number;
   
   // Scroll and positioning
   scrollPositionScale: number;
@@ -113,6 +114,7 @@ function getDefaultBrowserConfig(): BrowserConfig {
     clickTimeout: 3000,
     tabIdPrefix: 'tab-',
     tabCounterPadding: 3,
+    consoleLogLimit: 1000,
     scrollPositionScale: 0.1,
     navigationDelay: 100,
     blankPageUrls: ['about:blank', ''],

camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts

@@ -1,6 +1,7 @@
 import {HybridBrowserSession} from './browser-session';
 import {ActionResult, BrowserAction, BrowserToolkitConfig, SnapshotResult, TabInfo, VisualMarkResult} from './types';
 import {ConfigLoader} from './config-loader';
+import {ConsoleMessage} from 'playwright';
 
 export class HybridBrowserToolkit {
   private session: HybridBrowserSession;
@@ -382,6 +383,21 @@ export class HybridBrowserToolkit {
     return this.executeActionWithSnapshot(action);
   }
 
+  async mouseControl(control: 'click' | 'right_click'| 'dblclick', x: number, y: number): Promise<any> {
+    const action: BrowserAction = { type: 'mouse_control', control, x, y };
+    return this.executeActionWithSnapshot(action);
+  }
+
+  async mouseDrag(from_ref: string, to_ref: string): Promise<any> {
+    const action: BrowserAction = { type: 'mouse_drag', from_ref, to_ref };
+    return this.executeActionWithSnapshot(action);
+  }
+
+  async pressKeys(keys: string[]): Promise<any> {
+    const action: BrowserAction = { type: 'press_key', keys};
+    return this.executeActionWithSnapshot(action);
+  }
+
   async back(): Promise<ActionResult> {
     const startTime = Date.now();
     
@@ -519,4 +535,93 @@ export class HybridBrowserToolkit {
     return await this.session.getTabInfo();
   }
 
-}
+  async getConsoleView(): Promise<any> {
+    const currentLogs = await this.session.getCurrentLogs();
+    // Format logs
+    return currentLogs.map(item => ({
+      type: item.type(),
+      text: item.text(),
+    }));
+  }
+
+  async consoleExecute(code: string): Promise<any> {
+    const startTime = Date.now();
+    try {
+      const page = await this.session.getCurrentPage();
+      
+      // Wrap the code to capture console.log output
+      const wrappedCode = `
+        (function() {
+          const _logs = [];
+          const originalLog = console.log;
+          console.log = function(...args) {
+            _logs.push(args.map(arg => {
+              try {
+                return typeof arg === 'object' ? JSON.stringify(arg) : String(arg);
+              } catch (e) {
+                return String(arg);
+              }
+            }).join(' '));
+            originalLog.apply(console, args);
+          };
+          
+          let result;
+          try {
+            result = eval(${JSON.stringify(code)});
+          } catch (e) {
+            try {
+              result = (function() { ${code} })();
+            } catch (error) {
+              console.log = originalLog;
+              throw error;
+            }
+          }
+          
+          console.log = originalLog;
+          return { result, logs: _logs };
+        })()
+      `;
+      
+      const evalResult = await page.evaluate(wrappedCode) as { result: any; logs: string[] };
+      const { result, logs } = evalResult;
+
+      const snapshotStart = Date.now();
+      const snapshot = await this.getPageSnapshot(this.viewportLimit);
+      const snapshotTime = Date.now() - snapshotStart;
+      const totalTime = Date.now() - startTime;
+
+      // Properly serialize the result
+      let resultStr: string;
+      try {
+        resultStr = JSON.stringify(result, null, 2);
+      } catch (e) {
+        // Fallback for non-serializable values
+        resultStr = String(result);
+      }
+
+      return {
+        result: `Console execution result: ${resultStr}`,
+        console_output: logs,
+        snapshot: snapshot,
+        timing: {
+          total_time_ms: totalTime,
+          snapshot_time_ms: snapshotTime,
+        },
+      };
+      
+    } catch (error) {
+      const totalTime = Date.now() - startTime;
+      return {
+        result: `Console execution failed: ${error}`,
+        console_output: [],
+        snapshot: '',
+        timing: {
+          total_time_ms: totalTime,
+          snapshot_time_ms: 0,
+        },
+      };
+    }
+  }
+
+}
+

camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts

@@ -101,7 +101,25 @@ export interface EnterAction {
   type: 'enter';
 }
 
-export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction;
+export interface MouseAction {
+  type: 'mouse_control';
+  control: 'click' | 'right_click' | 'dblclick';
+  x: number; 
+  y: number; 
+}
+
+export interface MouseDragAction {
+  type: 'mouse_drag';
+  from_ref: string;
+  to_ref: string;
+}
+
+export interface PressKeyAction {
+  type: 'press_key';
+  keys: string[];
+}
+
+export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction | MouseAction | MouseDragAction | PressKeyAction;
 
 export interface VisualMarkResult {
   text: string;

camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js

@@ -173,6 +173,18 @@ class WebSocketBrowserServer {
       case 'enter':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
         return await this.toolkit.enter();
+      
+      case 'mouse_control':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.mouseControl(params.control, params.x, params.y);
+
+      case 'mouse_drag':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.mouseDrag(params.from_ref, params.to_ref);
+
+      case 'press_key':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.pressKeys(params.keys);
 
       case 'back':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
@@ -194,6 +206,14 @@ class WebSocketBrowserServer {
         if (!this.toolkit) throw new Error('Toolkit not initialized');
         return await this.toolkit.getTabInfo();
 
+      case 'console_view':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.getConsoleView();
+
+      case 'console_exec':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.consoleExecute(params.code);
+
       case 'wait_user':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
         return await this.toolkit.waitUser(params.timeout);

camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py

@@ -537,6 +537,31 @@ class WebSocketBrowserWrapper:
         response = await self._send_command('enter', {})
         return response
 
+    @action_logger
+    async def mouse_control(
+        self, control: str, x: float, y: float
+    ) -> Dict[str, Any]:
+        """Control the mouse to interact with browser with x, y coordinates."""
+        response = await self._send_command(
+            'mouse_control', {'control': control, 'x': x, 'y': y}
+        )
+        return response
+
+    @action_logger
+    async def mouse_drag(self, from_ref: str, to_ref: str) -> Dict[str, Any]:
+        """Control the mouse to drag and drop in the browser using ref IDs."""
+        response = await self._send_command(
+            'mouse_drag',
+            {'from_ref': from_ref, 'to_ref': to_ref},
+        )
+        return response
+
+    @action_logger
+    async def press_key(self, keys: List[str]) -> Dict[str, Any]:
+        """Press key and key combinations."""
+        response = await self._send_command('press_key', {'keys': keys})
+        return response
+
     @action_logger
     async def back(self) -> Dict[str, Any]:
         """Navigate back."""
@@ -571,6 +596,22 @@ class WebSocketBrowserWrapper:
         # Fallback if wrapped in an object
         return response.get('tabs', [])
 
+    @action_logger
+    async def console_view(self) -> List[Dict[str, Any]]:
+        """Get current page console view"""
+        response = await self._send_command('console_view', {})
+
+        if isinstance(response, list):
+            return response
+
+        return response.get('logs', [])
+
+    @action_logger
+    async def console_exec(self, code: str) -> Dict[str, Any]:
+        """Execute javascript code and get result."""
+        response = await self._send_command('console_exec', {'code': code})
+        return response
+
     @action_logger
     async def wait_user(
         self, timeout_sec: Optional[float] = None

camel/toolkits/hybrid_browser_toolkit_py/actions.py

@@ -73,6 +73,9 @@ class ActionExecutor:
                 "extract": self._extract,
                 "scroll": self._scroll,
                 "enter": self._enter,
+                "mouse_control": self._mouse_control,
+                "mouse_drag": self._mouse_drag,
+                "press_key": self._press_key,
             }.get(action_type)
 
             if handler is None:
@@ -382,6 +385,150 @@ class ActionExecutor:
             "details": details,
         }
 
+    async def _mouse_control(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle mouse_control action based on the coordinates"""
+        control = action.get("control", "click")
+        x_coord = action.get("x", 0)
+        y_coord = action.get("y", 0)
+
+        details = {
+            "action_type": "mouse_control",
+            "target": f"coordinates : ({x_coord}, {y_coord})",
+        }
+        try:
+            if not self._valid_coordinates(x_coord, y_coord):
+                raise ValueError(
+                    "Invalid coordinates, outside viewport bounds :"
+                    f"({x_coord}, {y_coord})"
+                )
+            match control:
+                case "click":
+                    await self.page.mouse.click(x_coord, y_coord)
+                    message = "Action 'click' performed on the target"
+
+                case "right_click":
+                    await self.page.mouse.click(
+                        x_coord, y_coord, button="right"
+                    )
+                    message = "Action 'right_click' performed on the target"
+
+                case "dblclick":
+                    await self.page.mouse.dblclick(x_coord, y_coord)
+                    message = "Action 'dblclick' performed on the target"
+
+                case _:
+                    return {
+                        "message": f"Invalid control action {control}",
+                        "details": details,
+                    }
+
+            return {"message": message, "details": details}
+        except Exception as e:
+            return {"message": f"Action failed: {e}", "details": details}
+
+    async def _mouse_drag(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle mouse_drag action using ref IDs"""
+        from_ref = action.get("from_ref")
+        to_ref = action.get("to_ref")
+
+        if not from_ref or not to_ref:
+            return {
+                "message": "Error: mouse_drag requires from_ref and to_ref",
+                "details": {"error": "missing_refs"},
+            }
+
+        from_selector = f"[aria-ref='{from_ref}']"
+        to_selector = f"[aria-ref='{to_ref}']"
+
+        details = {
+            "action_type": "mouse_drag",
+            "from_ref": from_ref,
+            "to_ref": to_ref,
+            "from_selector": from_selector,
+            "to_selector": to_selector,
+        }
+
+        try:
+            # Get the source element
+            from_element = self.page.locator(from_selector)
+            from_count = await from_element.count()
+            if from_count == 0:
+                raise ValueError(
+                    f"Source element with ref '{from_ref}' not found"
+                )
+
+            # Get the target element
+            to_element = self.page.locator(to_selector)
+            to_count = await to_element.count()
+            if to_count == 0:
+                raise ValueError(
+                    f"Target element with ref '{to_ref}' not found"
+                )
+
+            # Get bounding boxes
+            from_box = await from_element.first.bounding_box()
+            to_box = await to_element.first.bounding_box()
+
+            if not from_box:
+                raise ValueError(
+                    f"Could not get bounding box for source element "
+                    f"with ref '{from_ref}'"
+                )
+            if not to_box:
+                raise ValueError(
+                    f"Could not get bounding box for target element "
+                    f"with ref '{to_ref}'"
+                )
+
+            # Calculate center coordinates
+            from_x = from_box['x'] + from_box['width'] / 2
+            from_y = from_box['y'] + from_box['height'] / 2
+            to_x = to_box['x'] + to_box['width'] / 2
+            to_y = to_box['y'] + to_box['height'] / 2
+
+            details.update(
+                {
+                    "from_coordinates": {"x": from_x, "y": from_y},
+                    "to_coordinates": {"x": to_x, "y": to_y},
+                }
+            )
+
+            # Perform the drag operation
+            await self.page.mouse.move(from_x, from_y)
+            await self.page.mouse.down()
+            # Destination coordinates
+            await self.page.mouse.move(to_x, to_y)
+            await self.page.mouse.up()
+
+            return {
+                "message": (
+                    f"Dragged from element [ref={from_ref}] to element "
+                    f"[ref={to_ref}]"
+                ),
+                "details": details,
+            }
+        except Exception as e:
+            return {"message": f"Action failed: {e}", "details": details}
+
+    async def _press_key(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle press_key action by combining the keys in a list."""
+        keys = action.get("keys", [])
+        if not keys:
+            return {
+                "message": "Error: No keys specified",
+                "details": {"action_type": "press_key", "keys": ""},
+            }
+        combined_keys = "+".join(keys)
+        details = {"action_type": "press_key", "keys": combined_keys}
+        try:
+            await self.page.keyboard.press(combined_keys)
+            return {
+                "message": "Pressed keys in the browser",
+                "details": details,
+            }
+        except Exception as e:
+            return {"message": f"Action failed: {e}", "details": details}
+
     # utilities
     async def _wait_dom_stable(self) -> None:
         r"""Wait for DOM to become stable before executing actions."""
@@ -402,6 +549,17 @@ class ActionExecutor:
         except Exception:
             pass  # Don't fail if wait times out
 
+    def _valid_coordinates(self, x_coord: float, y_coord: float) -> bool:
+        r"""Validate given coordinates against viewport bounds."""
+        viewport = self.page.viewport_size
+        if not viewport:
+            raise ValueError("Viewport size not available from current page.")
+
+        return (
+            0 <= x_coord <= viewport['width']
+            and 0 <= y_coord <= viewport['height']
+        )
+
     # static helpers
     @staticmethod
     def should_update_snapshot(action: Dict[str, Any]) -> bool: