camel-ai 0.2.74a4__py3-none-any.whl → 0.2.75a0__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py

@@ -64,11 +64,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
         "browser_select",
         "browser_scroll",
         "browser_enter",
+        "browser_mouse_control",
+        "browser_mouse_drag",
+        "browser_press_key",
         "browser_wait_user",
         "browser_solve_task",
         "browser_switch_tab",
         "browser_close_tab",
         "browser_get_tab_info",
+        "browser_console_view",
+        "browser_console_exec",
     ]
 
     def __init__(
@@ -863,6 +868,156 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
                 "total_tabs": 0,
             }
 
+    async def browser_mouse_control(
+        self, *, control: str, x: float, y: float
+    ) -> Dict[str, Any]:
+        r"""Control the mouse to interact with browser with x, y coordinates
+
+        Args:
+            control ([str]): The action to perform: 'click', 'right_click'
+            or 'dblclick'.
+            x (float): x-coordinate for the control action.
+            y (float): y-coordinate for the control action.
+
+        Returns:
+            Dict[str, Any]: A dictionary with the result of the action:
+                - "result" (str): Confirmation of the action.
+                - "snapshot" (str): A snapshot of the page after mouse
+                control action.
+                - "tabs" (List[Dict]): Information about all open tabs.
+                - "current_tab" (int): Index of the active tab.
+                - "total_tabs" (int): Total number of open tabs.
+        """
+        try:
+            ws_wrapper = await self._get_ws_wrapper()
+            result = await ws_wrapper.mouse_control(control, x, y)
+
+            # Add tab information
+            tab_info = await ws_wrapper.get_tab_info()
+            result.update(
+                {
+                    "tabs": tab_info,
+                    "current_tab": next(
+                        (
+                            i
+                            for i, tab in enumerate(tab_info)
+                            if tab.get("is_current")
+                        ),
+                        0,
+                    ),
+                    "total_tabs": len(tab_info),
+                }
+            )
+
+            return result
+        except Exception as e:
+            logger.error(f"Failed to control mouse: {e}")
+            return {
+                "result": f"Error with mouse control: {e}",
+                "snapshot": "",
+                "tabs": [],
+                "current_tab": 0,
+                "total_tabs": 0,
+            }
+
+    async def browser_mouse_drag(
+        self, *, from_ref: str, to_ref: str
+    ) -> Dict[str, Any]:
+        r"""Control the mouse to drag and drop in the browser using ref IDs.
+
+        Args:
+            from_ref (str): The `ref` ID of the source element to drag from.
+            to_ref (str): The `ref` ID of the target element to drag to.
+
+        Returns:
+            Dict[str, Any]: A dictionary with the result of the action:
+                - "result" (str): Confirmation of the action.
+                - "snapshot" (str): A new page snapshot.
+                - "tabs" (List[Dict]): Information about all open tabs.
+                - "current_tab" (int): Index of the active tab.
+                - "total_tabs" (int): Total number of open tabs.
+        """
+        try:
+            ws_wrapper = await self._get_ws_wrapper()
+            result = await ws_wrapper.mouse_drag(from_ref, to_ref)
+
+            # Add tab information
+            tab_info = await ws_wrapper.get_tab_info()
+            result.update(
+                {
+                    "tabs": tab_info,
+                    "current_tab": next(
+                        (
+                            i
+                            for i, tab in enumerate(tab_info)
+                            if tab.get("is_current")
+                        ),
+                        0,
+                    ),
+                    "total_tabs": len(tab_info),
+                }
+            )
+
+            return result
+        except Exception as e:
+            logger.error(f"Error with mouse drag and drop: {e}")
+            return {
+                "result": f"Error with mouse drag and drop: {e}",
+                "snapshot": "",
+                "tabs": [],
+                "current_tab": 0,
+                "total_tabs": 0,
+            }
+
+    async def browser_press_key(self, *, keys: List[str]) -> Dict[str, Any]:
+        r"""Press key and key combinations.
+        Supports single key press or combination of keys by concatenating
+        them with '+' separator.
+
+        Args:
+            keys (List[str]): key or list of keys.
+
+        Returns:
+            Dict[str, Any]: A dictionary with the result of the action:
+                - "result" (str): Confirmation of the action.
+                - "snapshot" (str): A snapshot of the page after
+                press key action.
+                - "tabs" (List[Dict]): Information about all open tabs.
+                - "current_tab" (int): Index of the active tab.
+                - "total_tabs" (int): Total number of open tabs.
+        """
+        try:
+            ws_wrapper = await self._get_ws_wrapper()
+            result = await ws_wrapper.press_key(keys)
+
+            # Add tab information
+            tab_info = await ws_wrapper.get_tab_info()
+            result.update(
+                {
+                    "tabs": tab_info,
+                    "current_tab": next(
+                        (
+                            i
+                            for i, tab in enumerate(tab_info)
+                            if tab.get("is_current")
+                        ),
+                        0,
+                    ),
+                    "total_tabs": len(tab_info),
+                }
+            )
+
+            return result
+        except Exception as e:
+            logger.error(f"Failed to press key: {e}")
+            return {
+                "result": f"Error with press key: {e}",
+                "snapshot": "",
+                "tabs": [],
+                "current_tab": 0,
+                "total_tabs": 0,
+            }
+
     async def browser_switch_tab(self, *, tab_id: str) -> Dict[str, Any]:
         r"""Switches to a different browser tab using its ID.
 
@@ -1002,6 +1157,71 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
                 "total_tabs": 0,
             }
 
+    async def browser_console_view(self) -> Dict[str, Any]:
+        r"""View current page console logs.
+
+        Returns:
+            Dict[str, Any]: A dictionary with tab information:
+                - "console_messages" (List[Dict]) : List of messages logged
+                in the current page
+
+        """
+        try:
+            ws_wrapper = await self._get_ws_wrapper()
+            console_logs = await ws_wrapper.console_view()
+
+            return {"console_messages": console_logs}
+        except Exception as e:
+            logger.error(f"Failed to get console view: {e}")
+            return {"console_messages": []}
+
+    async def browser_console_exec(self, code: str) -> Dict[str, Any]:
+        r"""Execute javascript code in the console of the current page and get
+        results.
+
+        Args:
+            code (str): JavaScript code to execute in the browser console.
+
+        Returns:
+            Dict[str, Any]: A dictionary with the result of the action:
+                - "result" (str): Confirmation of the action.
+                - "snapshot" (str): A snapshot of the active tab after
+                console execute action.
+                - "tabs" (List[Dict]): Information about remaining tabs.
+                - "current_tab" (int): Index of the new active tab.
+                - "total_tabs" (int): Total number of remaining tabs.
+        """
+        try:
+            ws_wrapper = await self._get_ws_wrapper()
+            result = await ws_wrapper.console_exec(code)
+
+            tab_info = await ws_wrapper.get_tab_info()
+            result.update(
+                {
+                    "tabs": tab_info,
+                    "current_tab": next(
+                        (
+                            i
+                            for i, tab in enumerate(tab_info)
+                            if tab.get("is_current")
+                        ),
+                        0,
+                    ),
+                    "total_tabs": len(tab_info),
+                }
+            )
+
+            return result
+        except Exception as e:
+            logger.error(f"Failed to execute javascript in console: {e}")
+            return {
+                "result": f"Error in code execution: {e}",
+                "snapshot": "",
+                "tabs": [],
+                "current_tab": 0,
+                "total_tabs": 0,
+            }
+
     # Additional methods for backward compatibility
     async def browser_wait_user(
         self, timeout_sec: Optional[float] = None
@@ -1146,10 +1366,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
             "browser_select": self.browser_select,
             "browser_scroll": self.browser_scroll,
             "browser_enter": self.browser_enter,
+            "browser_mouse_click": self.browser_mouse_control,
+            "browser_mouse_drag": self.browser_mouse_drag,
+            "browser_press_key": self.browser_press_key,
             "browser_wait_user": self.browser_wait_user,
             "browser_switch_tab": self.browser_switch_tab,
             "browser_close_tab": self.browser_close_tab,
             "browser_get_tab_info": self.browser_get_tab_info,
+            "browser_console_view": self.browser_console_view,
+            "browser_console_exec": self.browser_console_exec,
         }
 
         enabled_tools = []

camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts

@@ -1,4 +1,4 @@
-import { Page, Browser, BrowserContext, chromium } from 'playwright';
+import { Page, Browser, BrowserContext, chromium, ConsoleMessage } from 'playwright';
 import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming } from './types';
 import { ConfigLoader, StealthConfig } from './config-loader';
 
@@ -6,18 +6,43 @@ export class HybridBrowserSession {
   private browser: Browser | null = null;
   private context: BrowserContext | null = null;
   private pages: Map<string, Page> = new Map();
+  private consoleLogs: Map<string, ConsoleMessage[]> = new Map();
   private currentTabId: string | null = null;
   private tabCounter = 0;
   private configLoader: ConfigLoader;
   private scrollPosition: { x: number; y: number } = {x: 0, y: 0};
   private hasNavigatedBefore = false; //  Track if we've navigated before
+  private logLimit: number; 
 
   constructor(config: BrowserToolkitConfig = {}) {
     // Use ConfigLoader's fromPythonConfig to handle conversion properly
     this.configLoader = ConfigLoader.fromPythonConfig(config);
+    // Load browser configuration for console log limit, default to 1000
+    this.logLimit = this.configLoader.getBrowserConfig().consoleLogLimit || 1000;
   }
 
-  async ensureBrowser(): Promise<void> {
+  private registerNewPage(tabId: string, page: Page): void {
+    // Register page and logs with tabId
+    this.pages.set(tabId, page);
+    this.consoleLogs.set(tabId, []);
+    // Set up console log listener for the page
+    page.on('console', (msg: ConsoleMessage) => {
+      const logs = this.consoleLogs.get(tabId);
+      if (logs) {
+        logs.push(msg);
+        if (logs.length >  this.logLimit) {
+          logs.shift();
+        }
+      }
+    });
+
+    // Clean logs on page close
+    page.on('close', () => {
+      this.consoleLogs.delete(tabId);
+    });
+  }
+
+  async ensureBrowser(): Promise<void> {  
     if (this.browser) {
       return;
     }
@@ -57,7 +82,7 @@ export class HybridBrowserSession {
           // In CDP mode, only consider pages with about:blank as available
           if (pageUrl === 'about:blank') {
             const tabId = this.generateTabId();
-            this.pages.set(tabId, page);
+            this.registerNewPage(tabId, page);
             if (!this.currentTabId) {
               this.currentTabId = tabId;
               availablePageFound = true;
@@ -97,7 +122,7 @@ export class HybridBrowserSession {
         const pages = this.context.pages();
         if (pages.length > 0) {
           const initialTabId = this.generateTabId();
-          this.pages.set(initialTabId, pages[0]);
+          this.registerNewPage(initialTabId, pages[0]);
           this.currentTabId = initialTabId;
         }
       } else {
@@ -115,7 +140,7 @@ export class HybridBrowserSession {
         
         const initialPage = await this.context.newPage();
         const initialTabId = this.generateTabId();
-        this.pages.set(initialTabId, initialPage);
+        this.registerNewPage(initialTabId, initialPage);
         this.currentTabId = initialTabId;
       }
     }
@@ -139,6 +164,13 @@ export class HybridBrowserSession {
     return this.pages.get(this.currentTabId)!;
   }
 
+  async getCurrentLogs(): Promise<ConsoleMessage[]> {
+    if (!this.currentTabId || !this.consoleLogs.has(this.currentTabId)) {
+      return [];
+    }
+    return this.consoleLogs.get(this.currentTabId) || [];
+  }
+
   /**
    * Get current scroll position from the page
    */
@@ -343,7 +375,7 @@ export class HybridBrowserSession {
           
           // Generate tab ID for the new page
           const newTabId = this.generateTabId();
-          this.pages.set(newTabId, newPage);
+          this.registerNewPage(newTabId, newPage);
           
           // Set up page properties
           const browserConfig = this.configLoader.getBrowserConfig();
@@ -434,7 +466,97 @@ export class HybridBrowserSession {
     }
   }
 
+  /**
+   *  Simplified mouse control implementation
+   */
+  private async performMouseControl(page: Page, control: string, x: number, y: number): Promise<{ success: boolean; error?: string }> {
+    try {
+      const viewport = page.viewportSize();
+      if (!viewport) {
+        return { success: false, error: 'Viewport size not available from page.' };
+      }
+      if (x < 0 || y < 0 || x > viewport.width || y > viewport.height) {
+        return { success: false, error: `Invalid coordinates, outside viewport bounds: (${x}, ${y})` };
+      }
+      switch (control) {
+        case 'click': {
+          await page.mouse.click(x, y);
+          break;
+        }
+        case 'right_click': {
+          await page.mouse.click(x, y, { button: 'right' });
+          break;
+        }
+        case 'dblclick': {
+          await page.mouse.dblclick(x, y);
+          break;
+        }
+        default:
+          return { success: false, error: `Invalid control action: ${control}` };
+      }
+      
+      return { success: true };
+    } catch (error) {
+      return { success: false, error: `Mouse action failed: ${error}` };
+    }
+  }
 
+  /**
+   *  Enhanced mouse drag and drop implementation using ref IDs
+   */
+  private async performMouseDrag(page: Page, fromRef: string, toRef: string): Promise<{ success: boolean; error?: string }> {
+    try {
+      // Ensure we have the latest snapshot
+      await (page as any)._snapshotForAI();
+      
+      // Get elements using Playwright's aria-ref selector
+      const fromSelector = `aria-ref=${fromRef}`;
+      const toSelector = `aria-ref=${toRef}`;
+      
+      const fromElement = await page.locator(fromSelector).first();
+      const toElement = await page.locator(toSelector).first();
+      
+      // Check if elements exist
+      const fromExists = await fromElement.count() > 0;
+      const toExists = await toElement.count() > 0;
+      
+      if (!fromExists) {
+        return { success: false, error: `Source element with ref ${fromRef} not found` };
+      }
+      
+      if (!toExists) {
+        return { success: false, error: `Target element with ref ${toRef} not found` };
+      }
+      
+      // Get the center coordinates of both elements
+      const fromBox = await fromElement.boundingBox();
+      const toBox = await toElement.boundingBox();
+      
+      if (!fromBox) {
+        return { success: false, error: `Could not get bounding box for source element with ref ${fromRef}` };
+      }
+      
+      if (!toBox) {
+        return { success: false, error: `Could not get bounding box for target element with ref ${toRef}` };
+      }
+      
+      const fromX = fromBox.x + fromBox.width / 2;
+      const fromY = fromBox.y + fromBox.height / 2;
+      const toX = toBox.x + toBox.width / 2;
+      const toY = toBox.y + toBox.height / 2;
+
+      // Perform the drag operation
+      await page.mouse.move(fromX, fromY);
+      await page.mouse.down();
+      // Destination coordinates
+      await page.mouse.move(toX, toY);
+      await page.mouse.up();
+
+      return { success: true };
+    } catch (error) {
+      return { success: false, error: `Mouse drag action failed: ${error}` };
+    }
+  }
 
   async executeAction(action: BrowserAction): Promise<ActionResult> {
     const startTime = Date.now();
@@ -519,6 +641,40 @@ export class HybridBrowserSession {
           actionExecutionTime = Date.now() - enterStart;
           break;
         }
+
+        case 'mouse_control': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const mouseControlStart = Date.now();
+          const mouseControlResult = await this.performMouseControl(page, action.control, action.x, action.y);
+
+          if (!mouseControlResult.success) {
+            throw new Error(`Action failed: ${mouseControlResult.error}`);
+          }
+          actionExecutionTime = Date.now() - mouseControlStart;
+          break;
+        }
+
+        case 'mouse_drag': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const mouseDragStart = Date.now();
+          const mouseDragResult = await this.performMouseDrag(page, action.from_ref, action.to_ref);
+
+          if (!mouseDragResult.success) {
+            throw new Error(`Action failed: ${mouseDragResult.error}`);
+          }
+          actionExecutionTime = Date.now() - mouseDragStart;
+          break;
+        }
+
+        case 'press_key': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const keyPressStart = Date.now();
+          // concatenate keys with '+' for key combinations
+          const keys = action.keys.join('+');
+          await page.keyboard.press(keys);
+          actionExecutionTime = Date.now() - keyPressStart;
+          break;
+        }
           
         default:
           throw new Error(`Unknown action type: ${(action as any).type}`);
@@ -651,7 +807,7 @@ export class HybridBrowserSession {
             if (!isTracked && pageUrl === 'about:blank') {
               newPage = page;
               newTabId = this.generateTabId();
-              this.pages.set(newTabId, newPage);
+              this.registerNewPage(newTabId, newPage);
               break;
             }
           }
@@ -663,7 +819,7 @@ export class HybridBrowserSession {
           // Non-CDP mode: create new page as usual
           newPage = await this.context.newPage();
           newTabId = this.generateTabId();
-          this.pages.set(newTabId, newPage);
+          this.registerNewPage(newTabId, newPage);
         }
         
         // Set up page properties

camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts

@@ -30,6 +30,7 @@ export interface BrowserConfig {
   // Tab management
   tabIdPrefix: string;
   tabCounterPadding: number;
+  consoleLogLimit: number;
   
   // Scroll and positioning
   scrollPositionScale: number;
@@ -113,6 +114,7 @@ function getDefaultBrowserConfig(): BrowserConfig {
     clickTimeout: 3000,
     tabIdPrefix: 'tab-',
     tabCounterPadding: 3,
+    consoleLogLimit: 1000,
     scrollPositionScale: 0.1,
     navigationDelay: 100,
     blankPageUrls: ['about:blank', ''],

camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts

@@ -1,6 +1,7 @@
 import {HybridBrowserSession} from './browser-session';
 import {ActionResult, BrowserAction, BrowserToolkitConfig, SnapshotResult, TabInfo, VisualMarkResult} from './types';
 import {ConfigLoader} from './config-loader';
+import {ConsoleMessage} from 'playwright';
 
 export class HybridBrowserToolkit {
   private session: HybridBrowserSession;
@@ -382,6 +383,21 @@ export class HybridBrowserToolkit {
     return this.executeActionWithSnapshot(action);
   }
 
+  async mouseControl(control: 'click' | 'right_click'| 'dblclick', x: number, y: number): Promise<any> {
+    const action: BrowserAction = { type: 'mouse_control', control, x, y };
+    return this.executeActionWithSnapshot(action);
+  }
+
+  async mouseDrag(from_ref: string, to_ref: string): Promise<any> {
+    const action: BrowserAction = { type: 'mouse_drag', from_ref, to_ref };
+    return this.executeActionWithSnapshot(action);
+  }
+
+  async pressKeys(keys: string[]): Promise<any> {
+    const action: BrowserAction = { type: 'press_key', keys};
+    return this.executeActionWithSnapshot(action);
+  }
+
   async back(): Promise<ActionResult> {
     const startTime = Date.now();
     
@@ -519,4 +535,93 @@ export class HybridBrowserToolkit {
     return await this.session.getTabInfo();
   }
 
-}
+  async getConsoleView(): Promise<any> {
+    const currentLogs = await this.session.getCurrentLogs();
+    // Format logs
+    return currentLogs.map(item => ({
+      type: item.type(),
+      text: item.text(),
+    }));
+  }
+
+  async consoleExecute(code: string): Promise<any> {
+    const startTime = Date.now();
+    try {
+      const page = await this.session.getCurrentPage();
+      
+      // Wrap the code to capture console.log output
+      const wrappedCode = `
+        (function() {
+          const _logs = [];
+          const originalLog = console.log;
+          console.log = function(...args) {
+            _logs.push(args.map(arg => {
+              try {
+                return typeof arg === 'object' ? JSON.stringify(arg) : String(arg);
+              } catch (e) {
+                return String(arg);
+              }
+            }).join(' '));
+            originalLog.apply(console, args);
+          };
+          
+          let result;
+          try {
+            result = eval(${JSON.stringify(code)});
+          } catch (e) {
+            try {
+              result = (function() { ${code} })();
+            } catch (error) {
+              console.log = originalLog;
+              throw error;
+            }
+          }
+          
+          console.log = originalLog;
+          return { result, logs: _logs };
+        })()
+      `;
+      
+      const evalResult = await page.evaluate(wrappedCode) as { result: any; logs: string[] };
+      const { result, logs } = evalResult;
+
+      const snapshotStart = Date.now();
+      const snapshot = await this.getPageSnapshot(this.viewportLimit);
+      const snapshotTime = Date.now() - snapshotStart;
+      const totalTime = Date.now() - startTime;
+
+      // Properly serialize the result
+      let resultStr: string;
+      try {
+        resultStr = JSON.stringify(result, null, 2);
+      } catch (e) {
+        // Fallback for non-serializable values
+        resultStr = String(result);
+      }
+
+      return {
+        result: `Console execution result: ${resultStr}`,
+        console_output: logs,
+        snapshot: snapshot,
+        timing: {
+          total_time_ms: totalTime,
+          snapshot_time_ms: snapshotTime,
+        },
+      };
+      
+    } catch (error) {
+      const totalTime = Date.now() - startTime;
+      return {
+        result: `Console execution failed: ${error}`,
+        console_output: [],
+        snapshot: '',
+        timing: {
+          total_time_ms: totalTime,
+          snapshot_time_ms: 0,
+        },
+      };
+    }
+  }
+
+}
+

camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts

@@ -101,7 +101,25 @@ export interface EnterAction {
   type: 'enter';
 }
 
-export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction;
+export interface MouseAction {
+  type: 'mouse_control';
+  control: 'click' | 'right_click' | 'dblclick';
+  x: number; 
+  y: number; 
+}
+
+export interface MouseDragAction {
+  type: 'mouse_drag';
+  from_ref: string;
+  to_ref: string;
+}
+
+export interface PressKeyAction {
+  type: 'press_key';
+  keys: string[];
+}
+
+export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction | MouseAction | MouseDragAction | PressKeyAction;
 
 export interface VisualMarkResult {
   text: string;