camel-ai 0.2.74a4__py3-none-any.whl → 0.2.75__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts

@@ -1,6 +1,7 @@
 import {HybridBrowserSession} from './browser-session';
 import {ActionResult, BrowserAction, BrowserToolkitConfig, SnapshotResult, TabInfo, VisualMarkResult} from './types';
 import {ConfigLoader} from './config-loader';
+import {ConsoleMessage} from 'playwright';
 
 export class HybridBrowserToolkit {
   private session: HybridBrowserSession;
@@ -68,35 +69,52 @@ export class HybridBrowserToolkit {
   }
 
   async visitPage(url: string): Promise<any> {
-    const result = await this.session.visitPage(url);
-    
-    // Format response for Python layer compatibility
-    const response: any = {
-      result: result.message,
-      snapshot: '',
-    };
-    
-    if (result.success) {
-      const snapshotStart = Date.now();
-      response.snapshot = await this.getPageSnapshot(this.viewportLimit);
-      const snapshotTime = Date.now() - snapshotStart;
+    try {
+      // Ensure browser is initialized before visiting page
+      await this.session.ensureBrowser();
       
+      const result = await this.session.visitPage(url);
+      
+      // Format response for Python layer compatibility
+      const response: any = {
+        result: result.message,
+        snapshot: '',
+      };
+      
+      if (result.success) {
+        const snapshotStart = Date.now();
+        response.snapshot = await this.getPageSnapshot(this.viewportLimit);
+        const snapshotTime = Date.now() - snapshotStart;
+        
+        if (result.timing) {
+          result.timing.snapshot_time_ms = snapshotTime;
+        }
+      }
+      
+      // Include timing if available
       if (result.timing) {
-        result.timing.snapshot_time_ms = snapshotTime;
+        response.timing = result.timing;
       }
+      
+      // Include newTabId if present
+      if (result.newTabId) {
+        response.newTabId = result.newTabId;
+      }
+      
+      return response;
+    } catch (error) {
+      console.error('[visitPage] Error:', error);
+      return {
+        result: `Navigation to ${url} failed: ${error}`,
+        snapshot: '',
+        timing: {
+          total_time_ms: 0,
+          navigation_time_ms: 0,
+          dom_content_loaded_time_ms: 0,
+          network_idle_time_ms: 0,
+        }
+      };
     }
-    
-    // Include timing if available
-    if (result.timing) {
-      response.timing = result.timing;
-    }
-    
-    // Include newTabId if present
-    if (result.newTabId) {
-      response.newTabId = result.newTabId;
-    }
-    
-    return response;
   }
 
   async getPageSnapshot(viewportLimit: boolean = false): Promise<string> {
@@ -178,7 +196,40 @@ export class HybridBrowserToolkit {
       // Use sharp for image processing
       const sharp = require('sharp');
       const page = await this.session.getCurrentPage();
-      const viewport = page.viewportSize() || { width: 1280, height: 720 };
+      let viewport = page.viewportSize();
+      
+      // In CDP mode, viewportSize might be null, get it from window dimensions
+      if (!viewport) {
+        const windowSize = await page.evaluate(() => ({
+          width: window.innerWidth,
+          height: window.innerHeight
+        }));
+        viewport = windowSize;
+      }
+      
+      // Get device pixel ratio to handle high DPI screens
+      const dpr = await page.evaluate(() => window.devicePixelRatio) || 1;
+      
+      // Get actual screenshot dimensions
+      const metadata = await sharp(screenshotBuffer).metadata();
+      const screenshotWidth = metadata.width || viewport.width;
+      const screenshotHeight = metadata.height || viewport.height;
+      
+      // Calculate scaling factor between CSS pixels and screenshot pixels
+      const scaleX = screenshotWidth / viewport.width;
+      const scaleY = screenshotHeight / viewport.height;
+      
+      // Debug logging for CDP mode
+      if (process.env.HYBRID_BROWSER_DEBUG === '1') {
+        console.log('[CDP Debug] Viewport size:', viewport);
+        console.log('[CDP Debug] Device pixel ratio:', dpr);
+        console.log('[CDP Debug] Screenshot dimensions:', { width: screenshotWidth, height: screenshotHeight });
+        console.log('[CDP Debug] Scale factors:', { scaleX, scaleY });
+        console.log('[CDP Debug] Elements with coordinates:', elementsWithCoords.length);
+        elementsWithCoords.slice(0, 3).forEach(([ref, element]) => {
+          console.log(`[CDP Debug] Element ${ref}:`, element.coordinates);
+        });
+      }
       
       // Filter elements visible in viewport
       const visibleElements = elementsWithCoords.filter(([ref, element]) => {
@@ -197,18 +248,19 @@ export class HybridBrowserToolkit {
         const coords = element.coordinates!;
         const isClickable = clickableElements.has(ref);
         
-        // Use original coordinates for elements within viewport
-        // Clamp only to prevent marks from extending beyond screenshot bounds
-        const x = Math.max(0, coords.x);
-        const y = Math.max(0, coords.y);
-        const maxWidth = viewport.width - x;
-        const maxHeight = viewport.height - y;
-        const width = Math.min(coords.width, maxWidth);
-        const height = Math.min(coords.height, maxHeight);
+        // Scale coordinates from CSS pixels to screenshot pixels
+        const x = Math.max(0, coords.x * scaleX);
+        const y = Math.max(0, coords.y * scaleY);
+        const width = coords.width * scaleX;
+        const height = coords.height * scaleY;
+        
+        // Clamp to screenshot bounds
+        const clampedWidth = Math.min(width, screenshotWidth - x);
+        const clampedHeight = Math.min(height, screenshotHeight - y);
         
         // Position text to be visible even if element is partially cut off
-        const textX = Math.max(2, Math.min(x + 2, viewport.width - 40));
-        const textY = Math.max(14, Math.min(y + 14, viewport.height - 4));
+        const textX = Math.max(2, Math.min(x + 2, screenshotWidth - 40));
+        const textY = Math.max(14, Math.min(y + 14, screenshotHeight - 4));
         
         // Different colors for clickable vs non-clickable elements
         const colors = isClickable ? {
@@ -222,7 +274,7 @@ export class HybridBrowserToolkit {
         };
         
         return `
-          <rect x="${x}" y="${y}" width="${width}" height="${height}" 
+          <rect x="${x}" y="${y}" width="${clampedWidth}" height="${clampedHeight}" 
                 fill="${colors.fill}" stroke="${colors.stroke}" stroke-width="2" rx="2"/>
           <text x="${textX}" y="${textY}" font-family="Arial, sans-serif" 
                 font-size="12" fill="${colors.textFill}" font-weight="bold">${ref}</text>
@@ -230,7 +282,7 @@ export class HybridBrowserToolkit {
       }).join('');
       
       const svgOverlay = `
-        <svg width="${viewport.width}" height="${viewport.height}" xmlns="http://www.w3.org/2000/svg">
+        <svg width="${screenshotWidth}" height="${screenshotHeight}" xmlns="http://www.w3.org/2000/svg">
           ${marks}
         </svg>
       `;
@@ -362,8 +414,20 @@ export class HybridBrowserToolkit {
     return this.executeActionWithSnapshot(action);
   }
 
-  async type(ref: string, text: string): Promise<any> {
-    const action: BrowserAction = { type: 'type', ref, text };
+  async type(refOrInputs: string | Array<{ ref: string; text: string }>, text?: string): Promise<any> {
+    let action: BrowserAction;
+    
+    if (typeof refOrInputs === 'string') {
+      // Single input mode (backward compatibility)
+      if (text === undefined) {
+        throw new Error('Text parameter is required when ref is a string');
+      }
+      action = { type: 'type', ref: refOrInputs, text };
+    } else {
+      // Multiple inputs mode
+      action = { type: 'type', inputs: refOrInputs };
+    }
+    
     return this.executeActionWithSnapshot(action);
   }
 
@@ -382,6 +446,21 @@ export class HybridBrowserToolkit {
     return this.executeActionWithSnapshot(action);
   }
 
+  async mouseControl(control: 'click' | 'right_click'| 'dblclick', x: number, y: number): Promise<any> {
+    const action: BrowserAction = { type: 'mouse_control', control, x, y };
+    return this.executeActionWithSnapshot(action);
+  }
+
+  async mouseDrag(from_ref: string, to_ref: string): Promise<any> {
+    const action: BrowserAction = { type: 'mouse_drag', from_ref, to_ref };
+    return this.executeActionWithSnapshot(action);
+  }
+
+  async pressKeys(keys: string[]): Promise<any> {
+    const action: BrowserAction = { type: 'press_key', keys};
+    return this.executeActionWithSnapshot(action);
+  }
+
   async back(): Promise<ActionResult> {
     const startTime = Date.now();
     
@@ -519,4 +598,93 @@ export class HybridBrowserToolkit {
     return await this.session.getTabInfo();
   }
 
-}
+  async getConsoleView(): Promise<any> {
+    const currentLogs = await this.session.getCurrentLogs();
+    // Format logs
+    return currentLogs.map(item => ({
+      type: item.type(),
+      text: item.text(),
+    }));
+  }
+
+  async consoleExecute(code: string): Promise<any> {
+    const startTime = Date.now();
+    try {
+      const page = await this.session.getCurrentPage();
+      
+      // Wrap the code to capture console.log output
+      const wrappedCode = `
+        (function() {
+          const _logs = [];
+          const originalLog = console.log;
+          console.log = function(...args) {
+            _logs.push(args.map(arg => {
+              try {
+                return typeof arg === 'object' ? JSON.stringify(arg) : String(arg);
+              } catch (e) {
+                return String(arg);
+              }
+            }).join(' '));
+            originalLog.apply(console, args);
+          };
+          
+          let result;
+          try {
+            result = eval(${JSON.stringify(code)});
+          } catch (e) {
+            try {
+              result = (function() { ${code} })();
+            } catch (error) {
+              console.log = originalLog;
+              throw error;
+            }
+          }
+          
+          console.log = originalLog;
+          return { result, logs: _logs };
+        })()
+      `;
+      
+      const evalResult = await page.evaluate(wrappedCode) as { result: any; logs: string[] };
+      const { result, logs } = evalResult;
+
+      const snapshotStart = Date.now();
+      const snapshot = await this.getPageSnapshot(this.viewportLimit);
+      const snapshotTime = Date.now() - snapshotStart;
+      const totalTime = Date.now() - startTime;
+
+      // Properly serialize the result
+      let resultStr: string;
+      try {
+        resultStr = JSON.stringify(result, null, 2);
+      } catch (e) {
+        // Fallback for non-serializable values
+        resultStr = String(result);
+      }
+
+      return {
+        result: `Console execution result: ${resultStr}`,
+        console_output: logs,
+        snapshot: snapshot,
+        timing: {
+          total_time_ms: totalTime,
+          snapshot_time_ms: snapshotTime,
+        },
+      };
+      
+    } catch (error) {
+      const totalTime = Date.now() - startTime;
+      return {
+        result: `Console execution failed: ${error}`,
+        console_output: [],
+        snapshot: '',
+        timing: {
+          total_time_ms: totalTime,
+          snapshot_time_ms: 0,
+        },
+      };
+    }
+  }
+
+}
+

camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts

@@ -81,8 +81,9 @@ export interface ClickAction {
 
 export interface TypeAction {
   type: 'type';
-  ref: string;
-  text: string;
+  ref?: string;  // Optional for backward compatibility
+  text?: string; // Optional for backward compatibility
+  inputs?: Array<{ ref: string; text: string }>; // New field for multiple inputs
 }
 
 export interface SelectAction {
@@ -101,7 +102,25 @@ export interface EnterAction {
   type: 'enter';
 }
 
-export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction;
+export interface MouseAction {
+  type: 'mouse_control';
+  control: 'click' | 'right_click' | 'dblclick';
+  x: number; 
+  y: number; 
+}
+
+export interface MouseDragAction {
+  type: 'mouse_drag';
+  from_ref: string;
+  to_ref: string;
+}
+
+export interface PressKeyAction {
+  type: 'press_key';
+  keys: string[];
+}
+
+export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction | MouseAction | MouseDragAction | PressKeyAction;
 
 export interface VisualMarkResult {
   text: string;

camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js

@@ -160,7 +160,14 @@ class WebSocketBrowserServer {
 
       case 'type':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
-        return await this.toolkit.type(params.ref, params.text);
+        // Handle both single input and multiple inputs
+        if (params.inputs) {
+          // Multiple inputs mode - pass inputs array directly
+          return await this.toolkit.type(params.inputs);
+        } else {
+          // Single input mode - pass ref and text
+          return await this.toolkit.type(params.ref, params.text);
+        }
 
       case 'select':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
@@ -173,6 +180,18 @@ class WebSocketBrowserServer {
       case 'enter':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
         return await this.toolkit.enter();
+      
+      case 'mouse_control':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.mouseControl(params.control, params.x, params.y);
+
+      case 'mouse_drag':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.mouseDrag(params.from_ref, params.to_ref);
+
+      case 'press_key':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.pressKeys(params.keys);
 
       case 'back':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
@@ -194,6 +213,14 @@ class WebSocketBrowserServer {
         if (!this.toolkit) throw new Error('Toolkit not initialized');
         return await this.toolkit.getTabInfo();
 
+      case 'console_view':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.getConsoleView();
+
+      case 'console_exec':
+        if (!this.toolkit) throw new Error('Toolkit not initialized');
+        return await this.toolkit.consoleExecute(params.code);
+
       case 'wait_user':
         if (!this.toolkit) throw new Error('Toolkit not initialized');
         return await this.toolkit.waitUser(params.timeout);

camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py

@@ -396,6 +396,9 @@ class WebSocketBrowserWrapper:
         """Send a command to the WebSocket server and get response."""
         await self._ensure_connection()
 
+        # Process params to ensure refs have 'e' prefix
+        params = self._process_refs_in_params(params)
+
         message_id = str(uuid.uuid4())
         message = {'id': message_id, 'command': command, 'params': params}
 
@@ -503,6 +506,55 @@ class WebSocketBrowserWrapper:
 
         return ToolResult(text=response['text'], images=response['images'])
 
+    def _ensure_ref_prefix(self, ref: str) -> str:
+        """Ensure ref has proper prefix"""
+        if not ref:
+            return ref
+
+        # If ref is purely numeric, add 'e' prefix for main frame
+        if ref.isdigit():
+            return f'e{ref}'
+
+        return ref
+
+    def _process_refs_in_params(
+        self, params: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Process parameters to ensure all refs have 'e' prefix."""
+        if not params:
+            return params
+
+        # Create a copy to avoid modifying the original
+        processed = params.copy()
+
+        # Handle direct ref parameters
+        if 'ref' in processed:
+            processed['ref'] = self._ensure_ref_prefix(processed['ref'])
+
+        # Handle from_ref and to_ref for drag operations
+        if 'from_ref' in processed:
+            processed['from_ref'] = self._ensure_ref_prefix(
+                processed['from_ref']
+            )
+        if 'to_ref' in processed:
+            processed['to_ref'] = self._ensure_ref_prefix(processed['to_ref'])
+
+        # Handle inputs array for type_multiple
+        if 'inputs' in processed and isinstance(processed['inputs'], list):
+            processed_inputs = []
+            for input_item in processed['inputs']:
+                if isinstance(input_item, dict) and 'ref' in input_item:
+                    processed_input = input_item.copy()
+                    processed_input['ref'] = self._ensure_ref_prefix(
+                        input_item['ref']
+                    )
+                    processed_inputs.append(processed_input)
+                else:
+                    processed_inputs.append(input_item)
+            processed['inputs'] = processed_inputs
+
+        return processed
+
     @action_logger
     async def click(self, ref: str) -> Dict[str, Any]:
         """Click an element."""
@@ -515,6 +567,14 @@ class WebSocketBrowserWrapper:
         response = await self._send_command('type', {'ref': ref, 'text': text})
         return response
 
+    @action_logger
+    async def type_multiple(
+        self, inputs: List[Dict[str, str]]
+    ) -> Dict[str, Any]:
+        """Type text into multiple elements."""
+        response = await self._send_command('type', {'inputs': inputs})
+        return response
+
     @action_logger
     async def select(self, ref: str, value: str) -> Dict[str, Any]:
         """Select an option."""
@@ -537,6 +597,31 @@ class WebSocketBrowserWrapper:
         response = await self._send_command('enter', {})
         return response
 
+    @action_logger
+    async def mouse_control(
+        self, control: str, x: float, y: float
+    ) -> Dict[str, Any]:
+        """Control the mouse to interact with browser with x, y coordinates."""
+        response = await self._send_command(
+            'mouse_control', {'control': control, 'x': x, 'y': y}
+        )
+        return response
+
+    @action_logger
+    async def mouse_drag(self, from_ref: str, to_ref: str) -> Dict[str, Any]:
+        """Control the mouse to drag and drop in the browser using ref IDs."""
+        response = await self._send_command(
+            'mouse_drag',
+            {'from_ref': from_ref, 'to_ref': to_ref},
+        )
+        return response
+
+    @action_logger
+    async def press_key(self, keys: List[str]) -> Dict[str, Any]:
+        """Press key and key combinations."""
+        response = await self._send_command('press_key', {'keys': keys})
+        return response
+
     @action_logger
     async def back(self) -> Dict[str, Any]:
         """Navigate back."""
@@ -571,6 +656,22 @@ class WebSocketBrowserWrapper:
         # Fallback if wrapped in an object
         return response.get('tabs', [])
 
+    @action_logger
+    async def console_view(self) -> List[Dict[str, Any]]:
+        """Get current page console view"""
+        response = await self._send_command('console_view', {})
+
+        if isinstance(response, list):
+            return response
+
+        return response.get('logs', [])
+
+    @action_logger
+    async def console_exec(self, code: str) -> Dict[str, Any]:
+        """Execute javascript code and get result."""
+        response = await self._send_command('console_exec', {'code': code})
+        return response
+
     @action_logger
     async def wait_user(
         self, timeout_sec: Optional[float] = None