camel-ai 0.2.73a1__py3-none-any.whl → 0.2.73a3__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js

@@ -78,8 +78,52 @@ class WebSocketBrowserServer {
     switch (command) {
       case 'init':
         console.log('Initializing toolkit with params:', JSON.stringify(params, null, 2));
-        this.toolkit = new HybridBrowserToolkit(params);
-        return { message: 'Toolkit initialized' };
+        
+        // Check if CDP is available first
+        let useCdp = false;
+        let cdpUrl = params.cdpUrl || 'http://localhost:9222';
+        
+        // Extract base URL and port for validation
+        const baseUrl = cdpUrl.includes('/devtools/') ? cdpUrl.split('/devtools/')[0] : cdpUrl;
+        
+        try {
+          // Test if Chrome debug port is accessible and get page URL
+          const response = await fetch(`${baseUrl}/json`);
+          if (response.ok) {
+            const pages = await response.json();
+            if (pages && pages.length > 0) {
+              // If user provided a specific page URL, use it; otherwise use first available
+              if (cdpUrl.includes('/devtools/page/') || cdpUrl.includes('/devtools/browser/')) {
+                useCdp = true;
+                console.log(`Using provided CDP URL: ${cdpUrl}`);
+              } else {
+                // Use the first available page
+                const firstPage = pages[0];
+                const pageUrl = firstPage.devtoolsFrontendUrl;
+                const pageId = pageUrl.match(/ws=localhost:\d+(.*)$/)?.[1];
+                
+                if (pageId) {
+                  useCdp = true;
+                  cdpUrl = `${baseUrl}${pageId}`;
+                  console.log(`Chrome debug port detected, using CDP connection to: ${pageId}`);
+                }
+              }
+            }
+          }
+        } catch (error) {
+          console.log('Chrome debug port not accessible, will start new browser instance');
+        }
+        
+        const config = {
+          connectOverCdp: useCdp,
+          cdpUrl: useCdp ? cdpUrl : undefined,
+          headless: false,
+          ...params
+        };
+        
+        console.log('Final config:', JSON.stringify(config, null, 2));
+        this.toolkit = new HybridBrowserToolkit(config);
+        return { message: 'Toolkit initialized with CDP connection' };
 
       case 'open_browser':
         if (!this.toolkit) throw new Error('Toolkit not initialized');

camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py

@@ -110,6 +110,11 @@ class WebSocketBrowserWrapper:
         self.process: Optional[subprocess.Popen] = None
         self.websocket = None
         self.server_port = None
+        self._send_lock = asyncio.Lock()  # Lock for sending messages
+        self._receive_task = None  # Background task for receiving messages
+        self._pending_responses: Dict[
+            str, asyncio.Future[Dict[str, Any]]
+        ] = {}  # Message ID -> Future
 
         # Logging configuration
         self.browser_log_to_file = (config or {}).get(
@@ -251,11 +256,22 @@ class WebSocketBrowserWrapper:
                 f"Failed to connect to WebSocket server: {e}"
             ) from e
 
+        # Start the background receiver task
+        self._receive_task = asyncio.create_task(self._receive_loop())
+
         # Initialize the browser toolkit
         await self._send_command('init', self.config)
 
     async def stop(self):
         """Stop the WebSocket connection and server."""
+        # Cancel the receiver task
+        if self._receive_task and not self._receive_task.done():
+            self._receive_task.cancel()
+            try:
+                await self._receive_task
+            except asyncio.CancelledError:
+                pass
+
         if self.websocket:
             try:
                 await self._send_command('shutdown', {})
@@ -327,6 +343,39 @@ class WebSocketBrowserWrapper:
         except Exception as e:
             logger.error(f"Failed to write to log file: {e}")
 
+    async def _receive_loop(self):
+        r"""Background task to receive messages from WebSocket."""
+        try:
+            while self.websocket:
+                try:
+                    response_data = await self.websocket.recv()
+                    response = json.loads(response_data)
+
+                    message_id = response.get('id')
+                    if message_id and message_id in self._pending_responses:
+                        # Set the result for the waiting coroutine
+                        future = self._pending_responses.pop(message_id)
+                        if not future.done():
+                            future.set_result(response)
+                    else:
+                        # Log unexpected messages
+                        logger.warning(
+                            f"Received unexpected message: {response}"
+                        )
+
+                except asyncio.CancelledError:
+                    break
+                except Exception as e:
+                    logger.error(f"Error in receive loop: {e}")
+                    # Notify all pending futures of the error
+                    for future in self._pending_responses.values():
+                        if not future.done():
+                            future.set_exception(e)
+                    self._pending_responses.clear()
+                    break
+        finally:
+            logger.debug("Receive loop terminated")
+
     async def _ensure_connection(self) -> None:
         """Ensure WebSocket connection is alive."""
         if not self.websocket:
@@ -350,39 +399,39 @@ class WebSocketBrowserWrapper:
         message_id = str(uuid.uuid4())
         message = {'id': message_id, 'command': command, 'params': params}
 
-        try:
-            # Send command
-            if self.websocket is None:
-                raise RuntimeError("WebSocket connection not established")
-            await self.websocket.send(json.dumps(message))
+        # Create a future for this message
+        future: asyncio.Future[Dict[str, Any]] = asyncio.Future()
+        self._pending_responses[message_id] = future
 
-            # Wait for response with matching ID
-            while True:
-                try:
-                    if self.websocket is None:
-                        raise RuntimeError("WebSocket connection lost")
-                    response_data = await asyncio.wait_for(
-                        self.websocket.recv(), timeout=60.0
-                    )
-                    response = json.loads(response_data)
-
-                    # Check if this is the response we're waiting for
-                    if response.get('id') == message_id:
-                        if not response.get('success'):
-                            raise RuntimeError(
-                                f"Command failed: {response.get('error')}"
-                            )
-                        return response['result']
+        try:
+            # Use lock only for sending to prevent interleaved messages
+            async with self._send_lock:
+                if self.websocket is None:
+                    raise RuntimeError("WebSocket connection not established")
+                await self.websocket.send(json.dumps(message))
+
+            # Wait for response (no lock needed, handled by background
+            # receiver)
+            try:
+                response = await asyncio.wait_for(future, timeout=60.0)
 
-                except asyncio.TimeoutError:
+                if not response.get('success'):
                     raise RuntimeError(
-                        f"Timeout waiting for response to command: {command}"
+                        f"Command failed: {response.get('error')}"
                     )
-                except json.JSONDecodeError as e:
-                    logger.warning(f"Failed to decode WebSocket response: {e}")
-                    continue
+                return response['result']
+
+            except asyncio.TimeoutError:
+                # Remove from pending if timeout
+                self._pending_responses.pop(message_id, None)
+                raise RuntimeError(
+                    f"Timeout waiting for response to command: {command}"
+                )
 
         except Exception as e:
+            # Clean up the pending response
+            self._pending_responses.pop(message_id, None)
+
             # Check if it's a connection closed error
             if (
                 "close frame" in str(e)

camel/toolkits/hybrid_browser_toolkit_py/__init__.py

@@ -0,0 +1,17 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from .hybrid_browser_toolkit import HybridBrowserToolkit
+
+__all__ = ["HybridBrowserToolkit"]

camel/toolkits/hybrid_browser_toolkit_py/actions.py

@@ -0,0 +1,417 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from .config_loader import ConfigLoader
+
+if TYPE_CHECKING:
+    from playwright.async_api import Page
+
+
+class ActionExecutor:
+    r"""Executes high-level actions (click, type …) on a Playwright Page."""
+
+    def __init__(
+        self,
+        page: "Page",
+        session: Optional[Any] = None,
+        default_timeout: Optional[int] = None,
+        short_timeout: Optional[int] = None,
+        max_scroll_amount: Optional[int] = None,
+    ):
+        self.page = page
+        self.session = session  # HybridBrowserSession instance
+
+        # Configure timeouts using the config file with optional overrides
+        self.default_timeout = ConfigLoader.get_action_timeout(default_timeout)
+        self.short_timeout = ConfigLoader.get_short_timeout(short_timeout)
+        self.max_scroll_amount = ConfigLoader.get_max_scroll_amount(
+            max_scroll_amount
+        )
+
+    # ------------------------------------------------------------------
+    # Public helpers
+    # ------------------------------------------------------------------
+    async def execute(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Execute an action and return detailed result information."""
+        if not action:
+            return {
+                "success": False,
+                "message": "No action to execute",
+                "details": {},
+            }
+
+        action_type = action.get("type")
+        if not action_type:
+            return {
+                "success": False,
+                "message": "Error: action has no type",
+                "details": {},
+            }
+
+        try:
+            # small helper to ensure basic stability
+            # await self._wait_dom_stable()
+
+            handler = {
+                "click": self._click,
+                "type": self._type,
+                "select": self._select,
+                "wait": self._wait,
+                "extract": self._extract,
+                "scroll": self._scroll,
+                "enter": self._enter,
+            }.get(action_type)
+
+            if handler is None:
+                return {
+                    "success": False,
+                    "message": f"Error: Unknown action type '{action_type}'",
+                    "details": {"action_type": action_type},
+                }
+
+            result = await handler(action)
+            return {
+                "success": True,
+                "message": result["message"],
+                "details": result.get("details", {}),
+            }
+        except Exception as exc:
+            return {
+                "success": False,
+                "message": f"Error executing {action_type}: {exc}",
+                "details": {"action_type": action_type, "error": str(exc)},
+            }
+
+    # ------------------------------------------------------------------
+    # Internal handlers
+    # ------------------------------------------------------------------
+    async def _click(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle click actions with new tab support for any clickable
+        element."""
+        ref = action.get("ref")
+        text = action.get("text")
+        selector = action.get("selector")
+        if not (ref or text or selector):
+            return {
+                "message": "Error: click requires ref/text/selector",
+                "details": {"error": "missing_selector"},
+            }
+
+        # Build strategies in priority order
+        strategies = []
+        if ref:
+            strategies.append(f"[aria-ref='{ref}']")
+        if selector:
+            strategies.append(selector)
+        if text:
+            strategies.append(f'text="{text}"')
+
+        details: Dict[str, Any] = {
+            "ref": ref,
+            "selector": selector,
+            "text": text,
+            "strategies_tried": [],
+            "successful_strategy": None,
+            "click_method": None,
+            "new_tab_created": False,
+        }
+
+        # Find the first valid selector
+        found_selector = None
+        for sel in strategies:
+            if await self.page.locator(sel).count() > 0:
+                found_selector = sel
+                break
+
+        if not found_selector:
+            details['error'] = "Element not found with any strategy"
+            return {
+                "message": "Error: Click failed, element not found",
+                "details": details,
+            }
+
+        element = self.page.locator(found_selector).first
+        details['successful_strategy'] = found_selector
+
+        # Attempt ctrl+click first (always)
+        try:
+            if self.session:
+                async with self.page.context.expect_page(
+                    timeout=self.short_timeout
+                ) as new_page_info:
+                    await element.click(modifiers=["ControlOrMeta"])
+                new_page = await new_page_info.value
+                await new_page.wait_for_load_state('domcontentloaded')
+                new_tab_index = await self.session.register_page(new_page)
+                if new_tab_index is not None:
+                    await self.session.switch_to_tab(new_tab_index)
+                    self.page = new_page
+                details.update(
+                    {
+                        "click_method": "ctrl_click_new_tab",
+                        "new_tab_created": True,
+                        "new_tab_index": new_tab_index,
+                    }
+                )
+                return {
+                    "message": f"Clicked element (ctrl click), opened in new "
+                    f"tab {new_tab_index}",
+                    "details": details,
+                }
+            else:
+                await element.click(modifiers=["ControlOrMeta"])
+                details["click_method"] = "ctrl_click_no_session"
+                return {
+                    "message": f"Clicked element (ctrl click, no"
+                    f" session): {found_selector}",
+                    "details": details,
+                }
+        except asyncio.TimeoutError:
+            # No new tab was opened, click may have still worked
+            details["click_method"] = "ctrl_click_same_tab"
+            return {
+                "message": f"Clicked element (ctrl click, "
+                f"same tab): {found_selector}",
+                "details": details,
+            }
+        except Exception as e:
+            details['strategies_tried'].append(
+                {
+                    'selector': found_selector,
+                    'method': 'ctrl_click',
+                    'error': str(e),
+                }
+            )
+            # Fall through to fallback
+
+        # Fallback to normal force click if ctrl+click fails
+        try:
+            await element.click(force=True, timeout=self.default_timeout)
+            details["click_method"] = "playwright_force_click"
+            return {
+                "message": f"Fallback clicked element: {found_selector}",
+                "details": details,
+            }
+        except Exception as e:
+            details["click_method"] = "playwright_force_click_failed"
+            details["error"] = str(e)
+            return {
+                "message": f"Error: All click strategies "
+                f"failed for {found_selector}",
+                "details": details,
+            }
+
+    async def _type(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle typing text into input fields."""
+        ref = action.get("ref")
+        selector = action.get("selector")
+        text = action.get("text", "")
+        if not (ref or selector):
+            return {
+                "message": "Error: type requires ref/selector",
+                "details": {"error": "missing_selector"},
+            }
+
+        target = selector or f"[aria-ref='{ref}']"
+        details = {
+            "ref": ref,
+            "selector": selector,
+            "target": target,
+            "text": text,
+            "text_length": len(text),
+        }
+
+        try:
+            await self.page.fill(target, text, timeout=self.short_timeout)
+            return {
+                "message": f"Typed '{text}' into {target}",
+                "details": details,
+            }
+        except Exception as exc:
+            details["error"] = str(exc)
+            return {"message": f"Type failed: {exc}", "details": details}
+
+    async def _select(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle selecting options from dropdowns."""
+        ref = action.get("ref")
+        selector = action.get("selector")
+        value = action.get("value", "")
+        if not (ref or selector):
+            return {
+                "message": "Error: select requires ref/selector",
+                "details": {"error": "missing_selector"},
+            }
+
+        target = selector or f"[aria-ref='{ref}']"
+        details = {
+            "ref": ref,
+            "selector": selector,
+            "target": target,
+            "value": value,
+        }
+
+        try:
+            await self.page.select_option(
+                target, value, timeout=self.default_timeout
+            )
+            return {
+                "message": f"Selected '{value}' in {target}",
+                "details": details,
+            }
+        except Exception as exc:
+            details["error"] = str(exc)
+            return {"message": f"Select failed: {exc}", "details": details}
+
+    async def _wait(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle wait actions."""
+        details: Dict[str, Any] = {
+            "wait_type": None,
+            "timeout": None,
+            "selector": None,
+        }
+
+        if "timeout" in action:
+            ms = int(action["timeout"])
+            details["wait_type"] = "timeout"
+            details["timeout"] = ms
+            await asyncio.sleep(ms / 1000)
+            return {"message": f"Waited {ms}ms", "details": details}
+        if "selector" in action:
+            sel = action["selector"]
+            details["wait_type"] = "selector"
+            details["selector"] = sel
+            await self.page.wait_for_selector(
+                sel, timeout=self.default_timeout
+            )
+            return {"message": f"Waited for {sel}", "details": details}
+        return {
+            "message": "Error: wait requires timeout/selector",
+            "details": details,
+        }
+
+    async def _extract(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle text extraction from elements."""
+        ref = action.get("ref")
+        if not ref:
+            return {
+                "message": "Error: extract requires ref",
+                "details": {"error": "missing_ref"},
+            }
+
+        target = f"[aria-ref='{ref}']"
+        details = {"ref": ref, "target": target}
+
+        await self.page.wait_for_selector(target, timeout=self.default_timeout)
+        txt = await self.page.text_content(target)
+
+        details["extracted_text"] = txt
+        details["text_length"] = len(txt) if txt else 0
+
+        return {
+            "message": f"Extracted: {txt[:100] if txt else 'None'}",
+            "details": details,
+        }
+
+    async def _scroll(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle page scrolling with safe parameter validation."""
+        direction = action.get("direction", "down")
+        amount = action.get("amount", 300)
+
+        details = {
+            "direction": direction,
+            "requested_amount": amount,
+            "actual_amount": None,
+            "scroll_offset": None,
+        }
+
+        # Validate inputs to prevent injection
+        if direction not in ("up", "down"):
+            return {
+                "message": "Error: direction must be 'up' or 'down'",
+                "details": details,
+            }
+
+        try:
+            # Safely convert amount to integer and clamp to reasonable range
+            amount_int = int(amount)
+            amount_int = max(
+                -self.max_scroll_amount,
+                min(self.max_scroll_amount, amount_int),
+            )  # Clamp to max_scroll_amount range
+            details["actual_amount"] = amount_int
+        except (ValueError, TypeError):
+            return {
+                "message": "Error: amount must be a valid number",
+                "details": details,
+            }
+
+        # Use safe evaluation with bound parameters
+        scroll_offset = amount_int if direction == "down" else -amount_int
+        details["scroll_offset"] = scroll_offset
+
+        await self.page.evaluate(
+            "offset => window.scrollBy(0, offset)", scroll_offset
+        )
+        await asyncio.sleep(0.5)
+        return {
+            "message": f"Scrolled {direction} by {abs(amount_int)}px",
+            "details": details,
+        }
+
+    async def _enter(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle Enter key press on the currently focused element."""
+        details = {"action_type": "enter", "target": "focused_element"}
+
+        # Press Enter on whatever element currently has focus
+        await self.page.keyboard.press("Enter")
+        return {
+            "message": "Pressed Enter on focused element",
+            "details": details,
+        }
+
+    # utilities
+    async def _wait_dom_stable(self) -> None:
+        r"""Wait for DOM to become stable before executing actions."""
+        try:
+            # Wait for basic DOM content loading
+            await self.page.wait_for_load_state(
+                'domcontentloaded', timeout=self.short_timeout
+            )
+
+            # Try to wait for network idle briefly
+            try:
+                await self.page.wait_for_load_state(
+                    'networkidle', timeout=self.short_timeout
+                )
+            except Exception:
+                pass  # Network idle is optional
+
+        except Exception:
+            pass  # Don't fail if wait times out
+
+    # static helpers
+    @staticmethod
+    def should_update_snapshot(action: Dict[str, Any]) -> bool:
+        r"""Determine if an action requires a snapshot update."""
+        change_types = {
+            "click",
+            "type",
+            "select",
+            "scroll",
+            "navigate",
+            "enter",
+        }
+        return action.get("type") in change_types