camel-ai 0.2.73a0__py3-none-any.whl → 0.2.73a2__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts

@@ -50,20 +50,28 @@ export class HybridBrowserSession {
       // Handle existing pages
       const pages = this.context.pages();
       if (pages.length > 0) {
-        // Map existing pages
+        // Map existing pages - for CDP, only use pages with about:blank URL
+        let availablePageFound = false;
         for (const page of pages) {
-          const tabId = this.generateTabId();
-          this.pages.set(tabId, page);
-          if (!this.currentTabId) {
-            this.currentTabId = tabId;
+          const pageUrl = page.url();
+          // In CDP mode, only consider pages with about:blank as available
+          if (pageUrl === 'about:blank') {
+            const tabId = this.generateTabId();
+            this.pages.set(tabId, page);
+            if (!this.currentTabId) {
+              this.currentTabId = tabId;
+              availablePageFound = true;
+            }
           }
         }
+        
+        // If no available blank pages found in CDP mode, we cannot create new ones
+        if (!availablePageFound) {
+          throw new Error('No available blank tabs found in CDP mode. The frontend should have pre-created blank tabs.');
+        }
       } else {
-        // Create initial page if none exist
-        const initialPage = await this.context.newPage();
-        const initialTabId = this.generateTabId();
-        this.pages.set(initialTabId, initialPage);
-        this.currentTabId = initialTabId;
+        // In CDP mode, newPage is not supported
+        throw new Error('No pages available in CDP mode and newPage() is not supported. Ensure the frontend has pre-created blank tabs.');
       }
     } else {
       // Original launch logic
@@ -626,18 +634,39 @@ export class HybridBrowserSession {
           throw new Error('Browser context not initialized');
         }
         
-        
         const navigationStart = Date.now();
         
-        // Create a new page (tab)
-        const newPage = await this.context.newPage();
+        // In CDP mode, find an available blank tab instead of creating new page
+        let newPage: Page | null = null;
+        let newTabId: string | null = null;
         
-        // Generate tab ID for the new page
-        const newTabId = this.generateTabId();
-        this.pages.set(newTabId, newPage);
+        const browserConfig = this.configLoader.getBrowserConfig();
+        if (browserConfig.connectOverCdp) {
+          // CDP mode: find an available blank tab
+          const allPages = this.context.pages();
+          for (const page of allPages) {
+            const pageUrl = page.url();
+            // Check if this page is not already tracked and is blank
+            const isTracked = Array.from(this.pages.values()).includes(page);
+            if (!isTracked && pageUrl === 'about:blank') {
+              newPage = page;
+              newTabId = this.generateTabId();
+              this.pages.set(newTabId, newPage);
+              break;
+            }
+          }
+          
+          if (!newPage || !newTabId) {
+            throw new Error('No available blank tabs in CDP mode. Frontend should create more blank tabs when half are used.');
+          }
+        } else {
+          // Non-CDP mode: create new page as usual
+          newPage = await this.context.newPage();
+          newTabId = this.generateTabId();
+          this.pages.set(newTabId, newPage);
+        }
         
         // Set up page properties
-        const browserConfig = this.configLoader.getBrowserConfig();
         newPage.setDefaultNavigationTimeout(browserConfig.navigationTimeout);
         newPage.setDefaultTimeout(browserConfig.navigationTimeout);
         

camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js

@@ -78,8 +78,52 @@ class WebSocketBrowserServer {
     switch (command) {
       case 'init':
         console.log('Initializing toolkit with params:', JSON.stringify(params, null, 2));
-        this.toolkit = new HybridBrowserToolkit(params);
-        return { message: 'Toolkit initialized' };
+        
+        // Check if CDP is available first
+        let useCdp = false;
+        let cdpUrl = params.cdpUrl || 'http://localhost:9222';
+        
+        // Extract base URL and port for validation
+        const baseUrl = cdpUrl.includes('/devtools/') ? cdpUrl.split('/devtools/')[0] : cdpUrl;
+        
+        try {
+          // Test if Chrome debug port is accessible and get page URL
+          const response = await fetch(`${baseUrl}/json`);
+          if (response.ok) {
+            const pages = await response.json();
+            if (pages && pages.length > 0) {
+              // If user provided a specific page URL, use it; otherwise use first available
+              if (cdpUrl.includes('/devtools/page/') || cdpUrl.includes('/devtools/browser/')) {
+                useCdp = true;
+                console.log(`Using provided CDP URL: ${cdpUrl}`);
+              } else {
+                // Use the first available page
+                const firstPage = pages[0];
+                const pageUrl = firstPage.devtoolsFrontendUrl;
+                const pageId = pageUrl.match(/ws=localhost:\d+(.*)$/)?.[1];
+                
+                if (pageId) {
+                  useCdp = true;
+                  cdpUrl = `${baseUrl}${pageId}`;
+                  console.log(`Chrome debug port detected, using CDP connection to: ${pageId}`);
+                }
+              }
+            }
+          }
+        } catch (error) {
+          console.log('Chrome debug port not accessible, will start new browser instance');
+        }
+        
+        const config = {
+          connectOverCdp: useCdp,
+          cdpUrl: useCdp ? cdpUrl : undefined,
+          headless: false,
+          ...params
+        };
+        
+        console.log('Final config:', JSON.stringify(config, null, 2));
+        this.toolkit = new HybridBrowserToolkit(config);
+        return { message: 'Toolkit initialized with CDP connection' };
 
       case 'open_browser':
         if (!this.toolkit) throw new Error('Toolkit not initialized');

camel/toolkits/hybrid_browser_toolkit_py/__init__.py

@@ -0,0 +1,17 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from .hybrid_browser_toolkit import HybridBrowserToolkit
+
+__all__ = ["HybridBrowserToolkit"]

camel/toolkits/hybrid_browser_toolkit_py/actions.py

@@ -0,0 +1,417 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from .config_loader import ConfigLoader
+
+if TYPE_CHECKING:
+    from playwright.async_api import Page
+
+
+class ActionExecutor:
+    r"""Executes high-level actions (click, type …) on a Playwright Page."""
+
+    def __init__(
+        self,
+        page: "Page",
+        session: Optional[Any] = None,
+        default_timeout: Optional[int] = None,
+        short_timeout: Optional[int] = None,
+        max_scroll_amount: Optional[int] = None,
+    ):
+        self.page = page
+        self.session = session  # HybridBrowserSession instance
+
+        # Configure timeouts using the config file with optional overrides
+        self.default_timeout = ConfigLoader.get_action_timeout(default_timeout)
+        self.short_timeout = ConfigLoader.get_short_timeout(short_timeout)
+        self.max_scroll_amount = ConfigLoader.get_max_scroll_amount(
+            max_scroll_amount
+        )
+
+    # ------------------------------------------------------------------
+    # Public helpers
+    # ------------------------------------------------------------------
+    async def execute(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Execute an action and return detailed result information."""
+        if not action:
+            return {
+                "success": False,
+                "message": "No action to execute",
+                "details": {},
+            }
+
+        action_type = action.get("type")
+        if not action_type:
+            return {
+                "success": False,
+                "message": "Error: action has no type",
+                "details": {},
+            }
+
+        try:
+            # small helper to ensure basic stability
+            # await self._wait_dom_stable()
+
+            handler = {
+                "click": self._click,
+                "type": self._type,
+                "select": self._select,
+                "wait": self._wait,
+                "extract": self._extract,
+                "scroll": self._scroll,
+                "enter": self._enter,
+            }.get(action_type)
+
+            if handler is None:
+                return {
+                    "success": False,
+                    "message": f"Error: Unknown action type '{action_type}'",
+                    "details": {"action_type": action_type},
+                }
+
+            result = await handler(action)
+            return {
+                "success": True,
+                "message": result["message"],
+                "details": result.get("details", {}),
+            }
+        except Exception as exc:
+            return {
+                "success": False,
+                "message": f"Error executing {action_type}: {exc}",
+                "details": {"action_type": action_type, "error": str(exc)},
+            }
+
+    # ------------------------------------------------------------------
+    # Internal handlers
+    # ------------------------------------------------------------------
+    async def _click(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle click actions with new tab support for any clickable
+        element."""
+        ref = action.get("ref")
+        text = action.get("text")
+        selector = action.get("selector")
+        if not (ref or text or selector):
+            return {
+                "message": "Error: click requires ref/text/selector",
+                "details": {"error": "missing_selector"},
+            }
+
+        # Build strategies in priority order
+        strategies = []
+        if ref:
+            strategies.append(f"[aria-ref='{ref}']")
+        if selector:
+            strategies.append(selector)
+        if text:
+            strategies.append(f'text="{text}"')
+
+        details: Dict[str, Any] = {
+            "ref": ref,
+            "selector": selector,
+            "text": text,
+            "strategies_tried": [],
+            "successful_strategy": None,
+            "click_method": None,
+            "new_tab_created": False,
+        }
+
+        # Find the first valid selector
+        found_selector = None
+        for sel in strategies:
+            if await self.page.locator(sel).count() > 0:
+                found_selector = sel
+                break
+
+        if not found_selector:
+            details['error'] = "Element not found with any strategy"
+            return {
+                "message": "Error: Click failed, element not found",
+                "details": details,
+            }
+
+        element = self.page.locator(found_selector).first
+        details['successful_strategy'] = found_selector
+
+        # Attempt ctrl+click first (always)
+        try:
+            if self.session:
+                async with self.page.context.expect_page(
+                    timeout=self.short_timeout
+                ) as new_page_info:
+                    await element.click(modifiers=["ControlOrMeta"])
+                new_page = await new_page_info.value
+                await new_page.wait_for_load_state('domcontentloaded')
+                new_tab_index = await self.session.register_page(new_page)
+                if new_tab_index is not None:
+                    await self.session.switch_to_tab(new_tab_index)
+                    self.page = new_page
+                details.update(
+                    {
+                        "click_method": "ctrl_click_new_tab",
+                        "new_tab_created": True,
+                        "new_tab_index": new_tab_index,
+                    }
+                )
+                return {
+                    "message": f"Clicked element (ctrl click), opened in new "
+                    f"tab {new_tab_index}",
+                    "details": details,
+                }
+            else:
+                await element.click(modifiers=["ControlOrMeta"])
+                details["click_method"] = "ctrl_click_no_session"
+                return {
+                    "message": f"Clicked element (ctrl click, no"
+                    f" session): {found_selector}",
+                    "details": details,
+                }
+        except asyncio.TimeoutError:
+            # No new tab was opened, click may have still worked
+            details["click_method"] = "ctrl_click_same_tab"
+            return {
+                "message": f"Clicked element (ctrl click, "
+                f"same tab): {found_selector}",
+                "details": details,
+            }
+        except Exception as e:
+            details['strategies_tried'].append(
+                {
+                    'selector': found_selector,
+                    'method': 'ctrl_click',
+                    'error': str(e),
+                }
+            )
+            # Fall through to fallback
+
+        # Fallback to normal force click if ctrl+click fails
+        try:
+            await element.click(force=True, timeout=self.default_timeout)
+            details["click_method"] = "playwright_force_click"
+            return {
+                "message": f"Fallback clicked element: {found_selector}",
+                "details": details,
+            }
+        except Exception as e:
+            details["click_method"] = "playwright_force_click_failed"
+            details["error"] = str(e)
+            return {
+                "message": f"Error: All click strategies "
+                f"failed for {found_selector}",
+                "details": details,
+            }
+
+    async def _type(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle typing text into input fields."""
+        ref = action.get("ref")
+        selector = action.get("selector")
+        text = action.get("text", "")
+        if not (ref or selector):
+            return {
+                "message": "Error: type requires ref/selector",
+                "details": {"error": "missing_selector"},
+            }
+
+        target = selector or f"[aria-ref='{ref}']"
+        details = {
+            "ref": ref,
+            "selector": selector,
+            "target": target,
+            "text": text,
+            "text_length": len(text),
+        }
+
+        try:
+            await self.page.fill(target, text, timeout=self.short_timeout)
+            return {
+                "message": f"Typed '{text}' into {target}",
+                "details": details,
+            }
+        except Exception as exc:
+            details["error"] = str(exc)
+            return {"message": f"Type failed: {exc}", "details": details}
+
+    async def _select(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle selecting options from dropdowns."""
+        ref = action.get("ref")
+        selector = action.get("selector")
+        value = action.get("value", "")
+        if not (ref or selector):
+            return {
+                "message": "Error: select requires ref/selector",
+                "details": {"error": "missing_selector"},
+            }
+
+        target = selector or f"[aria-ref='{ref}']"
+        details = {
+            "ref": ref,
+            "selector": selector,
+            "target": target,
+            "value": value,
+        }
+
+        try:
+            await self.page.select_option(
+                target, value, timeout=self.default_timeout
+            )
+            return {
+                "message": f"Selected '{value}' in {target}",
+                "details": details,
+            }
+        except Exception as exc:
+            details["error"] = str(exc)
+            return {"message": f"Select failed: {exc}", "details": details}
+
+    async def _wait(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle wait actions."""
+        details: Dict[str, Any] = {
+            "wait_type": None,
+            "timeout": None,
+            "selector": None,
+        }
+
+        if "timeout" in action:
+            ms = int(action["timeout"])
+            details["wait_type"] = "timeout"
+            details["timeout"] = ms
+            await asyncio.sleep(ms / 1000)
+            return {"message": f"Waited {ms}ms", "details": details}
+        if "selector" in action:
+            sel = action["selector"]
+            details["wait_type"] = "selector"
+            details["selector"] = sel
+            await self.page.wait_for_selector(
+                sel, timeout=self.default_timeout
+            )
+            return {"message": f"Waited for {sel}", "details": details}
+        return {
+            "message": "Error: wait requires timeout/selector",
+            "details": details,
+        }
+
+    async def _extract(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle text extraction from elements."""
+        ref = action.get("ref")
+        if not ref:
+            return {
+                "message": "Error: extract requires ref",
+                "details": {"error": "missing_ref"},
+            }
+
+        target = f"[aria-ref='{ref}']"
+        details = {"ref": ref, "target": target}
+
+        await self.page.wait_for_selector(target, timeout=self.default_timeout)
+        txt = await self.page.text_content(target)
+
+        details["extracted_text"] = txt
+        details["text_length"] = len(txt) if txt else 0
+
+        return {
+            "message": f"Extracted: {txt[:100] if txt else 'None'}",
+            "details": details,
+        }
+
+    async def _scroll(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle page scrolling with safe parameter validation."""
+        direction = action.get("direction", "down")
+        amount = action.get("amount", 300)
+
+        details = {
+            "direction": direction,
+            "requested_amount": amount,
+            "actual_amount": None,
+            "scroll_offset": None,
+        }
+
+        # Validate inputs to prevent injection
+        if direction not in ("up", "down"):
+            return {
+                "message": "Error: direction must be 'up' or 'down'",
+                "details": details,
+            }
+
+        try:
+            # Safely convert amount to integer and clamp to reasonable range
+            amount_int = int(amount)
+            amount_int = max(
+                -self.max_scroll_amount,
+                min(self.max_scroll_amount, amount_int),
+            )  # Clamp to max_scroll_amount range
+            details["actual_amount"] = amount_int
+        except (ValueError, TypeError):
+            return {
+                "message": "Error: amount must be a valid number",
+                "details": details,
+            }
+
+        # Use safe evaluation with bound parameters
+        scroll_offset = amount_int if direction == "down" else -amount_int
+        details["scroll_offset"] = scroll_offset
+
+        await self.page.evaluate(
+            "offset => window.scrollBy(0, offset)", scroll_offset
+        )
+        await asyncio.sleep(0.5)
+        return {
+            "message": f"Scrolled {direction} by {abs(amount_int)}px",
+            "details": details,
+        }
+
+    async def _enter(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        r"""Handle Enter key press on the currently focused element."""
+        details = {"action_type": "enter", "target": "focused_element"}
+
+        # Press Enter on whatever element currently has focus
+        await self.page.keyboard.press("Enter")
+        return {
+            "message": "Pressed Enter on focused element",
+            "details": details,
+        }
+
+    # utilities
+    async def _wait_dom_stable(self) -> None:
+        r"""Wait for DOM to become stable before executing actions."""
+        try:
+            # Wait for basic DOM content loading
+            await self.page.wait_for_load_state(
+                'domcontentloaded', timeout=self.short_timeout
+            )
+
+            # Try to wait for network idle briefly
+            try:
+                await self.page.wait_for_load_state(
+                    'networkidle', timeout=self.short_timeout
+                )
+            except Exception:
+                pass  # Network idle is optional
+
+        except Exception:
+            pass  # Don't fail if wait times out
+
+    # static helpers
+    @staticmethod
+    def should_update_snapshot(action: Dict[str, Any]) -> bool:
+        r"""Determine if an action requires a snapshot update."""
+        change_types = {
+            "click",
+            "type",
+            "select",
+            "scroll",
+            "navigate",
+            "enter",
+        }
+        return action.get("type") in change_types