camel-ai 0.2.67__py3-none-any.whl → 0.2.69a1__py3-none-any.whl

camel/toolkits/non_visual_browser_toolkit/actions.py

@@ -0,0 +1,196 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+from typing import TYPE_CHECKING, Any, Dict
+
+if TYPE_CHECKING:
+    from playwright.async_api import Page
+
+
+class ActionExecutor:
+    r"""Executes high-level actions (click, type …) on a Playwright Page."""
+
+    # Configuration constants
+    DEFAULT_TIMEOUT = 5000  # 5 seconds
+    SHORT_TIMEOUT = 2000  # 2 seconds
+
+    def __init__(self, page: "Page"):
+        self.page = page
+
+    # ------------------------------------------------------------------
+    # Public helpers
+    # ------------------------------------------------------------------
+    async def execute(self, action: Dict[str, Any]) -> str:
+        if not action:
+            return "No action to execute"
+
+        action_type = action.get("type")
+        if not action_type:
+            return "Error: action has no type"
+
+        try:
+            # small helper to ensure basic stability
+            await self._wait_dom_stable()
+
+            handler = {
+                "click": self._click,
+                "type": self._type,
+                "select": self._select,
+                "wait": self._wait,
+                "extract": self._extract,
+                "scroll": self._scroll,
+                "enter": self._enter,
+            }.get(action_type)
+
+            if handler is None:
+                return f"Error: Unknown action type '{action_type}'"
+
+            return await handler(action)
+        except Exception as exc:
+            return f"Error executing {action_type}: {exc}"
+
+    # ------------------------------------------------------------------
+    # Internal handlers
+    # ------------------------------------------------------------------
+    async def _click(self, action: Dict[str, Any]) -> str:
+        ref = action.get("ref")
+        text = action.get("text")
+        selector = action.get("selector")
+        if not (ref or text or selector):
+            return "Error: click requires ref/text/selector"
+
+        strategies = []
+        if selector:
+            strategies.append(selector)
+        if text:
+            strategies.append(f'text="{text}"')
+        if ref:
+            strategies.append(f"[aria-ref='{ref}']")
+
+        for sel in strategies:
+            try:
+                if await self.page.locator(sel).count() > 0:
+                    await self.page.click(
+                        sel, timeout=self.SHORT_TIMEOUT, force=True
+                    )
+                    return f"Clicked element via {sel}"
+            except Exception:
+                pass
+        return "Error: Could not click element"
+
+    async def _type(self, action: Dict[str, Any]) -> str:
+        ref = action.get("ref")
+        selector = action.get("selector")
+        text = action.get("text", "")
+        if not (ref or selector):
+            return "Error: type requires ref/selector"
+        target = selector or f"[aria-ref='{ref}']"
+        try:
+            await self.page.fill(target, text, timeout=self.SHORT_TIMEOUT)
+            return f"Typed '{text}' into {target}"
+        except Exception as exc:
+            return f"Type failed: {exc}"
+
+    async def _select(self, action: Dict[str, Any]) -> str:
+        ref = action.get("ref")
+        selector = action.get("selector")
+        value = action.get("value", "")
+        if not (ref or selector):
+            return "Error: select requires ref/selector"
+        target = selector or f"[aria-ref='{ref}']"
+        try:
+            await self.page.select_option(
+                target, value, timeout=self.DEFAULT_TIMEOUT
+            )
+            return f"Selected '{value}' in {target}"
+        except Exception as exc:
+            return f"Select failed: {exc}"
+
+    async def _wait(self, action: Dict[str, Any]) -> str:
+        if "timeout" in action:
+            ms = action["timeout"]
+            await asyncio.sleep(ms / 1000)
+            return f"Waited {ms}ms"
+        if "selector" in action:
+            sel = action["selector"]
+            await self.page.wait_for_selector(
+                sel, timeout=self.DEFAULT_TIMEOUT
+            )
+            return f"Waited for {sel}"
+        return "Error: wait requires timeout/selector"
+
+    async def _extract(self, action: Dict[str, Any]) -> str:
+        ref = action.get("ref")
+        if not ref:
+            return "Error: extract requires ref"
+        target = f"[aria-ref='{ref}']"
+        await self.page.wait_for_selector(target, timeout=self.DEFAULT_TIMEOUT)
+        txt = await self.page.text_content(target)
+        return f"Extracted: {txt[:100] if txt else 'None'}"
+
+    async def _scroll(self, action: Dict[str, Any]) -> str:
+        direction = action.get("direction", "down")
+        amount = action.get("amount", 300)
+
+        # Validate inputs to prevent injection
+        if direction not in ("up", "down"):
+            return "Error: direction must be 'up' or 'down'"
+
+        try:
+            # Safely convert amount to integer and clamp to reasonable range
+            amount_int = int(amount)
+            amount_int = max(
+                -5000, min(5000, amount_int)
+            )  # Clamp between -5000 and 5000
+        except (ValueError, TypeError):
+            return "Error: amount must be a valid number"
+
+        # Use safe evaluation with bound parameters
+        scroll_offset = amount_int if direction == "down" else -amount_int
+        await self.page.evaluate(f"window.scrollBy(0, {scroll_offset})")
+        await asyncio.sleep(0.5)
+        return f"Scrolled {direction} by {abs(amount_int)}px"
+
+    async def _enter(self, action: Dict[str, Any]) -> str:
+        ref = action.get("ref")
+        selector = action.get("selector")
+        if ref:
+            await self.page.focus(f"[aria-ref='{ref}']")
+        elif selector:
+            await self.page.focus(selector)
+        await self.page.keyboard.press("Enter")
+        await asyncio.sleep(0.3)
+        return "Pressed Enter"
+
+    # utilities
+    async def _wait_dom_stable(self) -> None:
+        try:
+            await self.page.wait_for_load_state(
+                'domcontentloaded', timeout=self.SHORT_TIMEOUT
+            )
+        except Exception:
+            pass
+
+    # static helpers
+    @staticmethod
+    def should_update_snapshot(action: Dict[str, Any]) -> bool:
+        change_types = {
+            "click",
+            "type",
+            "select",
+            "scroll",
+            "navigate",
+            "enter",
+        }
+        return action.get("type") in change_types

camel/toolkits/non_visual_browser_toolkit/agent.py

@@ -0,0 +1,278 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import logging
+import re
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from camel.models import BaseModelBackend, ModelFactory
+from camel.types import ModelPlatformType, ModelType
+
+from .actions import ActionExecutor
+from .nv_browser_session import NVBrowserSession
+
+if TYPE_CHECKING:
+    from camel.agents import ChatAgent
+
+logger = logging.getLogger(__name__)
+
+
+class PlaywrightLLMAgent:
+    """High-level orchestration: snapshot ↔ LLM ↔ action executor."""
+
+    # System prompt as class constant to avoid recreation
+    SYSTEM_PROMPT = """
+You are a web automation assistant.
+
+" Analyse the page snapshot and create a short high-level plan, "
+"then output the FIRST action to start with.\n\n"
+"Return a JSON object in *exactly* this shape:\n"
+"Action format json_object examples:\n"
+"{\n  \"plan\": [\"Step 1\", \"Step 2\"],\n  \"action\": {\n    \"type\": 
+\"click\",\n    \"ref\": \"e1\"\n  }\n}\n\n"
+"If task is already complete:\n"
+"{\n  \"plan\": [],\n  \"action\": {\n    \"type\": \"finish\",
+\n    \"ref\": null,\n    \"summary\": \"Task was already completed. Summary 
+of what was found...\"\n  }\n}"
+
+Available action types:
+- 'click': {"type": "click", "ref": "e1"} or {"type": "click", "text": 
+"Button Text"} or {"type": "click", "selector": "button"}
+- 'type': {"type": "type", "ref": "e1", "text": "search text"} or {"type": 
+"type", "selector": "input", "text": "search text"}
+- 'select': {"type": "select", "ref": "e1", "value": "option"} or {"type": 
+"select", "selector": "select", "value": "option"}
+- 'wait': {"type": "wait", "timeout": 2000} or {"type": "wait", "selector": 
+"#element"}
+- 'scroll': {"type": "scroll", "direction": "down", "amount": 300}
+- 'enter': {"type": "enter", "ref": "e1"} or {"type": "enter", "selector": 
+"input[name=q]"} or {"type": "enter"}
+- 'navigate': {"type": "navigate", "url": "https://example.com"}
+- 'finish': {"type": "finish", "ref": null, "summary": "task completion 
+summary"}
+
+IMPORTANT: 
+- For 'click': Use 'ref' from snapshot, or 'text' for visible text, 
+or 'selector' for CSS selectors
+- For 'type'/'select': Use 'ref' from snapshot or 'selector' for CSS selectors
+- Only use 'ref' values that exist in the snapshot (e.g., ref=e1, ref=e2, etc.)
+- Use 'finish' when the task is completed successfully with a summary of 
+what was accomplished
+- Use 'enter' to press the Enter key (optionally focus an element first)
+- Use 'navigate' to open a new URL before interacting further
+- click can choose radio, checkbox...
+        """
+
+    def __init__(
+        self,
+        *,
+        user_data_dir: Optional[str] = None,
+        headless: bool = False,
+        model_backend: Optional[BaseModelBackend] = None,
+    ):
+        self._session = NVBrowserSession(
+            headless=headless, user_data_dir=user_data_dir
+        )
+        from camel.agents import ChatAgent
+
+        # Populated lazily after first page load
+        self.action_history: List[Dict[str, Any]] = []
+        if model_backend is None:
+            model_backend = ModelFactory.create(
+                model_platform=ModelPlatformType.OPENAI,
+                model_type=ModelType.GPT_4O_MINI,
+                model_config_dict={"temperature": 0, "top_p": 1},
+            )
+        self.model_backend = model_backend
+        # Reuse ChatAgent instance to avoid recreation overhead
+        self._chat_agent: Optional[ChatAgent] = None
+
+    async def navigate(self, url: str) -> str:
+        try:
+            # NVBrowserSession handles waits internally
+            logger.debug("Navigated to URL: %s", url)
+            await self._session.visit(url)
+            return await self._session.get_snapshot(force_refresh=True)
+        except Exception as exc:
+            return f"Error: could not navigate - {exc}"
+
+    def _get_chat_agent(self) -> "ChatAgent":
+        """Get or create the ChatAgent instance."""
+        from camel.agents import ChatAgent
+
+        if self._chat_agent is None:
+            self._chat_agent = ChatAgent(
+                system_message=self.SYSTEM_PROMPT, model=self.model_backend
+            )
+        return self._chat_agent
+
+    def _safe_parse_json(self, content: str) -> Dict[str, Any]:
+        r"""Safely parse JSON from LLM response with multiple fallback
+        strategies.
+        """
+        # First attempt: direct parsing
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            pass
+
+        # Second attempt: extract JSON-like block using regex
+        # Look for content between outermost braces
+        json_pattern = re.compile(
+            r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', re.DOTALL
+        )
+        json_matches = json_pattern.findall(content)
+
+        for match in json_matches:
+            try:
+                return json.loads(match)
+            except json.JSONDecodeError:
+                continue
+
+        # Third attempt: try to find and parse line by line
+        lines = content.split('\n')
+        json_lines = []
+        in_json = False
+
+        for line in lines:
+            line = line.strip()
+            if line.startswith('{'):
+                in_json = True
+                json_lines = [line]
+            elif in_json:
+                json_lines.append(line)
+                if line.endswith('}'):
+                    try:
+                        json_text = '\n'.join(json_lines)
+                        return json.loads(json_text)
+                    except json.JSONDecodeError:
+                        pass
+                    in_json = False
+                    json_lines = []
+
+        # Fallback: return default structure
+        logger.warning(
+            "Could not parse JSON from LLM response: %s", content[:200]
+        )
+        return {
+            "plan": ["Could not parse response"],
+            "action": {
+                "type": "finish",
+                "ref": None,
+                "summary": "Parsing error",
+            },
+        }
+
+    def _llm_call(
+        self,
+        prompt: str,
+        snapshot: str,
+        is_initial: bool,
+        history: Optional[List[Dict[str, Any]]] = None,
+    ) -> Dict[str, Any]:
+        """Call the LLM (via CAMEL ChatAgent) to get plan & next action."""
+        # Build user message
+        if is_initial:
+            user_content = f"Snapshot:\n{snapshot}\n\nTask: {prompt}"
+        else:
+            hist_lines = [
+                (
+                    f"{i + 1}. {'✅' if h['success'] else '❌'} "
+                    f"{h['action']['type']} -> {h['result']}"
+                )
+                for i, h in enumerate(history or [])
+            ]
+            user_content = (
+                f"Snapshot:\n{snapshot}\n\nHistory:\n"
+                + "\n".join(hist_lines)
+                + f"\n\nTask: {prompt}"
+            )
+
+        # Run ChatAgent
+        chat_agent = self._get_chat_agent()
+        response = chat_agent.step(user_content)
+        content = response.msgs[0].content if response.msgs else "{}"
+
+        # Safely parse JSON response
+        return self._safe_parse_json(content)
+
+    async def process_command(self, prompt: str, max_steps: int = 15):
+        # initial full snapshot
+        full_snapshot = await self._session.get_snapshot()
+        assert self._session.snapshot is not None
+        meta = self._session.snapshot.last_info
+        logger.info("Initial snapshot priorities=%s", meta["priorities"])
+        logger.debug("Full snapshot:\n%s", full_snapshot)
+
+        plan_resp = self._llm_call(
+            prompt, full_snapshot or "", is_initial=True
+        )
+        plan = plan_resp.get("plan", [])
+        action = plan_resp.get("action")
+
+        logger.info("Plan generated: %s", json.dumps(plan, ensure_ascii=False))
+
+        steps = 0
+        while action and steps < max_steps:
+            if action.get("type") == "finish":
+                logger.info("Task finished: %s", action.get("summary", "Done"))
+                break
+
+            result = await self._run_action(action)
+            logger.debug("Executed action: %s | Result: %s", action, result)
+
+            self.action_history.append(
+                {
+                    "action": action,
+                    "result": result,
+                    "success": "Error" not in result,
+                }
+            )
+
+            diff_snapshot = await self._session.get_snapshot(
+                force_refresh=ActionExecutor.should_update_snapshot(action),
+                diff_only=True,
+            )
+            assert self._session.snapshot is not None
+            meta = self._session.snapshot.last_info
+            logger.debug(
+                "Snapshot after action (diff=%s):\n%s",
+                meta["is_diff"],
+                diff_snapshot,
+            )
+
+            # Update full snapshot if page changed
+            if meta["is_diff"] and not diff_snapshot.startswith(
+                "- Page Snapshot (no structural changes)"
+            ):
+                assert self._session.snapshot is not None
+                full_snapshot = self._session.snapshot.snapshot_data or ""
+
+            action = self._llm_call(
+                prompt,
+                full_snapshot or "",
+                is_initial=False,
+                history=self.action_history,
+            ).get("action")
+            steps += 1
+
+        logger.info("Process completed with %d steps", steps)
+
+    async def _run_action(self, action: Dict[str, Any]) -> str:
+        if action.get("type") == "navigate":
+            return await self.navigate(action.get("url", ""))
+        return await self._session.exec_action(action)
+
+    async def close(self):
+        await self._session.close()