camel-ai 0.2.68__py3-none-any.whl → 0.2.69a1__py3-none-any.whl

camel/toolkits/non_visual_browser_toolkit/agent.py

@@ -0,0 +1,278 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import logging
+import re
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from camel.models import BaseModelBackend, ModelFactory
+from camel.types import ModelPlatformType, ModelType
+
+from .actions import ActionExecutor
+from .nv_browser_session import NVBrowserSession
+
+if TYPE_CHECKING:
+    from camel.agents import ChatAgent
+
+logger = logging.getLogger(__name__)
+
+
+class PlaywrightLLMAgent:
+    """High-level orchestration: snapshot ↔ LLM ↔ action executor."""
+
+    # System prompt as class constant to avoid recreation
+    SYSTEM_PROMPT = """
+You are a web automation assistant.
+
+" Analyse the page snapshot and create a short high-level plan, "
+"then output the FIRST action to start with.\n\n"
+"Return a JSON object in *exactly* this shape:\n"
+"Action format json_object examples:\n"
+"{\n  \"plan\": [\"Step 1\", \"Step 2\"],\n  \"action\": {\n    \"type\": 
+\"click\",\n    \"ref\": \"e1\"\n  }\n}\n\n"
+"If task is already complete:\n"
+"{\n  \"plan\": [],\n  \"action\": {\n    \"type\": \"finish\",
+\n    \"ref\": null,\n    \"summary\": \"Task was already completed. Summary 
+of what was found...\"\n  }\n}"
+
+Available action types:
+- 'click': {"type": "click", "ref": "e1"} or {"type": "click", "text": 
+"Button Text"} or {"type": "click", "selector": "button"}
+- 'type': {"type": "type", "ref": "e1", "text": "search text"} or {"type": 
+"type", "selector": "input", "text": "search text"}
+- 'select': {"type": "select", "ref": "e1", "value": "option"} or {"type": 
+"select", "selector": "select", "value": "option"}
+- 'wait': {"type": "wait", "timeout": 2000} or {"type": "wait", "selector": 
+"#element"}
+- 'scroll': {"type": "scroll", "direction": "down", "amount": 300}
+- 'enter': {"type": "enter", "ref": "e1"} or {"type": "enter", "selector": 
+"input[name=q]"} or {"type": "enter"}
+- 'navigate': {"type": "navigate", "url": "https://example.com"}
+- 'finish': {"type": "finish", "ref": null, "summary": "task completion 
+summary"}
+
+IMPORTANT: 
+- For 'click': Use 'ref' from snapshot, or 'text' for visible text, 
+or 'selector' for CSS selectors
+- For 'type'/'select': Use 'ref' from snapshot or 'selector' for CSS selectors
+- Only use 'ref' values that exist in the snapshot (e.g., ref=e1, ref=e2, etc.)
+- Use 'finish' when the task is completed successfully with a summary of 
+what was accomplished
+- Use 'enter' to press the Enter key (optionally focus an element first)
+- Use 'navigate' to open a new URL before interacting further
+- click can choose radio, checkbox...
+        """
+
+    def __init__(
+        self,
+        *,
+        user_data_dir: Optional[str] = None,
+        headless: bool = False,
+        model_backend: Optional[BaseModelBackend] = None,
+    ):
+        self._session = NVBrowserSession(
+            headless=headless, user_data_dir=user_data_dir
+        )
+        from camel.agents import ChatAgent
+
+        # Populated lazily after first page load
+        self.action_history: List[Dict[str, Any]] = []
+        if model_backend is None:
+            model_backend = ModelFactory.create(
+                model_platform=ModelPlatformType.OPENAI,
+                model_type=ModelType.GPT_4O_MINI,
+                model_config_dict={"temperature": 0, "top_p": 1},
+            )
+        self.model_backend = model_backend
+        # Reuse ChatAgent instance to avoid recreation overhead
+        self._chat_agent: Optional[ChatAgent] = None
+
+    async def navigate(self, url: str) -> str:
+        try:
+            # NVBrowserSession handles waits internally
+            logger.debug("Navigated to URL: %s", url)
+            await self._session.visit(url)
+            return await self._session.get_snapshot(force_refresh=True)
+        except Exception as exc:
+            return f"Error: could not navigate - {exc}"
+
+    def _get_chat_agent(self) -> "ChatAgent":
+        """Get or create the ChatAgent instance."""
+        from camel.agents import ChatAgent
+
+        if self._chat_agent is None:
+            self._chat_agent = ChatAgent(
+                system_message=self.SYSTEM_PROMPT, model=self.model_backend
+            )
+        return self._chat_agent
+
+    def _safe_parse_json(self, content: str) -> Dict[str, Any]:
+        r"""Safely parse JSON from LLM response with multiple fallback
+        strategies.
+        """
+        # First attempt: direct parsing
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            pass
+
+        # Second attempt: extract JSON-like block using regex
+        # Look for content between outermost braces
+        json_pattern = re.compile(
+            r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', re.DOTALL
+        )
+        json_matches = json_pattern.findall(content)
+
+        for match in json_matches:
+            try:
+                return json.loads(match)
+            except json.JSONDecodeError:
+                continue
+
+        # Third attempt: try to find and parse line by line
+        lines = content.split('\n')
+        json_lines = []
+        in_json = False
+
+        for line in lines:
+            line = line.strip()
+            if line.startswith('{'):
+                in_json = True
+                json_lines = [line]
+            elif in_json:
+                json_lines.append(line)
+                if line.endswith('}'):
+                    try:
+                        json_text = '\n'.join(json_lines)
+                        return json.loads(json_text)
+                    except json.JSONDecodeError:
+                        pass
+                    in_json = False
+                    json_lines = []
+
+        # Fallback: return default structure
+        logger.warning(
+            "Could not parse JSON from LLM response: %s", content[:200]
+        )
+        return {
+            "plan": ["Could not parse response"],
+            "action": {
+                "type": "finish",
+                "ref": None,
+                "summary": "Parsing error",
+            },
+        }
+
+    def _llm_call(
+        self,
+        prompt: str,
+        snapshot: str,
+        is_initial: bool,
+        history: Optional[List[Dict[str, Any]]] = None,
+    ) -> Dict[str, Any]:
+        """Call the LLM (via CAMEL ChatAgent) to get plan & next action."""
+        # Build user message
+        if is_initial:
+            user_content = f"Snapshot:\n{snapshot}\n\nTask: {prompt}"
+        else:
+            hist_lines = [
+                (
+                    f"{i + 1}. {'✅' if h['success'] else '❌'} "
+                    f"{h['action']['type']} -> {h['result']}"
+                )
+                for i, h in enumerate(history or [])
+            ]
+            user_content = (
+                f"Snapshot:\n{snapshot}\n\nHistory:\n"
+                + "\n".join(hist_lines)
+                + f"\n\nTask: {prompt}"
+            )
+
+        # Run ChatAgent
+        chat_agent = self._get_chat_agent()
+        response = chat_agent.step(user_content)
+        content = response.msgs[0].content if response.msgs else "{}"
+
+        # Safely parse JSON response
+        return self._safe_parse_json(content)
+
+    async def process_command(self, prompt: str, max_steps: int = 15):
+        # initial full snapshot
+        full_snapshot = await self._session.get_snapshot()
+        assert self._session.snapshot is not None
+        meta = self._session.snapshot.last_info
+        logger.info("Initial snapshot priorities=%s", meta["priorities"])
+        logger.debug("Full snapshot:\n%s", full_snapshot)
+
+        plan_resp = self._llm_call(
+            prompt, full_snapshot or "", is_initial=True
+        )
+        plan = plan_resp.get("plan", [])
+        action = plan_resp.get("action")
+
+        logger.info("Plan generated: %s", json.dumps(plan, ensure_ascii=False))
+
+        steps = 0
+        while action and steps < max_steps:
+            if action.get("type") == "finish":
+                logger.info("Task finished: %s", action.get("summary", "Done"))
+                break
+
+            result = await self._run_action(action)
+            logger.debug("Executed action: %s | Result: %s", action, result)
+
+            self.action_history.append(
+                {
+                    "action": action,
+                    "result": result,
+                    "success": "Error" not in result,
+                }
+            )
+
+            diff_snapshot = await self._session.get_snapshot(
+                force_refresh=ActionExecutor.should_update_snapshot(action),
+                diff_only=True,
+            )
+            assert self._session.snapshot is not None
+            meta = self._session.snapshot.last_info
+            logger.debug(
+                "Snapshot after action (diff=%s):\n%s",
+                meta["is_diff"],
+                diff_snapshot,
+            )
+
+            # Update full snapshot if page changed
+            if meta["is_diff"] and not diff_snapshot.startswith(
+                "- Page Snapshot (no structural changes)"
+            ):
+                assert self._session.snapshot is not None
+                full_snapshot = self._session.snapshot.snapshot_data or ""
+
+            action = self._llm_call(
+                prompt,
+                full_snapshot or "",
+                is_initial=False,
+                history=self.action_history,
+            ).get("action")
+            steps += 1
+
+        logger.info("Process completed with %d steps", steps)
+
+    async def _run_action(self, action: Dict[str, Any]) -> str:
+        if action.get("type") == "navigate":
+            return await self.navigate(action.get("url", ""))
+        return await self._session.exec_action(action)
+
+    async def close(self):
+        await self._session.close()

camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py

@@ -0,0 +1,363 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from camel.models import BaseModelBackend
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+from .agent import PlaywrightLLMAgent
+
+# session wrapper
+from .nv_browser_session import NVBrowserSession
+
+
+class BrowserNonVisualToolkit(BaseToolkit):
+    r"""A lightweight, *non-visual* browser toolkit exposing primitive
+    Playwright actions as CAMEL `FunctionTool`s.
+    """
+
+    def __init__(
+        self,
+        *,
+        headless: bool = True,
+        user_data_dir: Optional[str] = None,
+        web_agent_model: Optional[BaseModelBackend] = None,
+    ) -> None:
+        super().__init__()
+        self._headless = headless
+        self._user_data_dir = user_data_dir
+        self.web_agent_model = web_agent_model  # Currently unused but kept
+        # for compatibility
+
+        # Encapsulated browser session
+        self._session = NVBrowserSession(
+            headless=headless, user_data_dir=user_data_dir
+        )
+
+        # Optional higher-level agent (only if user supplies model)
+        self._agent: Optional[PlaywrightLLMAgent] = None
+
+    def __del__(self):
+        r"""Ensure cleanup when toolkit is garbage collected."""
+        # Note: __del__ cannot be async, so we schedule cleanup if needed
+        import asyncio
+
+        try:
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                task = loop.create_task(self.close_browser())
+                # Don't wait for completion to avoid blocking
+                del task
+            else:
+                asyncio.run(self.close_browser())
+        except Exception:
+            pass  # Don't fail during garbage collection
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+    async def _ensure_browser(self):
+        await self._session.ensure_browser()
+
+    async def _require_page(self):
+        await self._session.ensure_browser()
+        return await self._session.get_page()
+
+    def _validate_ref(self, ref: str, method_name: str) -> None:
+        """Validate that ref parameter is a non-empty string."""
+        if not ref or not isinstance(ref, str):
+            raise ValueError(
+                f"{method_name}(): 'ref' must be a non-empty string, "
+                f"got: {ref}"
+            )
+
+    # ------------------------------------------------------------------
+    # Tool implementations
+    # ------------------------------------------------------------------
+    async def open_browser(
+        self, start_url: Optional[str] = None
+    ) -> Dict[str, str]:
+        r"""Launch a Playwright browser session.
+
+        Args:
+            start_url (Optional[str]): If provided, the page will navigate to
+                this URL immediately after the browser launches.
+
+        Returns:
+            Dict[str, str]: Keys: ``result`` for action outcome,
+            ``snapshot`` for full DOM snapshot.
+        """
+        await self._session.ensure_browser()
+        if start_url:
+            return await self.visit_page(start_url)
+        # If no start_url provided, still capture initial snapshot
+        snapshot = await self._session.get_snapshot(
+            force_refresh=True, diff_only=False
+        )
+        return {"result": "Browser session started.", "snapshot": snapshot}
+
+    async def close_browser(self) -> str:
+        r"""Terminate the current browser session and free all resources.
+
+        Returns:
+            str: Confirmation message.
+        """
+        # Close agent if it exists
+        if self._agent is not None:
+            try:
+                await self._agent.close()
+            except Exception:
+                pass  # Don't fail if agent cleanup fails
+            self._agent = None
+
+        # Close session
+        await self._session.close()
+        return "Browser session closed."
+
+    # Navigation / page state ------------------------------------------------
+    async def visit_page(self, url: str) -> Dict[str, str]:
+        """Navigate the current page to the specified URL.
+
+        Args:
+            url (str): The destination URL.
+
+        Returns:
+            Dict[str, str]: Keys: ``result`` for action outcome,
+            ``snapshot`` for full DOM snapshot.
+        """
+        if not url or not isinstance(url, str):
+            raise ValueError("visit_page(): 'url' must be a non-empty string")
+
+        nav_result = await self._session.visit(url)
+        snapshot = await self._session.get_snapshot(
+            force_refresh=True, diff_only=False
+        )
+        return {"result": nav_result, "snapshot": snapshot}
+
+    async def get_page_snapshot(
+        self, *, force_refresh: bool = False, diff_only: bool = False
+    ) -> str:
+        r"""Capture a YAML-like structural snapshot of the DOM.
+
+        Args:
+            force_refresh (bool): When ``True`` always re-generate the
+            snapshot even
+                if the URL has not changed.
+            diff_only (bool): If ``True`` return only the diff relative to the
+                previous snapshot.
+
+        Returns:
+            str: Formatted snapshot string.
+        """
+        return await self._session.get_snapshot(
+            force_refresh=force_refresh, diff_only=diff_only
+        )
+
+    # Element-level wrappers -------------------------------------------------
+    async def click(self, *, ref: str) -> Dict[str, str]:
+        r"""Click an element identified by ``ref``
+
+        Args:
+            ref (str): Element reference ID extracted from snapshot (e.g.
+            ``"e3"``).
+
+        Returns:
+            Dict[str, str]: Result message from ``ActionExecutor``.
+        """
+        self._validate_ref(ref, "click")
+
+        action: Dict[str, Any] = {"type": "click", "ref": ref}
+        return await self._exec_with_snapshot(action)
+
+    async def type(self, *, ref: str, text: str) -> Dict[str, str]:
+        r"""Type text into an input or textarea element.
+
+        Args:
+            ref (str): Element reference ID extracted from snapshot (e.g.
+            ``"e3"``).
+            text (str): The text to enter.
+
+        Returns:
+            Dict[str, str]: Execution result message.
+        """
+        self._validate_ref(ref, "type")
+
+        action: Dict[str, Any] = {"type": "type", "ref": ref, "text": text}
+        return await self._exec_with_snapshot(action)
+
+    async def select(self, *, ref: str, value: str) -> Dict[str, str]:
+        r"""Select an option in a ``<select>`` element.
+
+        Args:
+            ref (str): Element reference ID.
+            value (str): The value / option to select.
+
+        Returns:
+            Dict[str, str]: Execution result message.
+        """
+        self._validate_ref(ref, "select")
+
+        action: Dict[str, Any] = {"type": "select", "ref": ref, "value": value}
+        return await self._exec_with_snapshot(action)
+
+    async def scroll(self, *, direction: str, amount: int) -> Dict[str, str]:
+        r"""Scroll the page.
+
+        Args:
+            direction (str): ``"down"`` or ``"up"``.
+            amount (int): Pixel distance to scroll.
+
+        Returns:
+            Dict[str, str]: Execution result message.
+        """
+        if direction not in ("up", "down"):
+            raise ValueError("scroll(): 'direction' must be 'up' or 'down'")
+
+        action = {"type": "scroll", "direction": direction, "amount": amount}
+        return await self._exec_with_snapshot(action)
+
+    async def wait(
+        self, *, timeout_ms: int | None = None, selector: str | None = None
+    ) -> Dict[str, str]:
+        r"""Explicit wait utility.
+
+        Args:
+            timeout_ms (Optional[int]): Milliseconds to sleep.
+            selector (Optional[str]): Wait until this CSS selector appears
+                in DOM.
+
+        Returns:
+            Dict[str, str]: Execution result message.
+        """
+        # Default to 1 000 ms sleep when no arguments provided
+        if timeout_ms is None and selector is None:
+            timeout_ms = 1000
+
+        action: Dict[str, Any] = {"type": "wait"}
+        if timeout_ms is not None:
+            action["timeout"] = timeout_ms
+        if selector is not None:
+            action["selector"] = selector
+        return await self._exec_with_snapshot(action)
+
+    async def extract(self, *, ref: str) -> Dict[str, str]:
+        r"""Extract text content from an element.
+
+        Args:
+            ref (str): Element reference ID obtained from snapshot.
+
+        Returns:
+            Dict[str, str]: Extracted text or error message.
+        """
+        self._validate_ref(ref, "extract")
+        return await self._exec_with_snapshot({"type": "extract", "ref": ref})
+
+    async def enter(self, *, ref: str) -> Dict[str, str]:
+        r"""Press the Enter key.
+
+        Args:
+            ref (str): Element reference ID to focus before pressing.
+
+        Returns:
+            Dict[str, str]: Execution result message.
+        """
+        self._validate_ref(ref, "enter")
+
+        action: Dict[str, Any] = {"type": "enter", "ref": ref}
+        return await self._exec_with_snapshot(action)
+
+    # Helper to run through ActionExecutor
+    async def _exec(self, action: Dict[str, Any]) -> str:
+        return await self._session.exec_action(action)
+
+    async def _exec_with_snapshot(
+        self, action: Dict[str, Any]
+    ) -> Dict[str, str]:
+        r"""Execute action and, if DOM structure changed, include snapshot
+        diff.
+        """
+        result = await self._session.exec_action(action)
+
+        # Only capture diff if action type typically changes DOM
+        from .actions import ActionExecutor
+
+        if not ActionExecutor.should_update_snapshot(action):
+            return {"result": result}
+
+        # Capture structural diff to previous snapshot
+        diff = await self._session.get_snapshot(
+            force_refresh=True, diff_only=True
+        )
+
+        if diff.startswith("- Page Snapshot (no structural changes)"):
+            return {"result": result}
+
+        return {"result": result, "snapshot": diff}
+
+    # ------------------------------------------------------------------
+    # Optional PlaywrightLLMAgent helpers
+    # ------------------------------------------------------------------
+    def _ensure_agent(self) -> PlaywrightLLMAgent:
+        r"""Create PlaywrightLLMAgent on first use if `web_agent_model`
+        provided."""
+        if self.web_agent_model is None:
+            raise RuntimeError(
+                "web_agent_model not supplied - high-level task planning is "
+                "unavailable."
+            )
+
+        if self._agent is None:
+            self._agent = PlaywrightLLMAgent(
+                headless=self._headless,
+                user_data_dir=self._user_data_dir,
+                model_backend=self.web_agent_model,
+            )
+        return self._agent
+
+    async def solve_task(
+        self, task_prompt: str, start_url: str, max_steps: int = 15
+    ) -> str:
+        r"""Use LLM agent to autonomously complete the task (requires
+        `web_agent_model`)."""
+
+        agent = self._ensure_agent()
+        await agent.navigate(start_url)
+        await agent.process_command(task_prompt, max_steps=max_steps)
+        return "Task processing finished - see stdout for detailed trace."
+
+    # ------------------------------------------------------------------
+    # Toolkit registration
+    # ------------------------------------------------------------------
+    def get_tools(self) -> List[FunctionTool]:
+        base_tools = [
+            FunctionTool(self.open_browser),
+            FunctionTool(self.close_browser),
+            FunctionTool(self.visit_page),
+            FunctionTool(self.get_page_snapshot),
+            FunctionTool(self.click),
+            FunctionTool(self.type),
+            FunctionTool(self.select),
+            FunctionTool(self.scroll),
+            FunctionTool(self.wait),
+            FunctionTool(self.extract),
+            FunctionTool(self.enter),
+        ]
+
+        if self.web_agent_model is not None:
+            base_tools.append(FunctionTool(self.solve_task))
+
+        return base_tools