gdmcode 0.1.0__py3-none-any.whl

src/tools/browser_tool.py

@@ -0,0 +1,352 @@
+"""Browser automation tools for gdm — DOM interaction, screenshots, navigation.
+
+These tools communicate with the running Chrome extension via the bridge server.
+Before using any browser tool, start the bridge: `gdm browser start` and load
+the gdm-chrome extension in Chrome.
+
+Three tools are registered at import time with a null bridge client:
+  - browser_dom    — fill, click, assert, read DOM
+  - browser_capture — take screenshots
+  - browser_nav    — navigate, wait for elements, reload
+
+The AgentLoop or /browser command wires up the BridgeClient once the bridge is running.
+Call set_bridge_client() to inject a live client into all three tools.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import uuid
+from pathlib import Path
+from typing import Any, ClassVar
+
+from src.tools import REGISTRY, ToolBase, ToolResult
+
+__all__ = [
+    "BrowserDomTool",
+    "BrowserCaptureTool",
+    "BrowserNavTool",
+    "set_bridge_client",
+]
+
+log = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_DEFAULT_ACTION_TIMEOUT_SECS: float = 10.0
+_SCREENSHOT_DIR_NAME: str = "screenshots"
+
+_DOM_ACTIONS: frozenset[str] = frozenset({
+    "click",
+    "fill",
+    "clear",
+    "type_text",
+    "select_option",
+    "keyboard_input",
+    "check",
+    "uncheck",
+    "hover",
+    "focus",
+    "assert_text",
+    "assert_visible",
+    "assert_hidden",
+    "assert_value",
+    "assert_checked",
+    "get_text",
+    "get_value",
+    "get_attribute",
+    # "eval" removed — arbitrary JS execution is prohibited (security)
+    "scroll_to",
+    "wait_for",
+})
+
+_NAV_ACTIONS: frozenset[str] = frozenset({
+    "navigate",
+    "reload",
+    "back",
+    "forward",
+    "wait",
+})
+
+# Shared bridge client — set once by set_bridge_client()
+_shared_bridge: Any = None
+
+
+def set_bridge_client(client: Any) -> None:
+    """Inject a live BridgeClient into all browser tools.
+
+    Call this after `gdm browser start` connects the bridge.
+    """
+    global _shared_bridge
+    _shared_bridge = client
+    log.info("BridgeClient wired into browser tools")
+
+
+# ---------------------------------------------------------------------------
+# Browser DOM tool
+# ---------------------------------------------------------------------------
+
+class BrowserDomTool(ToolBase):
+    """Interact with DOM elements in the browser via the gdm bridge.
+
+    Supports fill, click, assert, read, eval, and keyboard actions.
+    Requires `gdm browser start` and the gdm-chrome extension connected.
+    """
+
+    name: ClassVar[str] = "browser_dom"
+    description: ClassVar[str] = (
+        "Interact with the browser DOM: fill inputs, click buttons, assert text, "
+        "and read element values. "
+        "Requires the gdm bridge to be running (`gdm browser start`) and the "
+        "gdm-chrome extension loaded in Chrome."
+    )
+    input_schema: ClassVar[dict[str, Any]] = {
+        "type": "object",
+        "required": ["action"],
+        "properties": {
+            "action": {
+                "type": "string",
+                "enum": sorted(_DOM_ACTIONS),
+                "description": "The DOM action to perform.",
+            },
+            "selector": {
+                "type": "string",
+                "description": "CSS selector, XPath, or text-based selector.",
+            },
+            "value": {
+                "type": "string",
+                "description": "Value for fill/select_option/keyboard_input.",
+            },
+            "expected": {
+                "type": "string",
+                "description": "Expected value for assert_* actions.",
+            },
+            "attribute": {
+                "type": "string",
+                "description": "Attribute name for get_attribute.",
+            },
+            "exact": {
+                "type": "boolean",
+                "description": "Exact match for assert_text (default: substring).",
+            },
+            "timeout": {
+                "type": "number",
+                "description": "Action timeout in seconds (default: 10).",
+            },
+            "tab_id": {
+                "type": "integer",
+                "description": "Target tab ID (optional — uses bound tab if omitted).",
+            },
+        },
+        "additionalProperties": False,
+    }
+
+    def execute(self, params: dict[str, Any]) -> ToolResult:
+        action = params.get("action", "")
+        if action not in _DOM_ACTIONS:
+            return ToolResult(output="", error=f"Unknown DOM action: {action!r}")
+
+        bridge = _shared_bridge
+        if bridge is None or not bridge.is_connected:
+            return ToolResult(
+                output="",
+                error=(
+                    "Browser bridge not connected. "
+                    "Run `gdm browser start` then load the gdm-chrome extension."
+                ),
+            )
+
+        timeout = float(params.get("timeout", _DEFAULT_ACTION_TIMEOUT_SECS))
+        action_params = {k: v for k, v in params.items() if k not in ("action", "timeout")}
+
+        try:
+            response = bridge.send_command(action=action, params=action_params, timeout=timeout)
+        except TimeoutError:
+            return ToolResult(output="", error=f"Browser action '{action}' timed out after {timeout}s")
+        except ConnectionError as exc:
+            return ToolResult(output="", error=f"Bridge connection lost: {exc}")
+
+        if response.get("ok"):
+            return ToolResult(
+                output=json.dumps(response, indent=2),
+                metadata={"action": action, "selector": params.get("selector", "")},
+            )
+        return ToolResult(
+            output="",
+            error=response.get("error", "Unknown browser error"),
+            metadata={"action": action, "response": response},
+        )
+
+
+# ---------------------------------------------------------------------------
+# Browser capture tool
+# ---------------------------------------------------------------------------
+
+class BrowserCaptureTool(ToolBase):
+    """Take a screenshot of the current browser tab.
+
+    Screenshots are saved to .context-memory/screenshots/ and the file path
+    is returned. The agent can reference this path for vision analysis.
+    """
+
+    name: ClassVar[str] = "browser_capture"
+    description: ClassVar[str] = (
+        "Take a screenshot of the current browser tab. "
+        "Returns the file path to the saved PNG. "
+        "Use this to visually inspect UI state, verify layouts, or detect issues."
+    )
+    input_schema: ClassVar[dict[str, Any]] = {
+        "type": "object",
+        "properties": {
+            "tab_id": {
+                "type": "integer",
+                "description": "Target tab ID (optional — uses bound tab if omitted).",
+            },
+            "full_page": {
+                "type": "boolean",
+                "description": "Capture full scrollable page (default: visible viewport only).",
+            },
+        },
+        "additionalProperties": False,
+    }
+
+    def __init__(self, screenshot_dir: Path | None = None) -> None:
+        self._screenshot_dir = screenshot_dir or (
+            Path(".context-memory") / _SCREENSHOT_DIR_NAME
+        )
+
+    def execute(self, params: dict[str, Any]) -> ToolResult:
+        bridge = _shared_bridge
+        if bridge is None or not bridge.is_connected:
+            return ToolResult(
+                output="",
+                error="Browser bridge not connected. Run `gdm browser start`.",
+            )
+
+        try:
+            response = bridge.send_command(
+                action="screenshot",
+                params={
+                    "tab_id": params.get("tab_id"),
+                    "full_page": params.get("full_page", False),
+                },
+                timeout=15.0,
+            )
+        except TimeoutError:
+            return ToolResult(output="", error="Screenshot timed out")
+        except ConnectionError as exc:
+            return ToolResult(output="", error=f"Bridge connection lost: {exc}")
+
+        if not response.get("ok"):
+            return ToolResult(output="", error=response.get("error", "Screenshot failed"))
+
+        screenshot_b64: str = response.get("screenshot", "")
+        if not screenshot_b64:
+            return ToolResult(output="", error="No screenshot data received")
+
+        # Strip data URL prefix if present
+        if screenshot_b64.startswith("data:"):
+            screenshot_b64 = screenshot_b64.split(",", 1)[1]
+
+        self._screenshot_dir.mkdir(parents=True, exist_ok=True)
+        filename = f"screenshot_{uuid.uuid4().hex[:8]}.png"
+        filepath = self._screenshot_dir / filename
+        filepath.write_bytes(base64.b64decode(screenshot_b64))
+
+        return ToolResult(
+            output=f"Screenshot saved: {filepath}\nSize: {filepath.stat().st_size:,} bytes",
+            metadata={"path": str(filepath), "tab_id": params.get("tab_id")},
+        )
+
+
+# ---------------------------------------------------------------------------
+# Browser nav tool
+# ---------------------------------------------------------------------------
+
+class BrowserNavTool(ToolBase):
+    """Navigate the browser to a URL, wait for conditions, or reload."""
+
+    name: ClassVar[str] = "browser_nav"
+    description: ClassVar[str] = (
+        "Navigate the browser to a URL, wait for an element to appear/disappear, "
+        "or reload the page. Use before DOM actions to ensure the right page is loaded."
+    )
+    input_schema: ClassVar[dict[str, Any]] = {
+        "type": "object",
+        "required": ["action"],
+        "properties": {
+            "action": {
+                "type": "string",
+                "enum": sorted(_NAV_ACTIONS),
+                "description": "Navigation action: navigate, wait, reload, back, forward.",
+            },
+            "url": {
+                "type": "string",
+                "description": "URL to navigate to (required for 'navigate').",
+            },
+            "selector": {
+                "type": "string",
+                "description": "CSS selector to wait for (required for 'wait').",
+            },
+            "gone": {
+                "type": "boolean",
+                "description": "If true, wait for element to disappear (default: false).",
+            },
+            "timeout": {
+                "type": "number",
+                "description": "Wait timeout in seconds (default: 10).",
+            },
+            "tab_id": {
+                "type": "integer",
+                "description": "Target tab ID (optional).",
+            },
+        },
+        "additionalProperties": False,
+    }
+
+    def execute(self, params: dict[str, Any]) -> ToolResult:
+        action = params.get("action", "")
+        if action not in _NAV_ACTIONS:
+            return ToolResult(output="", error=f"Unknown nav action: {action!r}")
+
+        if action == "navigate" and not params.get("url"):
+            return ToolResult(output="", error="'url' is required for navigate action")
+
+        if action == "wait" and not params.get("selector"):
+            return ToolResult(output="", error="'selector' is required for wait action")
+
+        bridge = _shared_bridge
+        if bridge is None or not bridge.is_connected:
+            return ToolResult(output="", error="Browser bridge not connected.")
+
+        timeout = float(params.get("timeout", _DEFAULT_ACTION_TIMEOUT_SECS))
+        if action == "navigate":
+            timeout = max(timeout, 30.0)
+
+        action_params = {k: v for k, v in params.items() if k not in ("action", "timeout")}
+
+        try:
+            response = bridge.send_command(action=action, params=action_params, timeout=timeout)
+        except TimeoutError:
+            return ToolResult(output="", error=f"'{action}' timed out after {timeout}s")
+        except ConnectionError as exc:
+            return ToolResult(output="", error=f"Bridge connection lost: {exc}")
+
+        if response.get("ok"):
+            return ToolResult(
+                output=json.dumps(response, indent=2),
+                metadata={"action": action},
+            )
+        return ToolResult(output="", error=response.get("error", "Navigation failed"))
+
+
+# ---------------------------------------------------------------------------
+# Register tools
+# ---------------------------------------------------------------------------
+
+REGISTRY.register(BrowserDomTool())
+REGISTRY.register(BrowserCaptureTool())
+REGISTRY.register(BrowserNavTool())

src/tools/browser_tools.py

@@ -0,0 +1,179 @@
+"""Async browser tools client for gdm — communicates with Chrome extension via bridge server.
+
+BrowserTools (async) and BrowserToolsSync (sync wrapper) control a Chrome browser via
+the gdm Chrome extension bridge at ws://localhost:7681.
+
+websockets is an optional dependency; a clear ImportError is raised if missing at call time.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import dataclass
+from typing import Any
+
+try:
+    import websockets as websockets  # noqa: PLC0414  (re-export for monkeypatching in tests)
+    _WEBSOCKETS_AVAILABLE = True
+except ImportError:
+    websockets = None  # type: ignore[assignment]
+    _WEBSOCKETS_AVAILABLE = False
+
+__all__ = ["BrowserTools", "BrowserToolResult", "BrowserToolsSync"]
+
+# ---------------------------------------------------------------------------
+# JS safety: patterns blocked in execute_js
+# ---------------------------------------------------------------------------
+
+_DANGEROUS_JS_PATTERNS: tuple[str, ...] = (
+    "fetch(",
+    "XMLHttpRequest",
+    "import(",
+    "eval(",
+    "Function(",
+    "indexedDB",
+    "localStorage",
+    "sessionStorage",
+    "document.cookie",
+    "chrome.runtime",
+)
+
+
+# ---------------------------------------------------------------------------
+# BrowserToolResult
+# ---------------------------------------------------------------------------
+
+@dataclass
+class BrowserToolResult:
+    """Result of a browser tool operation."""
+
+    success: bool
+    data: Any
+    error: str | None = None
+
+
+# ---------------------------------------------------------------------------
+# BrowserTools (async)
+# ---------------------------------------------------------------------------
+
+class BrowserTools:
+    """Tools for controlling a browser via the gdm Chrome extension.
+
+    Each method opens a short-lived WebSocket connection to the bridge, sends
+    one JSON command, and awaits the response.  Requires ``gdm browser start``
+    and the gdm-chrome extension loaded in Chrome.
+    """
+
+    def __init__(self, bridge_url: str = "ws://localhost:7681") -> None:
+        self._bridge_url = bridge_url
+
+    # ── Public async API ──────────────────────────────────────────────────
+
+    async def get_page_content(self) -> BrowserToolResult:
+        """Get the visible text content of the active tab."""
+        return await self._async_send({"type": "get_page_content"})
+
+    async def click_element(self, selector: str) -> BrowserToolResult:
+        """Click a DOM element by CSS selector."""
+        return await self._async_send({"type": "click_element", "selector": selector})
+
+    async def fill_input(self, selector: str, value: str) -> BrowserToolResult:
+        """Fill an input field by CSS selector."""
+        return await self._async_send(
+            {"type": "fill_input", "selector": selector, "value": value}
+        )
+
+    async def get_selected_text(self) -> BrowserToolResult:
+        """Get currently selected text in the browser."""
+        return await self._async_send({"type": "get_selected_text"})
+
+    async def execute_js(self, code: str) -> BrowserToolResult:
+        """Execute JavaScript in the active tab (sandboxed).
+
+        Rejects code that contains dangerous patterns such as ``fetch``,
+        ``XMLHttpRequest``, ``eval``, dynamic ``import``, and Web Storage /
+        cookie / chrome-API access.
+        """
+        for pattern in _DANGEROUS_JS_PATTERNS:
+            if pattern in code:
+                return BrowserToolResult(
+                    success=False,
+                    data=None,
+                    error=f"Unsafe JS pattern rejected: '{pattern}'",
+                )
+        return await self._async_send({"type": "execute_js", "code": code})
+
+    async def screenshot(self) -> BrowserToolResult:
+        """Take a screenshot of the active tab. Returns base64 PNG in data field."""
+        return await self._async_send({"type": "screenshot"})
+
+    # ── Internal ──────────────────────────────────────────────────────────
+
+    def _send(self, msg: dict) -> BrowserToolResult:
+        """Sync wrapper around async send — useful for one-off calls outside an event loop."""
+        return asyncio.run(self._async_send(msg))
+
+    async def _async_send(self, msg: dict) -> BrowserToolResult:
+        """Open a WS connection, send *msg* as JSON, and return the parsed response."""
+        if not _WEBSOCKETS_AVAILABLE:
+            raise ImportError(
+                "The 'websockets' package is required for browser tools. "
+                "Install it with: pip install websockets"
+            )
+        try:
+            async with websockets.connect(self._bridge_url) as ws:
+                await ws.send(json.dumps(msg))
+                raw = await ws.recv()
+                data = json.loads(raw)
+                if data.get("ok"):
+                    return BrowserToolResult(success=True, data=data.get("data"))
+                return BrowserToolResult(
+                    success=False,
+                    data=None,
+                    error=data.get("error", "Unknown error from bridge"),
+                )
+        except (OSError, ConnectionRefusedError) as exc:
+            return BrowserToolResult(
+                success=False,
+                data=None,
+                error=f"Bridge connection failed: {exc}",
+            )
+        except Exception as exc:  # noqa: BLE001
+            return BrowserToolResult(
+                success=False,
+                data=None,
+                error=f"Browser tool error: {exc}",
+            )
+
+
+# ---------------------------------------------------------------------------
+# BrowserToolsSync (sync wrapper)
+# ---------------------------------------------------------------------------
+
+class BrowserToolsSync:
+    """Synchronous wrapper around :class:`BrowserTools` for non-async callers.
+
+    Each method calls ``asyncio.run()`` so it must be used outside any running
+    event loop (e.g. in scripts, CLI commands, or synchronous test helpers).
+    """
+
+    def __init__(self, bridge_url: str = "ws://localhost:7681") -> None:
+        self._async = BrowserTools(bridge_url)
+
+    def get_page_content(self) -> BrowserToolResult:
+        return asyncio.run(self._async.get_page_content())
+
+    def click_element(self, selector: str) -> BrowserToolResult:
+        return asyncio.run(self._async.click_element(selector))
+
+    def fill_input(self, selector: str, value: str) -> BrowserToolResult:
+        return asyncio.run(self._async.fill_input(selector, value))
+
+    def get_selected_text(self) -> BrowserToolResult:
+        return asyncio.run(self._async.get_selected_text())
+
+    def execute_js(self, code: str) -> BrowserToolResult:
+        return asyncio.run(self._async.execute_js(code))
+
+    def screenshot(self) -> BrowserToolResult:
+        return asyncio.run(self._async.screenshot())