sentienceapi 0.90.16__py3-none-any.whl → 0.98.0__py3-none-any.whl

sentience/backends/browser_use_adapter.py

@@ -0,0 +1,241 @@
+"""
+Browser-use adapter for Sentience SDK.
+
+This module provides BrowserUseAdapter which wraps browser-use's BrowserSession
+and provides a CDPBackendV0 for Sentience operations.
+
+Usage:
+    from browser_use import BrowserSession, BrowserProfile
+    from sentience import get_extension_dir
+    from sentience.backends import BrowserUseAdapter
+
+    # Create browser-use session with Sentience extension
+    profile = BrowserProfile(args=[f"--load-extension={get_extension_dir()}"])
+    session = BrowserSession(browser_profile=profile)
+    await session.start()
+
+    # Create Sentience adapter
+    adapter = BrowserUseAdapter(session)
+    backend = await adapter.create_backend()
+
+    # Use backend for Sentience operations
+    viewport = await backend.refresh_page_info()
+    await backend.mouse_click(100, 200)
+"""
+
+from typing import TYPE_CHECKING, Any
+
+from .cdp_backend import CDPBackendV0, CDPTransport
+
+if TYPE_CHECKING:
+    # Import browser-use types only for type checking
+    # This avoids requiring browser-use as a hard dependency
+    pass
+
+
+class BrowserUseCDPTransport(CDPTransport):
+    """
+    CDP transport implementation for browser-use.
+
+    Wraps browser-use's CDP client to provide the CDPTransport interface.
+    Uses cdp-use library pattern: cdp_client.send.Domain.method(params={}, session_id=)
+    """
+
+    def __init__(self, cdp_client: Any, session_id: str) -> None:
+        """
+        Initialize transport with browser-use CDP client.
+
+        Args:
+            cdp_client: browser-use's CDP client (from cdp_session.cdp_client)
+            session_id: CDP session ID (from cdp_session.session_id)
+        """
+        self._client = cdp_client
+        self._session_id = session_id
+
+    async def send(self, method: str, params: dict | None = None) -> dict:
+        """
+        Send CDP command using browser-use's cdp-use client.
+
+        Translates method name like "Runtime.evaluate" to
+        cdp_client.send.Runtime.evaluate(params={...}, session_id=...).
+
+        Args:
+            method: CDP method name, e.g., "Runtime.evaluate"
+            params: Method parameters
+
+        Returns:
+            CDP response dict
+        """
+        # Split method into domain and method name
+        # e.g., "Runtime.evaluate" -> ("Runtime", "evaluate")
+        parts = method.split(".", 1)
+        if len(parts) != 2:
+            raise ValueError(f"Invalid CDP method format: {method}")
+
+        domain_name, method_name = parts
+
+        # Get the domain object from cdp_client.send
+        domain = getattr(self._client.send, domain_name, None)
+        if domain is None:
+            raise ValueError(f"Unknown CDP domain: {domain_name}")
+
+        # Get the method from the domain
+        method_func = getattr(domain, method_name, None)
+        if method_func is None:
+            raise ValueError(f"Unknown CDP method: {method}")
+
+        # Call the method with params and session_id
+        result = await method_func(
+            params=params or {},
+            session_id=self._session_id,
+        )
+
+        # cdp-use returns the result directly or None
+        return result if result is not None else {}
+
+
+class BrowserUseAdapter:
+    """
+    Adapter to use Sentience with browser-use's BrowserSession.
+
+    This adapter:
+    1. Wraps browser-use's CDP client with BrowserUseCDPTransport
+    2. Creates CDPBackendV0 for Sentience operations
+    3. Provides access to the underlying page for extension calls
+
+    Example:
+        from browser_use import BrowserSession, BrowserProfile
+        from sentience import get_extension_dir, snapshot_async, SnapshotOptions
+        from sentience.backends import BrowserUseAdapter
+
+        # Setup browser-use with Sentience extension
+        profile = BrowserProfile(args=[f"--load-extension={get_extension_dir()}"])
+        session = BrowserSession(browser_profile=profile)
+        await session.start()
+
+        # Create adapter and backend
+        adapter = BrowserUseAdapter(session)
+        backend = await adapter.create_backend()
+
+        # Navigate (using browser-use)
+        page = await session.get_current_page()
+        await page.goto("https://example.com")
+
+        # Take Sentience snapshot (uses extension)
+        snap = await snapshot_async(adapter, SnapshotOptions())
+
+        # Use backend for precise clicking
+        await backend.mouse_click(snap.elements[0].bbox.x, snap.elements[0].bbox.y)
+    """
+
+    def __init__(self, session: Any) -> None:
+        """
+        Initialize adapter with browser-use BrowserSession.
+
+        Args:
+            session: browser-use BrowserSession instance
+        """
+        self._session = session
+        self._backend: CDPBackendV0 | None = None
+        self._transport: BrowserUseCDPTransport | None = None
+
+    @property
+    def page(self) -> Any:
+        """
+        Get the current Playwright page from browser-use.
+
+        This is needed for Sentience snapshot() which calls window.sentience.snapshot().
+
+        Returns:
+            Playwright Page object
+        """
+        # browser-use stores page in session
+        # Access pattern may vary by browser-use version
+        if hasattr(self._session, "page"):
+            return self._session.page
+        if hasattr(self._session, "_page"):
+            return self._session._page
+        if hasattr(self._session, "get_current_page"):
+            # This is async, but we need sync access for property
+            # Caller should use get_page_async() instead
+            raise RuntimeError("Use await adapter.get_page_async() to get the page")
+        raise RuntimeError("Could not find page in browser-use session")
+
+    async def get_page_async(self) -> Any:
+        """
+        Get the current Playwright page (async).
+
+        Returns:
+            Playwright Page object
+        """
+        if hasattr(self._session, "get_current_page"):
+            return await self._session.get_current_page()
+        return self.page
+
+    @property
+    def api_key(self) -> str | None:
+        """
+        API key for Sentience API (for snapshot compatibility).
+
+        Returns None since browser-use users pass api_key via SnapshotOptions.
+        """
+        return None
+
+    @property
+    def api_url(self) -> str | None:
+        """
+        API URL for Sentience API (for snapshot compatibility).
+
+        Returns None to use default.
+        """
+        return None
+
+    async def create_backend(self) -> CDPBackendV0:
+        """
+        Create CDP backend for Sentience operations.
+
+        This method:
+        1. Gets or creates a CDP session from browser-use
+        2. Creates BrowserUseCDPTransport to wrap the CDP client
+        3. Creates CDPBackendV0 with the transport
+
+        Returns:
+            CDPBackendV0 instance ready for use
+
+        Raises:
+            RuntimeError: If CDP session cannot be created
+        """
+        if self._backend is not None:
+            return self._backend
+
+        # Get CDP session from browser-use
+        # browser-use uses: cdp_session = await session.get_or_create_cdp_session()
+        if not hasattr(self._session, "get_or_create_cdp_session"):
+            raise RuntimeError(
+                "browser-use session does not have get_or_create_cdp_session method. "
+                "Make sure you're using a compatible version of browser-use."
+            )
+
+        cdp_session = await self._session.get_or_create_cdp_session()
+
+        # Extract CDP client and session ID
+        cdp_client = cdp_session.cdp_client
+        session_id = cdp_session.session_id
+
+        # Create transport and backend
+        self._transport = BrowserUseCDPTransport(cdp_client, session_id)
+        self._backend = CDPBackendV0(self._transport)
+
+        return self._backend
+
+    async def get_transport(self) -> BrowserUseCDPTransport:
+        """
+        Get the CDP transport (creates backend if needed).
+
+        Returns:
+            BrowserUseCDPTransport instance
+        """
+        if self._transport is None:
+            await self.create_backend()
+        assert self._transport is not None
+        return self._transport

sentience/backends/cdp_backend.py

@@ -0,0 +1,393 @@
+"""
+CDP Backend implementation for browser-use integration.
+
+This module provides CDPBackendV0, which implements BrowserBackend protocol
+using Chrome DevTools Protocol (CDP) commands.
+
+Usage with browser-use:
+    from browser_use import BrowserSession
+    from sentience.backends import CDPBackendV0
+    from sentience.backends.browser_use_adapter import BrowserUseAdapter
+
+    session = BrowserSession(...)
+    await session.start()
+
+    adapter = BrowserUseAdapter(session)
+    backend = await adapter.create_backend()
+
+    # Now use backend for Sentience operations
+    viewport = await backend.refresh_page_info()
+    await backend.mouse_click(100, 200)
+"""
+
+import asyncio
+import base64
+import time
+from typing import Any, Literal, Protocol, runtime_checkable
+
+from .protocol import BrowserBackend, LayoutMetrics, ViewportInfo
+
+
+@runtime_checkable
+class CDPTransport(Protocol):
+    """
+    Protocol for CDP transport layer.
+
+    This abstracts the actual CDP communication, allowing different
+    implementations (browser-use, Playwright CDP, raw WebSocket).
+    """
+
+    async def send(self, method: str, params: dict | None = None) -> dict:
+        """
+        Send a CDP command and return the result.
+
+        Args:
+            method: CDP method name, e.g., "Runtime.evaluate"
+            params: Method parameters
+
+        Returns:
+            CDP response dict
+        """
+        ...
+
+
+class CDPBackendV0:
+    """
+    CDP-based implementation of BrowserBackend.
+
+    This backend uses CDP commands to interact with the browser,
+    making it compatible with browser-use's CDP client.
+    """
+
+    def __init__(self, transport: CDPTransport) -> None:
+        """
+        Initialize CDP backend.
+
+        Args:
+            transport: CDP transport for sending commands
+        """
+        self._transport = transport
+        self._cached_viewport: ViewportInfo | None = None
+        self._execution_context_id: int | None = None
+
+    async def _get_execution_context(self) -> int:
+        """Get or create execution context ID for Runtime.callFunctionOn."""
+        if self._execution_context_id is not None:
+            return self._execution_context_id
+
+        # Enable Runtime domain if not already enabled
+        try:
+            await self._transport.send("Runtime.enable")
+        except Exception:
+            pass  # May already be enabled
+
+        # Get the main frame's execution context
+        result = await self._transport.send(
+            "Runtime.evaluate",
+            {
+                "expression": "1",
+                "returnByValue": True,
+            },
+        )
+
+        # Extract context ID from the result
+        if "executionContextId" in result:
+            self._execution_context_id = result["executionContextId"]
+        else:
+            # Fallback: use context ID 1 (main frame)
+            self._execution_context_id = 1
+
+        return self._execution_context_id
+
+    async def refresh_page_info(self) -> ViewportInfo:
+        """Cache viewport + scroll offsets; cheap & safe to call often."""
+        result = await self.eval(
+            """(() => ({
+                width: window.innerWidth,
+                height: window.innerHeight,
+                scroll_x: window.scrollX,
+                scroll_y: window.scrollY,
+                content_width: document.documentElement.scrollWidth,
+                content_height: document.documentElement.scrollHeight
+            }))()"""
+        )
+
+        self._cached_viewport = ViewportInfo(
+            width=result.get("width", 0),
+            height=result.get("height", 0),
+            scroll_x=result.get("scroll_x", 0),
+            scroll_y=result.get("scroll_y", 0),
+            content_width=result.get("content_width"),
+            content_height=result.get("content_height"),
+        )
+        return self._cached_viewport
+
+    async def eval(self, expression: str) -> Any:
+        """Evaluate JavaScript expression using Runtime.evaluate."""
+        result = await self._transport.send(
+            "Runtime.evaluate",
+            {
+                "expression": expression,
+                "returnByValue": True,
+                "awaitPromise": True,
+            },
+        )
+
+        # Check for exceptions
+        if "exceptionDetails" in result:
+            exc = result["exceptionDetails"]
+            text = exc.get("text", "Unknown error")
+            raise RuntimeError(f"JavaScript evaluation failed: {text}")
+
+        # Extract value from result
+        if "result" in result:
+            res = result["result"]
+            if res.get("type") == "undefined":
+                return None
+            return res.get("value")
+
+        return None
+
+    async def call(
+        self,
+        function_declaration: str,
+        args: list[Any] | None = None,
+    ) -> Any:
+        """Call JavaScript function using Runtime.callFunctionOn."""
+        # Build call arguments
+        call_args = []
+        if args:
+            for arg in args:
+                if arg is None:
+                    call_args.append({"value": None})
+                elif isinstance(arg, bool):
+                    call_args.append({"value": arg})
+                elif isinstance(arg, (int, float)):
+                    call_args.append({"value": arg})
+                elif isinstance(arg, str):
+                    call_args.append({"value": arg})
+                elif isinstance(arg, dict):
+                    call_args.append({"value": arg})
+                elif isinstance(arg, list):
+                    call_args.append({"value": arg})
+                else:
+                    # Serialize complex objects to JSON
+                    call_args.append({"value": str(arg)})
+
+        # We need an object ID to call function on
+        # Use globalThis (window) as the target
+        global_result = await self._transport.send(
+            "Runtime.evaluate",
+            {
+                "expression": "globalThis",
+                "returnByValue": False,
+            },
+        )
+
+        object_id = global_result.get("result", {}).get("objectId")
+        if not object_id:
+            # Fallback: evaluate the function directly
+            if args:
+                args_json = ", ".join(repr(a) if isinstance(a, str) else str(a) for a in args)
+                expression = f"({function_declaration})({args_json})"
+            else:
+                expression = f"({function_declaration})()"
+            return await self.eval(expression)
+
+        result = await self._transport.send(
+            "Runtime.callFunctionOn",
+            {
+                "functionDeclaration": function_declaration,
+                "objectId": object_id,
+                "arguments": call_args,
+                "returnByValue": True,
+                "awaitPromise": True,
+            },
+        )
+
+        # Check for exceptions
+        if "exceptionDetails" in result:
+            exc = result["exceptionDetails"]
+            text = exc.get("text", "Unknown error")
+            raise RuntimeError(f"JavaScript call failed: {text}")
+
+        # Extract value from result
+        if "result" in result:
+            res = result["result"]
+            if res.get("type") == "undefined":
+                return None
+            return res.get("value")
+
+        return None
+
+    async def get_layout_metrics(self) -> LayoutMetrics:
+        """Get page layout metrics using Page.getLayoutMetrics."""
+        result = await self._transport.send("Page.getLayoutMetrics")
+
+        # Extract metrics from result
+        layout_viewport = result.get("layoutViewport", {})
+        content_size = result.get("contentSize", {})
+        visual_viewport = result.get("visualViewport", {})
+
+        return LayoutMetrics(
+            viewport_x=visual_viewport.get("pageX", 0),
+            viewport_y=visual_viewport.get("pageY", 0),
+            viewport_width=visual_viewport.get(
+                "clientWidth", layout_viewport.get("clientWidth", 0)
+            ),
+            viewport_height=visual_viewport.get(
+                "clientHeight", layout_viewport.get("clientHeight", 0)
+            ),
+            content_width=content_size.get("width", 0),
+            content_height=content_size.get("height", 0),
+            device_scale_factor=visual_viewport.get("scale", 1.0),
+        )
+
+    async def screenshot_png(self) -> bytes:
+        """Capture viewport screenshot as PNG bytes."""
+        result = await self._transport.send(
+            "Page.captureScreenshot",
+            {
+                "format": "png",
+                "captureBeyondViewport": False,
+            },
+        )
+
+        data = result.get("data", "")
+        return base64.b64decode(data)
+
+    async def mouse_move(self, x: float, y: float) -> None:
+        """Move mouse to viewport coordinates."""
+        await self._transport.send(
+            "Input.dispatchMouseEvent",
+            {
+                "type": "mouseMoved",
+                "x": x,
+                "y": y,
+            },
+        )
+
+    async def mouse_click(
+        self,
+        x: float,
+        y: float,
+        button: Literal["left", "right", "middle"] = "left",
+        click_count: int = 1,
+    ) -> None:
+        """Click at viewport coordinates."""
+        # Mouse down
+        await self._transport.send(
+            "Input.dispatchMouseEvent",
+            {
+                "type": "mousePressed",
+                "x": x,
+                "y": y,
+                "button": button,
+                "clickCount": click_count,
+            },
+        )
+
+        # Small delay between press and release
+        await asyncio.sleep(0.05)
+
+        # Mouse up
+        await self._transport.send(
+            "Input.dispatchMouseEvent",
+            {
+                "type": "mouseReleased",
+                "x": x,
+                "y": y,
+                "button": button,
+                "clickCount": click_count,
+            },
+        )
+
+    async def wheel(
+        self,
+        delta_y: float,
+        x: float | None = None,
+        y: float | None = None,
+    ) -> None:
+        """Scroll using mouse wheel."""
+        # Get viewport center if coordinates not provided
+        if x is None or y is None:
+            if self._cached_viewport is None:
+                await self.refresh_page_info()
+            assert self._cached_viewport is not None
+            x = x if x is not None else self._cached_viewport.width / 2
+            y = y if y is not None else self._cached_viewport.height / 2
+
+        await self._transport.send(
+            "Input.dispatchMouseEvent",
+            {
+                "type": "mouseWheel",
+                "x": x,
+                "y": y,
+                "deltaX": 0,
+                "deltaY": delta_y,
+            },
+        )
+
+    async def type_text(self, text: str) -> None:
+        """Type text using keyboard input."""
+        for char in text:
+            # Key down
+            await self._transport.send(
+                "Input.dispatchKeyEvent",
+                {
+                    "type": "keyDown",
+                    "text": char,
+                },
+            )
+
+            # Char event (for text input)
+            await self._transport.send(
+                "Input.dispatchKeyEvent",
+                {
+                    "type": "char",
+                    "text": char,
+                },
+            )
+
+            # Key up
+            await self._transport.send(
+                "Input.dispatchKeyEvent",
+                {
+                    "type": "keyUp",
+                    "text": char,
+                },
+            )
+
+            # Small delay between characters
+            await asyncio.sleep(0.01)
+
+    async def wait_ready_state(
+        self,
+        state: Literal["interactive", "complete"] = "interactive",
+        timeout_ms: int = 15000,
+    ) -> None:
+        """Wait for document.readyState using polling."""
+        start = time.monotonic()
+        timeout_sec = timeout_ms / 1000.0
+
+        # Map state to acceptable states
+        acceptable_states = {"complete"} if state == "complete" else {"interactive", "complete"}
+
+        while True:
+            elapsed = time.monotonic() - start
+            if elapsed >= timeout_sec:
+                raise TimeoutError(
+                    f"Timed out waiting for document.readyState='{state}' " f"after {timeout_ms}ms"
+                )
+
+            current_state = await self.eval("document.readyState")
+            if current_state in acceptable_states:
+                return
+
+            # Poll every 100ms
+            await asyncio.sleep(0.1)
+
+    async def get_url(self) -> str:
+        """Get current page URL."""
+        result = await self.eval("window.location.href")
+        return result if result else ""