sentienceapi 0.92.2__py3-none-any.whl → 0.98.0__py3-none-any.whl

sentience/backends/actions.py

@@ -0,0 +1,372 @@
+"""
+Backend-agnostic actions for browser-use integration.
+
+These actions work with any BrowserBackend implementation,
+enabling Sentience grounding with browser-use or other frameworks.
+
+Usage with browser-use:
+    from sentience.backends import BrowserUseAdapter
+    from sentience.backends.actions import click, type_text, scroll
+
+    adapter = BrowserUseAdapter(session)
+    backend = await adapter.create_backend()
+
+    # Take snapshot and click element
+    snap = await snapshot_from_backend(backend)
+    element = find(snap, 'role=button[name="Submit"]')
+    await click(backend, element.bbox)
+"""
+
+import asyncio
+import time
+from typing import TYPE_CHECKING, Any, Literal
+
+from ..cursor_policy import CursorPolicy, build_human_cursor_path
+from ..models import ActionResult, BBox, Snapshot
+
+if TYPE_CHECKING:
+    from .protocol import BrowserBackend
+
+
+async def click(
+    backend: "BrowserBackend",
+    target: BBox | dict[str, float] | tuple[float, float],
+    button: Literal["left", "right", "middle"] = "left",
+    click_count: int = 1,
+    move_first: bool = True,
+    cursor_policy: CursorPolicy | None = None,
+) -> ActionResult:
+    """
+    Click at coordinates using the backend.
+
+    Args:
+        backend: BrowserBackend implementation
+        target: Click target - BBox (clicks center), dict with x/y, or (x, y) tuple
+        button: Mouse button to click
+        click_count: Number of clicks (1=single, 2=double)
+        move_first: Whether to move mouse to position before clicking
+
+    Returns:
+        ActionResult with success status
+
+    Example:
+        # Click at coordinates
+        await click(backend, (100, 200))
+
+        # Click element bbox center
+        await click(backend, element.bbox)
+
+        # Double-click
+        await click(backend, element.bbox, click_count=2)
+    """
+    start_time = time.time()
+
+    # Resolve coordinates
+    x, y = _resolve_coordinates(target)
+    cursor_meta: dict | None = None
+
+    try:
+        # Optional mouse move for hover effects
+        if move_first:
+            if cursor_policy is not None and cursor_policy.mode == "human":
+                pos = getattr(backend, "_sentience_cursor_pos", None)
+                if not isinstance(pos, tuple) or len(pos) != 2:
+                    pos = (float(x), float(y))
+
+                cursor_meta = build_human_cursor_path(
+                    start=(float(pos[0]), float(pos[1])),
+                    target=(float(x), float(y)),
+                    policy=cursor_policy,
+                )
+                pts = cursor_meta.get("path", [])
+                duration_ms_move = int(cursor_meta.get("duration_ms") or 0)
+                per_step_s = (
+                    (duration_ms_move / max(1, len(pts))) / 1000.0 if duration_ms_move > 0 else 0.0
+                )
+                for p in pts:
+                    await backend.mouse_move(float(p["x"]), float(p["y"]))
+                    if per_step_s > 0:
+                        await asyncio.sleep(per_step_s)
+                pause_ms = int(cursor_meta.get("pause_before_click_ms") or 0)
+                if pause_ms > 0:
+                    await asyncio.sleep(pause_ms / 1000.0)
+            else:
+                await backend.mouse_move(x, y)
+                await asyncio.sleep(0.02)  # Brief pause for hover
+
+        # Perform click
+        await backend.mouse_click(x, y, button=button, click_count=click_count)
+        setattr(backend, "_sentience_cursor_pos", (float(x), float(y)))
+
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=True,
+            duration_ms=duration_ms,
+            outcome="dom_updated",
+            cursor=cursor_meta,
+        )
+    except Exception as e:
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=False,
+            duration_ms=duration_ms,
+            outcome="error",
+            error={"code": "click_failed", "reason": str(e)},
+            cursor=cursor_meta,
+        )
+
+
+async def type_text(
+    backend: "BrowserBackend",
+    text: str,
+    target: BBox | dict[str, float] | tuple[float, float] | None = None,
+    clear_first: bool = False,
+) -> ActionResult:
+    """
+    Type text, optionally clicking a target first.
+
+    Args:
+        backend: BrowserBackend implementation
+        text: Text to type
+        target: Optional click target before typing (BBox, dict, or tuple)
+        clear_first: If True, select all and delete before typing
+
+    Returns:
+        ActionResult with success status
+
+    Example:
+        # Type into focused element
+        await type_text(backend, "Hello World")
+
+        # Click input then type
+        await type_text(backend, "search query", target=search_box.bbox)
+
+        # Clear and type
+        await type_text(backend, "new value", target=input.bbox, clear_first=True)
+    """
+    start_time = time.time()
+
+    try:
+        # Click target if provided
+        if target is not None:
+            x, y = _resolve_coordinates(target)
+            await backend.mouse_click(x, y)
+            await asyncio.sleep(0.05)  # Wait for focus
+
+        # Clear existing content if requested
+        if clear_first:
+            # Select all (Ctrl+A / Cmd+A) and delete
+            await backend.eval("document.execCommand('selectAll')")
+            await asyncio.sleep(0.02)
+
+        # Type the text
+        await backend.type_text(text)
+
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=True,
+            duration_ms=duration_ms,
+            outcome="dom_updated",
+        )
+    except Exception as e:
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=False,
+            duration_ms=duration_ms,
+            outcome="error",
+            error={"code": "type_failed", "reason": str(e)},
+        )
+
+
+async def scroll(
+    backend: "BrowserBackend",
+    delta_y: float = 300,
+    target: BBox | dict[str, float] | tuple[float, float] | None = None,
+) -> ActionResult:
+    """
+    Scroll the page or element.
+
+    Args:
+        backend: BrowserBackend implementation
+        delta_y: Scroll amount (positive=down, negative=up)
+        target: Optional position for scroll (defaults to viewport center)
+
+    Returns:
+        ActionResult with success status
+
+    Example:
+        # Scroll down 300px
+        await scroll(backend, 300)
+
+        # Scroll up 500px
+        await scroll(backend, -500)
+
+        # Scroll at specific position
+        await scroll(backend, 200, target=(500, 300))
+    """
+    start_time = time.time()
+
+    try:
+        x: float | None = None
+        y: float | None = None
+
+        if target is not None:
+            x, y = _resolve_coordinates(target)
+
+        await backend.wheel(delta_y=delta_y, x=x, y=y)
+
+        # Wait for scroll to settle
+        await asyncio.sleep(0.1)
+
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=True,
+            duration_ms=duration_ms,
+            outcome="dom_updated",
+        )
+    except Exception as e:
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=False,
+            duration_ms=duration_ms,
+            outcome="error",
+            error={"code": "scroll_failed", "reason": str(e)},
+        )
+
+
+async def scroll_to_element(
+    backend: "BrowserBackend",
+    element_id: int,
+    behavior: Literal["smooth", "instant", "auto"] = "instant",
+    block: Literal["start", "center", "end", "nearest"] = "center",
+) -> ActionResult:
+    """
+    Scroll element into view using JavaScript scrollIntoView.
+
+    Args:
+        backend: BrowserBackend implementation
+        element_id: Element ID from snapshot (requires sentience_registry)
+        behavior: Scroll behavior
+        block: Vertical alignment
+
+    Returns:
+        ActionResult with success status
+    """
+    start_time = time.time()
+
+    try:
+        scrolled = await backend.eval(
+            f"""
+            (() => {{
+                const el = window.sentience_registry && window.sentience_registry[{element_id}];
+                if (el && el.scrollIntoView) {{
+                    el.scrollIntoView({{
+                        behavior: '{behavior}',
+                        block: '{block}',
+                        inline: 'nearest'
+                    }});
+                    return true;
+                }}
+                return false;
+            }})()
+        """
+        )
+
+        # Wait for scroll animation
+        wait_time = 0.3 if behavior == "smooth" else 0.05
+        await asyncio.sleep(wait_time)
+
+        duration_ms = int((time.time() - start_time) * 1000)
+
+        if scrolled:
+            return ActionResult(
+                success=True,
+                duration_ms=duration_ms,
+                outcome="dom_updated",
+            )
+        else:
+            return ActionResult(
+                success=False,
+                duration_ms=duration_ms,
+                outcome="error",
+                error={"code": "scroll_failed", "reason": "Element not found in registry"},
+            )
+    except Exception as e:
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=False,
+            duration_ms=duration_ms,
+            outcome="error",
+            error={"code": "scroll_failed", "reason": str(e)},
+        )
+
+
+async def wait_for_stable(
+    backend: "BrowserBackend",
+    state: Literal["interactive", "complete"] = "complete",
+    timeout_ms: int = 10000,
+) -> ActionResult:
+    """
+    Wait for page to reach stable state.
+
+    Args:
+        backend: BrowserBackend implementation
+        state: Target document.readyState
+        timeout_ms: Maximum wait time
+
+    Returns:
+        ActionResult with success status
+    """
+    start_time = time.time()
+
+    try:
+        await backend.wait_ready_state(state=state, timeout_ms=timeout_ms)
+
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=True,
+            duration_ms=duration_ms,
+            outcome="dom_updated",
+        )
+    except TimeoutError as e:
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=False,
+            duration_ms=duration_ms,
+            outcome="error",
+            error={"code": "timeout", "reason": str(e)},
+        )
+    except Exception as e:
+        duration_ms = int((time.time() - start_time) * 1000)
+        return ActionResult(
+            success=False,
+            duration_ms=duration_ms,
+            outcome="error",
+            error={"code": "wait_failed", "reason": str(e)},
+        )
+
+
+def _resolve_coordinates(
+    target: BBox | dict[str, float] | tuple[float, float],
+) -> tuple[float, float]:
+    """
+    Resolve target to (x, y) coordinates.
+
+    - BBox: Returns center point
+    - dict: Returns x, y keys (or center if width/height present)
+    - tuple: Returns as-is
+    """
+    if isinstance(target, BBox):
+        return (target.x + target.width / 2, target.y + target.height / 2)
+    elif isinstance(target, tuple):
+        return target
+    elif isinstance(target, dict):
+        # If has width/height, compute center
+        if "width" in target and "height" in target:
+            x = target.get("x", 0) + target["width"] / 2
+            y = target.get("y", 0) + target["height"] / 2
+            return (x, y)
+        # Otherwise use x/y directly
+        return (target.get("x", 0), target.get("y", 0))
+    else:
+        raise ValueError(f"Invalid target type: {type(target)}")

sentience/backends/browser_use_adapter.py

@@ -0,0 +1,241 @@
+"""
+Browser-use adapter for Sentience SDK.
+
+This module provides BrowserUseAdapter which wraps browser-use's BrowserSession
+and provides a CDPBackendV0 for Sentience operations.
+
+Usage:
+    from browser_use import BrowserSession, BrowserProfile
+    from sentience import get_extension_dir
+    from sentience.backends import BrowserUseAdapter
+
+    # Create browser-use session with Sentience extension
+    profile = BrowserProfile(args=[f"--load-extension={get_extension_dir()}"])
+    session = BrowserSession(browser_profile=profile)
+    await session.start()
+
+    # Create Sentience adapter
+    adapter = BrowserUseAdapter(session)
+    backend = await adapter.create_backend()
+
+    # Use backend for Sentience operations
+    viewport = await backend.refresh_page_info()
+    await backend.mouse_click(100, 200)
+"""
+
+from typing import TYPE_CHECKING, Any
+
+from .cdp_backend import CDPBackendV0, CDPTransport
+
+if TYPE_CHECKING:
+    # Import browser-use types only for type checking
+    # This avoids requiring browser-use as a hard dependency
+    pass
+
+
+class BrowserUseCDPTransport(CDPTransport):
+    """
+    CDP transport implementation for browser-use.
+
+    Wraps browser-use's CDP client to provide the CDPTransport interface.
+    Uses cdp-use library pattern: cdp_client.send.Domain.method(params={}, session_id=)
+    """
+
+    def __init__(self, cdp_client: Any, session_id: str) -> None:
+        """
+        Initialize transport with browser-use CDP client.
+
+        Args:
+            cdp_client: browser-use's CDP client (from cdp_session.cdp_client)
+            session_id: CDP session ID (from cdp_session.session_id)
+        """
+        self._client = cdp_client
+        self._session_id = session_id
+
+    async def send(self, method: str, params: dict | None = None) -> dict:
+        """
+        Send CDP command using browser-use's cdp-use client.
+
+        Translates method name like "Runtime.evaluate" to
+        cdp_client.send.Runtime.evaluate(params={...}, session_id=...).
+
+        Args:
+            method: CDP method name, e.g., "Runtime.evaluate"
+            params: Method parameters
+
+        Returns:
+            CDP response dict
+        """
+        # Split method into domain and method name
+        # e.g., "Runtime.evaluate" -> ("Runtime", "evaluate")
+        parts = method.split(".", 1)
+        if len(parts) != 2:
+            raise ValueError(f"Invalid CDP method format: {method}")
+
+        domain_name, method_name = parts
+
+        # Get the domain object from cdp_client.send
+        domain = getattr(self._client.send, domain_name, None)
+        if domain is None:
+            raise ValueError(f"Unknown CDP domain: {domain_name}")
+
+        # Get the method from the domain
+        method_func = getattr(domain, method_name, None)
+        if method_func is None:
+            raise ValueError(f"Unknown CDP method: {method}")
+
+        # Call the method with params and session_id
+        result = await method_func(
+            params=params or {},
+            session_id=self._session_id,
+        )
+
+        # cdp-use returns the result directly or None
+        return result if result is not None else {}
+
+
+class BrowserUseAdapter:
+    """
+    Adapter to use Sentience with browser-use's BrowserSession.
+
+    This adapter:
+    1. Wraps browser-use's CDP client with BrowserUseCDPTransport
+    2. Creates CDPBackendV0 for Sentience operations
+    3. Provides access to the underlying page for extension calls
+
+    Example:
+        from browser_use import BrowserSession, BrowserProfile
+        from sentience import get_extension_dir, snapshot_async, SnapshotOptions
+        from sentience.backends import BrowserUseAdapter
+
+        # Setup browser-use with Sentience extension
+        profile = BrowserProfile(args=[f"--load-extension={get_extension_dir()}"])
+        session = BrowserSession(browser_profile=profile)
+        await session.start()
+
+        # Create adapter and backend
+        adapter = BrowserUseAdapter(session)
+        backend = await adapter.create_backend()
+
+        # Navigate (using browser-use)
+        page = await session.get_current_page()
+        await page.goto("https://example.com")
+
+        # Take Sentience snapshot (uses extension)
+        snap = await snapshot_async(adapter, SnapshotOptions())
+
+        # Use backend for precise clicking
+        await backend.mouse_click(snap.elements[0].bbox.x, snap.elements[0].bbox.y)
+    """
+
+    def __init__(self, session: Any) -> None:
+        """
+        Initialize adapter with browser-use BrowserSession.
+
+        Args:
+            session: browser-use BrowserSession instance
+        """
+        self._session = session
+        self._backend: CDPBackendV0 | None = None
+        self._transport: BrowserUseCDPTransport | None = None
+
+    @property
+    def page(self) -> Any:
+        """
+        Get the current Playwright page from browser-use.
+
+        This is needed for Sentience snapshot() which calls window.sentience.snapshot().
+
+        Returns:
+            Playwright Page object
+        """
+        # browser-use stores page in session
+        # Access pattern may vary by browser-use version
+        if hasattr(self._session, "page"):
+            return self._session.page
+        if hasattr(self._session, "_page"):
+            return self._session._page
+        if hasattr(self._session, "get_current_page"):
+            # This is async, but we need sync access for property
+            # Caller should use get_page_async() instead
+            raise RuntimeError("Use await adapter.get_page_async() to get the page")
+        raise RuntimeError("Could not find page in browser-use session")
+
+    async def get_page_async(self) -> Any:
+        """
+        Get the current Playwright page (async).
+
+        Returns:
+            Playwright Page object
+        """
+        if hasattr(self._session, "get_current_page"):
+            return await self._session.get_current_page()
+        return self.page
+
+    @property
+    def api_key(self) -> str | None:
+        """
+        API key for Sentience API (for snapshot compatibility).
+
+        Returns None since browser-use users pass api_key via SnapshotOptions.
+        """
+        return None
+
+    @property
+    def api_url(self) -> str | None:
+        """
+        API URL for Sentience API (for snapshot compatibility).
+
+        Returns None to use default.
+        """
+        return None
+
+    async def create_backend(self) -> CDPBackendV0:
+        """
+        Create CDP backend for Sentience operations.
+
+        This method:
+        1. Gets or creates a CDP session from browser-use
+        2. Creates BrowserUseCDPTransport to wrap the CDP client
+        3. Creates CDPBackendV0 with the transport
+
+        Returns:
+            CDPBackendV0 instance ready for use
+
+        Raises:
+            RuntimeError: If CDP session cannot be created
+        """
+        if self._backend is not None:
+            return self._backend
+
+        # Get CDP session from browser-use
+        # browser-use uses: cdp_session = await session.get_or_create_cdp_session()
+        if not hasattr(self._session, "get_or_create_cdp_session"):
+            raise RuntimeError(
+                "browser-use session does not have get_or_create_cdp_session method. "
+                "Make sure you're using a compatible version of browser-use."
+            )
+
+        cdp_session = await self._session.get_or_create_cdp_session()
+
+        # Extract CDP client and session ID
+        cdp_client = cdp_session.cdp_client
+        session_id = cdp_session.session_id
+
+        # Create transport and backend
+        self._transport = BrowserUseCDPTransport(cdp_client, session_id)
+        self._backend = CDPBackendV0(self._transport)
+
+        return self._backend
+
+    async def get_transport(self) -> BrowserUseCDPTransport:
+        """
+        Get the CDP transport (creates backend if needed).
+
+        Returns:
+            BrowserUseCDPTransport instance
+        """
+        if self._transport is None:
+            await self.create_backend()
+        assert self._transport is not None
+        return self._transport