sentienceapi 0.90.16__py3-none-any.whl → 0.98.0__py3-none-any.whl

sentience/__init__.py

@@ -2,9 +2,39 @@
 Sentience Python SDK - AI Agent Browser Automation
 """
 
-from .actions import click, click_rect, press, type_text
-from .agent import SentienceAgent
+# Extension helpers (for browser-use integration)
+from ._extension_loader import (
+    get_extension_dir,
+    get_extension_version,
+    verify_extension_injected,
+    verify_extension_injected_async,
+    verify_extension_version,
+    verify_extension_version_async,
+)
+from .actions import click, click_rect, press, scroll_to, type_text
+from .agent import SentienceAgent, SentienceAgentAsync
 from .agent_config import AgentConfig
+from .agent_runtime import AgentRuntime, AssertionHandle
+
+# Backend-agnostic actions (aliased to avoid conflict with existing actions)
+# Browser backends (for browser-use integration)
+from .backends import (
+    BrowserBackend,
+    BrowserUseAdapter,
+    BrowserUseCDPTransport,
+    CachedSnapshot,
+    CDPBackendV0,
+    CDPTransport,
+    LayoutMetrics,
+    PlaywrightBackend,
+    ViewportInfo,
+)
+from .backends import click as backend_click
+from .backends import scroll as backend_scroll
+from .backends import scroll_to_element as backend_scroll_to_element
+from .backends import snapshot as backend_snapshot
+from .backends import type_text as backend_type_text
+from .backends import wait_for_stable as backend_wait_for_stable
 
 # Agent Layer (Phase 1 & 2)
 from .base_agent import BaseAgent
@@ -13,10 +43,8 @@ from .browser import SentienceBrowser
 # Tracing (v0.12.0+)
 from .cloud_tracing import CloudTraceSink, SentienceLogger
 from .conversational_agent import ConversationalAgent
+from .cursor_policy import CursorPolicy
 from .expect import expect
-
-# Formatting (v0.12.0+)
-from .formatting import format_snapshot_for_llm
 from .generator import ScriptGenerator, generate
 from .inspector import Inspector, inspect
 from .llm_provider import (
@@ -24,6 +52,8 @@ from .llm_provider import (
     LLMProvider,
     LLMResponse,
     LocalLLMProvider,
+    LocalVisionLLMProvider,
+    MLXVLMProvider,
     OpenAIProvider,
 )
 from .models import (  # Agent Layer Models
@@ -50,17 +80,22 @@ from .models import (  # Agent Layer Models
     ViewportRect,
     WaitResult,
 )
+
+# Ordinal support (Phase 3)
+from .ordinal import OrdinalIntent, boost_ordinal_elements, detect_ordinal_intent, select_by_ordinal
 from .overlay import clear_overlay, show_overlay
 from .query import find, query
 from .read import read
 from .recorder import Recorder, Trace, TraceStep, record
 from .screenshot import screenshot
+from .sentience_methods import AgentAction, SentienceMethod
 from .snapshot import snapshot
 from .text_search import find_text_rect
 from .tracer_factory import SENTIENCE_API_URL, create_tracer
 from .tracing import JsonlTraceSink, TraceEvent, Tracer, TraceSink
 
 # Utilities (v0.12.0+)
+# Import from utils package (re-exports from submodules for backward compatibility)
 from .utils import (
     canonical_snapshot_loose,
     canonical_snapshot_strict,
@@ -68,11 +103,62 @@ from .utils import (
     save_storage_state,
     sha256_digest,
 )
+
+# Formatting (v0.12.0+)
+from .utils.formatting import format_snapshot_for_llm
+
+# Verification (agent assertion loop)
+from .verification import (
+    AssertContext,
+    AssertOutcome,
+    Predicate,
+    all_of,
+    any_of,
+    custom,
+    element_count,
+    exists,
+    is_checked,
+    is_collapsed,
+    is_disabled,
+    is_enabled,
+    is_expanded,
+    is_unchecked,
+    not_exists,
+    url_contains,
+    url_matches,
+    value_contains,
+    value_equals,
+)
+from .visual_agent import SentienceVisualAgent, SentienceVisualAgentAsync
 from .wait import wait_for
 
-__version__ = "0.90.16"
+__version__ = "0.98.0"
 
 __all__ = [
+    # Extension helpers (for browser-use integration)
+    "get_extension_dir",
+    "get_extension_version",
+    "verify_extension_injected",
+    "verify_extension_injected_async",
+    "verify_extension_version",
+    "verify_extension_version_async",
+    # Browser backends (for browser-use integration)
+    "BrowserBackend",
+    "CDPTransport",
+    "CDPBackendV0",
+    "PlaywrightBackend",
+    "BrowserUseAdapter",
+    "BrowserUseCDPTransport",
+    "ViewportInfo",
+    "LayoutMetrics",
+    "backend_snapshot",
+    "CachedSnapshot",
+    # Backend-agnostic actions (prefixed to avoid conflicts)
+    "backend_click",
+    "backend_type_text",
+    "backend_scroll",
+    "backend_scroll_to_element",
+    "backend_wait_for_stable",
     # Core SDK
     "SentienceBrowser",
     "Snapshot",
@@ -87,7 +173,9 @@ __all__ = [
     "click",
     "type_text",
     "press",
+    "scroll_to",
     "click_rect",
+    "CursorPolicy",
     "wait_for",
     "expect",
     "Inspector",
@@ -116,7 +204,12 @@ __all__ = [
     "OpenAIProvider",
     "AnthropicProvider",
     "LocalLLMProvider",
+    "LocalVisionLLMProvider",
+    "MLXVLMProvider",
     "SentienceAgent",
+    "SentienceAgentAsync",
+    "SentienceVisualAgent",
+    "SentienceVisualAgentAsync",
     "ConversationalAgent",
     # Agent Layer Models
     "AgentActionResult",
@@ -150,4 +243,25 @@ __all__ = [
     "format_snapshot_for_llm",
     # Agent Config (v0.12.0+)
     "AgentConfig",
+    # Enums
+    "SentienceMethod",
+    "AgentAction",
+    # Verification (agent assertion loop)
+    "AgentRuntime",
+    "AssertContext",
+    "AssertOutcome",
+    "Predicate",
+    "url_matches",
+    "url_contains",
+    "exists",
+    "not_exists",
+    "element_count",
+    "all_of",
+    "any_of",
+    "custom",
+    # Ordinal support (Phase 3)
+    "OrdinalIntent",
+    "detect_ordinal_intent",
+    "select_by_ordinal",
+    "boost_ordinal_elements",
 ]

sentience/_extension_loader.py

@@ -1,8 +1,19 @@
 """
-Shared extension loading logic for sync and async implementations
+Shared extension loading logic for sync and async implementations.
+
+Provides:
+- get_extension_dir(): Returns path to bundled extension (for browser-use integration)
+- verify_extension_injected(): Verifies window.sentience API is available
+- get_extension_version(): Gets extension version from manifest
+- verify_extension_version(): Checks SDK-extension version compatibility
 """
 
+import json
 from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .protocols import AsyncPageProtocol, PageProtocol
 
 
 def find_extension_path() -> Path:
@@ -38,3 +49,147 @@ def find_extension_path() -> Path:
             f"2. {dev_ext_path}\n"
             "Make sure the extension is built and 'sentience/extension' directory exists."
         )
+
+
+def get_extension_dir() -> str:
+    """
+    Get path to the bundled Sentience extension directory.
+
+    Use this to load the extension into browser-use or other Chromium-based browsers:
+
+        from sentience import get_extension_dir
+        from browser_use import BrowserSession, BrowserProfile
+
+        profile = BrowserProfile(
+            args=[f"--load-extension={get_extension_dir()}"],
+        )
+        session = BrowserSession(browser_profile=profile)
+
+    Returns:
+        Absolute path to extension directory as string
+
+    Raises:
+        FileNotFoundError: If extension not found in package
+    """
+    return str(find_extension_path())
+
+
+def get_extension_version() -> str:
+    """
+    Get the version of the bundled extension from manifest.json.
+
+    Returns:
+        Version string (e.g., "2.2.0")
+
+    Raises:
+        FileNotFoundError: If extension or manifest not found
+    """
+    ext_path = find_extension_path()
+    manifest_path = ext_path / "manifest.json"
+    with open(manifest_path) as f:
+        manifest = json.load(f)
+    return manifest.get("version", "unknown")
+
+
+def verify_extension_injected(page: "PageProtocol") -> bool:
+    """
+    Verify the Sentience extension injected window.sentience API (sync).
+
+    Call this after navigating to a page to confirm the extension is working:
+
+        browser.goto("https://example.com")
+        if not verify_extension_injected(browser.page):
+            raise RuntimeError("Extension not injected")
+
+    Args:
+        page: Playwright Page object (sync)
+
+    Returns:
+        True if window.sentience.snapshot is available, False otherwise
+    """
+    try:
+        result = page.evaluate(
+            "(() => !!(window.sentience && typeof window.sentience.snapshot === 'function'))()"
+        )
+        return bool(result)
+    except Exception:
+        return False
+
+
+async def verify_extension_injected_async(page: "AsyncPageProtocol") -> bool:
+    """
+    Verify the Sentience extension injected window.sentience API (async).
+
+    Call this after navigating to a page to confirm the extension is working:
+
+        await browser.goto("https://example.com")
+        if not await verify_extension_injected_async(browser.page):
+            raise RuntimeError("Extension not injected")
+
+    Args:
+        page: Playwright Page object (async)
+
+    Returns:
+        True if window.sentience.snapshot is available, False otherwise
+    """
+    try:
+        result = await page.evaluate(
+            "(() => !!(window.sentience && typeof window.sentience.snapshot === 'function'))()"
+        )
+        return bool(result)
+    except Exception:
+        return False
+
+
+def verify_extension_version(page: "PageProtocol", expected: str | None = None) -> str | None:
+    """
+    Check extension version exposed in page (sync).
+
+    The extension sets window.__SENTIENCE_EXTENSION_VERSION__ when injected.
+
+    Args:
+        page: Playwright Page object (sync)
+        expected: If provided, raises RuntimeError on mismatch
+
+    Returns:
+        Version string if found, None if not set (page may not have injected yet)
+
+    Raises:
+        RuntimeError: If expected version provided and doesn't match
+    """
+    try:
+        got = page.evaluate("window.__SENTIENCE_EXTENSION_VERSION__ || null")
+    except Exception:
+        got = None
+
+    if expected and got and got != expected:
+        raise RuntimeError(f"Sentience extension version mismatch: expected {expected}, got {got}")
+    return got
+
+
+async def verify_extension_version_async(
+    page: "AsyncPageProtocol", expected: str | None = None
+) -> str | None:
+    """
+    Check extension version exposed in page (async).
+
+    The extension sets window.__SENTIENCE_EXTENSION_VERSION__ when injected.
+
+    Args:
+        page: Playwright Page object (async)
+        expected: If provided, raises RuntimeError on mismatch
+
+    Returns:
+        Version string if found, None if not set (page may not have injected yet)
+
+    Raises:
+        RuntimeError: If expected version provided and doesn't match
+    """
+    try:
+        got = await page.evaluate("window.__SENTIENCE_EXTENSION_VERSION__ || null")
+    except Exception:
+        got = None
+
+    if expected and got and got != expected:
+        raise RuntimeError(f"Sentience extension version mismatch: expected {expected}, got {got}")
+    return got

sentience/action_executor.py

@@ -0,0 +1,217 @@
+"""
+Action Executor for Sentience Agent.
+
+Handles parsing and execution of action commands (CLICK, TYPE, PRESS, FINISH).
+This separates action execution concerns from LLM interaction.
+"""
+
+import re
+from typing import Any, Union
+
+from .actions import click, click_async, press, press_async, type_text, type_text_async
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+from .models import Snapshot
+from .protocols import AsyncBrowserProtocol, BrowserProtocol
+
+
+class ActionExecutor:
+    """
+    Executes actions and handles parsing of action command strings.
+
+    This class encapsulates all action execution logic, making it easier to:
+    - Test action execution independently
+    - Add new action types in one place
+    - Handle action parsing errors consistently
+    """
+
+    def __init__(
+        self,
+        browser: SentienceBrowser | AsyncSentienceBrowser | BrowserProtocol | AsyncBrowserProtocol,
+    ):
+        """
+        Initialize action executor.
+
+        Args:
+            browser: SentienceBrowser, AsyncSentienceBrowser, or protocol-compatible instance
+                    (for testing, can use mock objects that implement BrowserProtocol)
+        """
+        self.browser = browser
+        # Check if browser is async - support both concrete types and protocols
+        # Check concrete types first (most reliable)
+        if isinstance(browser, AsyncSentienceBrowser):
+            self._is_async = True
+        elif isinstance(browser, SentienceBrowser):
+            self._is_async = False
+        else:
+            # For protocol-based browsers, check if methods are actually async
+            # This is more reliable than isinstance checks which can match both protocols
+            import inspect
+
+            start_method = getattr(browser, "start", None)
+            if start_method and inspect.iscoroutinefunction(start_method):
+                self._is_async = True
+            elif isinstance(browser, BrowserProtocol):
+                # If it implements BrowserProtocol and start is not async, it's sync
+                self._is_async = False
+            else:
+                # Default to sync for unknown types
+                self._is_async = False
+
+    def execute(self, action_str: str, snap: Snapshot) -> dict[str, Any]:
+        """
+        Parse action string and execute SDK call (synchronous).
+
+        Args:
+            action_str: Action string from LLM (e.g., "CLICK(42)", "TYPE(15, \"text\")")
+            snap: Current snapshot (for context, currently unused but kept for API consistency)
+
+        Returns:
+            Execution result dictionary with keys:
+            - success: bool
+            - action: str (e.g., "click", "type", "press", "finish")
+            - element_id: Optional[int] (for click/type actions)
+            - text: Optional[str] (for type actions)
+            - key: Optional[str] (for press actions)
+            - outcome: Optional[str] (action outcome)
+            - url_changed: Optional[bool] (for click actions)
+            - error: Optional[str] (if action failed)
+            - message: Optional[str] (for finish action)
+
+        Raises:
+            ValueError: If action format is unknown
+            RuntimeError: If called on async browser (use execute_async instead)
+        """
+        if self._is_async:
+            raise RuntimeError(
+                "ActionExecutor.execute() called on async browser. Use execute_async() instead."
+            )
+
+        # Parse CLICK(42)
+        if match := re.match(r"CLICK\s*\(\s*(\d+)\s*\)", action_str, re.IGNORECASE):
+            element_id = int(match.group(1))
+            result = click(self.browser, element_id)  # type: ignore
+            return {
+                "success": result.success,
+                "action": "click",
+                "element_id": element_id,
+                "outcome": result.outcome,
+                "url_changed": result.url_changed,
+                "cursor": getattr(result, "cursor", None),
+            }
+
+        # Parse TYPE(42, "hello world")
+        elif match := re.match(
+            r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)',
+            action_str,
+            re.IGNORECASE,
+        ):
+            element_id = int(match.group(1))
+            text = match.group(2)
+            result = type_text(self.browser, element_id, text)  # type: ignore
+            return {
+                "success": result.success,
+                "action": "type",
+                "element_id": element_id,
+                "text": text,
+                "outcome": result.outcome,
+            }
+
+        # Parse PRESS("Enter")
+        elif match := re.match(r'PRESS\s*\(\s*["\']([^"\']+)["\']\s*\)', action_str, re.IGNORECASE):
+            key = match.group(1)
+            result = press(self.browser, key)  # type: ignore
+            return {
+                "success": result.success,
+                "action": "press",
+                "key": key,
+                "outcome": result.outcome,
+            }
+
+        # Parse FINISH()
+        elif re.match(r"FINISH\s*\(\s*\)", action_str, re.IGNORECASE):
+            return {
+                "success": True,
+                "action": "finish",
+                "message": "Task marked as complete",
+            }
+
+        else:
+            raise ValueError(
+                f"Unknown action format: {action_str}\n"
+                f'Expected: CLICK(id), TYPE(id, "text"), PRESS("key"), or FINISH()'
+            )
+
+    async def execute_async(self, action_str: str, snap: Snapshot) -> dict[str, Any]:
+        """
+        Parse action string and execute SDK call (asynchronous).
+
+        Args:
+            action_str: Action string from LLM (e.g., "CLICK(42)", "TYPE(15, \"text\")")
+            snap: Current snapshot (for context, currently unused but kept for API consistency)
+
+        Returns:
+            Execution result dictionary (same format as execute())
+
+        Raises:
+            ValueError: If action format is unknown
+            RuntimeError: If called on sync browser (use execute() instead)
+        """
+        if not self._is_async:
+            raise RuntimeError(
+                "ActionExecutor.execute_async() called on sync browser. Use execute() instead."
+            )
+
+        # Parse CLICK(42)
+        if match := re.match(r"CLICK\s*\(\s*(\d+)\s*\)", action_str, re.IGNORECASE):
+            element_id = int(match.group(1))
+            result = await click_async(self.browser, element_id)  # type: ignore
+            return {
+                "success": result.success,
+                "action": "click",
+                "element_id": element_id,
+                "outcome": result.outcome,
+                "url_changed": result.url_changed,
+                "cursor": getattr(result, "cursor", None),
+            }
+
+        # Parse TYPE(42, "hello world")
+        elif match := re.match(
+            r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)',
+            action_str,
+            re.IGNORECASE,
+        ):
+            element_id = int(match.group(1))
+            text = match.group(2)
+            result = await type_text_async(self.browser, element_id, text)  # type: ignore
+            return {
+                "success": result.success,
+                "action": "type",
+                "element_id": element_id,
+                "text": text,
+                "outcome": result.outcome,
+            }
+
+        # Parse PRESS("Enter")
+        elif match := re.match(r'PRESS\s*\(\s*["\']([^"\']+)["\']\s*\)', action_str, re.IGNORECASE):
+            key = match.group(1)
+            result = await press_async(self.browser, key)  # type: ignore
+            return {
+                "success": result.success,
+                "action": "press",
+                "key": key,
+                "outcome": result.outcome,
+            }
+
+        # Parse FINISH()
+        elif re.match(r"FINISH\s*\(\s*\)", action_str, re.IGNORECASE):
+            return {
+                "success": True,
+                "action": "finish",
+                "message": "Task marked as complete",
+            }
+
+        else:
+            raise ValueError(
+                f"Unknown action format: {action_str}\n"
+                f'Expected: CLICK(id), TYPE(id, "text"), PRESS("key"), or FINISH()'
+            )