sentienceapi 0.90.9__py3-none-any.whl

sentience/__init__.py

@@ -0,0 +1,153 @@
+"""
+Sentience Python SDK - AI Agent Browser Automation
+"""
+
+from .actions import click, click_rect, press, type_text
+from .agent import SentienceAgent
+from .agent_config import AgentConfig
+
+# Agent Layer (Phase 1 & 2)
+from .base_agent import BaseAgent
+from .browser import SentienceBrowser
+
+# Tracing (v0.12.0+)
+from .cloud_tracing import CloudTraceSink, SentienceLogger
+from .conversational_agent import ConversationalAgent
+from .expect import expect
+
+# Formatting (v0.12.0+)
+from .formatting import format_snapshot_for_llm
+from .generator import ScriptGenerator, generate
+from .inspector import Inspector, inspect
+from .llm_provider import (
+    AnthropicProvider,
+    LLMProvider,
+    LLMResponse,
+    LocalLLMProvider,
+    OpenAIProvider,
+)
+from .models import (  # Agent Layer Models
+    ActionHistory,
+    ActionResult,
+    ActionTokenUsage,
+    AgentActionResult,
+    BBox,
+    Cookie,
+    Element,
+    LocalStorageItem,
+    OriginStorage,
+    ScreenshotConfig,
+    Snapshot,
+    SnapshotFilter,
+    SnapshotOptions,
+    StorageState,
+    TextContext,
+    TextMatch,
+    TextRect,
+    TextRectSearchResult,
+    TokenStats,
+    Viewport,
+    ViewportRect,
+    WaitResult,
+)
+from .overlay import clear_overlay, show_overlay
+from .query import find, query
+from .read import read
+from .recorder import Recorder, Trace, TraceStep, record
+from .screenshot import screenshot
+from .snapshot import snapshot
+from .text_search import find_text_rect
+from .tracer_factory import SENTIENCE_API_URL, create_tracer
+from .tracing import JsonlTraceSink, TraceEvent, Tracer, TraceSink
+
+# Utilities (v0.12.0+)
+from .utils import (
+    canonical_snapshot_loose,
+    canonical_snapshot_strict,
+    compute_snapshot_digests,
+    save_storage_state,
+    sha256_digest,
+)
+from .wait import wait_for
+
+__version__ = "0.90.9"
+
+__all__ = [
+    # Core SDK
+    "SentienceBrowser",
+    "Snapshot",
+    "Element",
+    "BBox",
+    "Viewport",
+    "ActionResult",
+    "WaitResult",
+    "snapshot",
+    "query",
+    "find",
+    "click",
+    "type_text",
+    "press",
+    "click_rect",
+    "wait_for",
+    "expect",
+    "Inspector",
+    "inspect",
+    "Recorder",
+    "Trace",
+    "TraceStep",
+    "record",
+    "ScriptGenerator",
+    "generate",
+    "read",
+    "screenshot",
+    "show_overlay",
+    "clear_overlay",
+    # Text Search
+    "find_text_rect",
+    "TextRectSearchResult",
+    "TextMatch",
+    "TextRect",
+    "ViewportRect",
+    "TextContext",
+    # Agent Layer (Phase 1 & 2)
+    "BaseAgent",
+    "LLMProvider",
+    "LLMResponse",
+    "OpenAIProvider",
+    "AnthropicProvider",
+    "LocalLLMProvider",
+    "SentienceAgent",
+    "ConversationalAgent",
+    # Agent Layer Models
+    "AgentActionResult",
+    "TokenStats",
+    "ActionHistory",
+    "ActionTokenUsage",
+    "SnapshotOptions",
+    "SnapshotFilter",
+    "ScreenshotConfig",
+    # Storage State Models (Auth Injection)
+    "StorageState",
+    "Cookie",
+    "LocalStorageItem",
+    "OriginStorage",
+    # Tracing (v0.12.0+)
+    "Tracer",
+    "TraceSink",
+    "JsonlTraceSink",
+    "CloudTraceSink",
+    "SentienceLogger",
+    "TraceEvent",
+    "create_tracer",
+    "SENTIENCE_API_URL",
+    # Utilities (v0.12.0+)
+    "canonical_snapshot_strict",
+    "canonical_snapshot_loose",
+    "compute_snapshot_digests",
+    "sha256_digest",
+    "save_storage_state",
+    # Formatting (v0.12.0+)
+    "format_snapshot_for_llm",
+    # Agent Config (v0.12.0+)
+    "AgentConfig",
+]

sentience/actions.py

@@ -0,0 +1,439 @@
+"""
+Actions v1 - click, type, press
+"""
+
+import time
+
+from .browser import SentienceBrowser
+from .models import ActionResult, BBox, Snapshot
+from .snapshot import snapshot
+
+
+def click(  # noqa: C901
+    browser: SentienceBrowser,
+    element_id: int,
+    use_mouse: bool = True,
+    take_snapshot: bool = False,
+) -> ActionResult:
+    """
+    Click an element by ID using hybrid approach (mouse simulation by default)
+
+    Args:
+        browser: SentienceBrowser instance
+        element_id: Element ID from snapshot
+        use_mouse: If True, use Playwright's mouse.click() at element center (hybrid approach).
+                   If False, use JS-based window.sentience.click() (legacy).
+        take_snapshot: Whether to take snapshot after action
+
+    Returns:
+        ActionResult
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    if use_mouse:
+        # Hybrid approach: Get element bbox from snapshot, calculate center, use mouse.click()
+        try:
+            snap = snapshot(browser)
+            element = None
+            for el in snap.elements:
+                if el.id == element_id:
+                    element = el
+                    break
+
+            if element:
+                # Calculate center of element bbox
+                center_x = element.bbox.x + element.bbox.width / 2
+                center_y = element.bbox.y + element.bbox.height / 2
+                # Use Playwright's native mouse click for realistic simulation
+                try:
+                    browser.page.mouse.click(center_x, center_y)
+                    success = True
+                except Exception:
+                    # If navigation happens, mouse.click might fail, but that's OK
+                    # The click still happened, just check URL change
+                    success = True
+            else:
+                # Fallback to JS click if element not found in snapshot
+                try:
+                    success = browser.page.evaluate(
+                        """
+                        (id) => {
+                            return window.sentience.click(id);
+                        }
+                        """,
+                        element_id,
+                    )
+                except Exception:
+                    # Navigation might have destroyed context, assume success if URL changed
+                    success = True
+        except Exception:
+            # Fallback to JS click on error
+            try:
+                success = browser.page.evaluate(
+                    """
+                    (id) => {
+                        return window.sentience.click(id);
+                    }
+                    """,
+                    element_id,
+                )
+            except Exception:
+                # Navigation might have destroyed context, assume success if URL changed
+                success = True
+    else:
+        # Legacy JS-based click
+        success = browser.page.evaluate(
+            """
+            (id) => {
+                return window.sentience.click(id);
+            }
+            """,
+            element_id,
+        )
+
+    # Wait a bit for navigation/DOM updates
+    try:
+        browser.page.wait_for_timeout(500)
+    except Exception:
+        # Navigation might have happened, context destroyed
+        pass
+
+    duration_ms = int((time.time() - start_time) * 1000)
+
+    # Check if URL changed (handle navigation gracefully)
+    try:
+        url_after = browser.page.url
+        url_changed = url_before != url_after
+    except Exception:
+        # Context destroyed due to navigation - assume URL changed
+        url_after = url_before
+        url_changed = True
+
+    # Determine outcome
+    outcome: str | None = None
+    if url_changed:
+        outcome = "navigated"
+    elif success:
+        outcome = "dom_updated"
+    else:
+        outcome = "error"
+
+    # Optional snapshot after
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        try:
+            snapshot_after = snapshot(browser)
+        except Exception:
+            # Navigation might have destroyed context
+            pass
+
+    return ActionResult(
+        success=success,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+        error=(
+            None
+            if success
+            else {
+                "code": "click_failed",
+                "reason": "Element not found or not clickable",
+            }
+        ),
+    )
+
+
+def type_text(
+    browser: SentienceBrowser, element_id: int, text: str, take_snapshot: bool = False
+) -> ActionResult:
+    """
+    Type text into an element (focus then input)
+
+    Args:
+        browser: SentienceBrowser instance
+        element_id: Element ID from snapshot
+        text: Text to type
+        take_snapshot: Whether to take snapshot after action
+
+    Returns:
+        ActionResult
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    # Focus element first using extension registry
+    focused = browser.page.evaluate(
+        """
+        (id) => {
+            const el = window.sentience_registry[id];
+            if (el) {
+                el.focus();
+                return true;
+            }
+            return false;
+        }
+        """,
+        element_id,
+    )
+
+    if not focused:
+        return ActionResult(
+            success=False,
+            duration_ms=int((time.time() - start_time) * 1000),
+            outcome="error",
+            error={"code": "focus_failed", "reason": "Element not found"},
+        )
+
+    # Type using Playwright keyboard
+    browser.page.keyboard.type(text)
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = snapshot(browser)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
+def press(browser: SentienceBrowser, key: str, take_snapshot: bool = False) -> ActionResult:
+    """
+    Press a keyboard key
+
+    Args:
+        browser: SentienceBrowser instance
+        key: Key to press (e.g., "Enter", "Escape", "Tab")
+        take_snapshot: Whether to take snapshot after action
+
+    Returns:
+        ActionResult
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    # Press key using Playwright
+    browser.page.keyboard.press(key)
+
+    # Wait a bit for navigation/DOM updates
+    browser.page.wait_for_timeout(500)
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = snapshot(browser)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
+def _highlight_rect(
+    browser: SentienceBrowser, rect: dict[str, float], duration_sec: float = 2.0
+) -> None:
+    """
+    Highlight a rectangle with a red border overlay
+
+    Args:
+        browser: SentienceBrowser instance
+        rect: Dictionary with x, y, width (w), height (h) keys
+        duration_sec: How long to show the highlight (default: 2 seconds)
+    """
+    if not browser.page:
+        return
+
+    # Create a unique ID for this highlight
+    highlight_id = f"sentience_highlight_{int(time.time() * 1000)}"
+
+    # Combine all arguments into a single object for Playwright
+    args = {
+        "rect": {
+            "x": rect["x"],
+            "y": rect["y"],
+            "w": rect["w"],
+            "h": rect["h"],
+        },
+        "highlightId": highlight_id,
+        "durationSec": duration_sec,
+    }
+
+    # Inject CSS and create overlay element
+    browser.page.evaluate(
+        """
+        (args) => {
+            const { rect, highlightId, durationSec } = args;
+            // Create overlay div
+            const overlay = document.createElement('div');
+            overlay.id = highlightId;
+            overlay.style.position = 'fixed';
+            overlay.style.left = `${rect.x}px`;
+            overlay.style.top = `${rect.y}px`;
+            overlay.style.width = `${rect.w}px`;
+            overlay.style.height = `${rect.h}px`;
+            overlay.style.border = '3px solid red';
+            overlay.style.borderRadius = '2px';
+            overlay.style.boxSizing = 'border-box';
+            overlay.style.pointerEvents = 'none';
+            overlay.style.zIndex = '999999';
+            overlay.style.backgroundColor = 'rgba(255, 0, 0, 0.1)';
+            overlay.style.transition = 'opacity 0.3s ease-out';
+
+            document.body.appendChild(overlay);
+
+            // Remove after duration
+            setTimeout(() => {
+                overlay.style.opacity = '0';
+                setTimeout(() => {
+                    if (overlay.parentNode) {
+                        overlay.parentNode.removeChild(overlay);
+                    }
+                }, 300); // Wait for fade-out transition
+            }, durationSec * 1000);
+        }
+        """,
+        args,
+    )
+
+
+def click_rect(
+    browser: SentienceBrowser,
+    rect: dict[str, float],
+    highlight: bool = True,
+    highlight_duration: float = 2.0,
+    take_snapshot: bool = False,
+) -> ActionResult:
+    """
+    Click at the center of a rectangle using Playwright's native mouse simulation.
+    This uses a hybrid approach: calculates center coordinates and uses mouse.click()
+    for realistic event simulation (triggers hover, focus, mousedown, mouseup).
+
+    Args:
+        browser: SentienceBrowser instance
+        rect: Dictionary with x, y, width (w), height (h) keys, or BBox object
+        highlight: Whether to show a red border highlight when clicking (default: True)
+        highlight_duration: How long to show the highlight in seconds (default: 2.0)
+        take_snapshot: Whether to take snapshot after action
+
+    Returns:
+        ActionResult
+
+    Example:
+        >>> click_rect(browser, {"x": 100, "y": 200, "w": 50, "h": 30})
+        >>> # Or using BBox object
+        >>> from sentience import BBox
+        >>> bbox = BBox(x=100, y=200, width=50, height=30)
+        >>> click_rect(browser, {"x": bbox.x, "y": bbox.y, "w": bbox.width, "h": bbox.height})
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    # Handle BBox object or dict
+    if isinstance(rect, BBox):
+        x = rect.x
+        y = rect.y
+        w = rect.width
+        h = rect.height
+    else:
+        x = rect.get("x", 0)
+        y = rect.get("y", 0)
+        w = rect.get("w") or rect.get("width", 0)
+        h = rect.get("h") or rect.get("height", 0)
+
+    if w <= 0 or h <= 0:
+        return ActionResult(
+            success=False,
+            duration_ms=0,
+            outcome="error",
+            error={
+                "code": "invalid_rect",
+                "reason": "Rectangle width and height must be positive",
+            },
+        )
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    # Calculate center of rectangle
+    center_x = x + w / 2
+    center_y = y + h / 2
+
+    # Show highlight before clicking (if enabled)
+    if highlight:
+        _highlight_rect(browser, {"x": x, "y": y, "w": w, "h": h}, highlight_duration)
+        # Small delay to ensure highlight is visible
+        browser.page.wait_for_timeout(50)
+
+    # Use Playwright's native mouse click for realistic simulation
+    # This triggers hover, focus, mousedown, mouseup sequences
+    try:
+        browser.page.mouse.click(center_x, center_y)
+        success = True
+    except Exception as e:
+        success = False
+        error_msg = str(e)
+
+    # Wait a bit for navigation/DOM updates
+    browser.page.wait_for_timeout(500)
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+
+    # Determine outcome
+    outcome: str | None = None
+    if url_changed:
+        outcome = "navigated"
+    elif success:
+        outcome = "dom_updated"
+    else:
+        outcome = "error"
+
+    # Optional snapshot after
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = snapshot(browser)
+
+    return ActionResult(
+        success=success,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+        error=(
+            None
+            if success
+            else {
+                "code": "click_failed",
+                "reason": error_msg if not success else "Click failed",
+            }
+        ),
+    )