sentienceapi 0.90.9__py3-none-any.whl

sentience/snapshot.py

@@ -0,0 +1,282 @@
+"""
+Snapshot functionality - calls window.sentience.snapshot() or server-side API
+"""
+
+import json
+import os
+import time
+from typing import Any
+
+import requests
+
+from .browser import SentienceBrowser
+from .models import Snapshot, SnapshotOptions
+
+# Maximum payload size for API requests (10MB server limit)
+MAX_PAYLOAD_BYTES = 10 * 1024 * 1024
+
+
+def _save_trace_to_file(raw_elements: list[dict[str, Any]], trace_path: str | None = None) -> None:
+    """
+    Save raw_elements to a JSON file for benchmarking/training
+
+    Args:
+        raw_elements: Raw elements data from snapshot
+        trace_path: Path to save trace file. If None, uses "trace_{timestamp}.json"
+    """
+    # Default filename if none provided
+    filename = trace_path or f"trace_{int(time.time())}.json"
+
+    # Ensure directory exists
+    directory = os.path.dirname(filename)
+    if directory:
+        os.makedirs(directory, exist_ok=True)
+
+    # Save the raw elements to JSON
+    with open(filename, "w") as f:
+        json.dump(raw_elements, f, indent=2)
+
+    print(f"[SDK] Trace saved to: {filename}")
+
+
+def snapshot(
+    browser: SentienceBrowser,
+    screenshot: bool | None = None,
+    limit: int | None = None,
+    filter: dict[str, Any] | None = None,
+    use_api: bool | None = None,
+    save_trace: bool = False,
+    trace_path: str | None = None,
+    show_overlay: bool = False,
+) -> Snapshot:
+    """
+    Take a snapshot of the current page
+
+    Args:
+        browser: SentienceBrowser instance
+        screenshot: Whether to capture screenshot (bool or dict with format/quality)
+        limit: Limit number of elements returned
+        filter: Filter options (min_area, allowed_roles, min_z_index)
+        use_api: Force use of server-side API if True, local extension if False.
+                 If None, uses API if api_key is set, otherwise uses local extension.
+        save_trace: Whether to save raw_elements to JSON for benchmarking/training
+        trace_path: Path to save trace file. If None, uses "trace_{timestamp}.json"
+        show_overlay: Show visual overlay highlighting elements in browser
+
+    Returns:
+        Snapshot object
+    """
+    # Build SnapshotOptions from individual parameters
+    options = SnapshotOptions(
+        screenshot=screenshot if screenshot is not None else False,
+        limit=limit if limit is not None else 50,
+        filter=filter,
+        use_api=use_api,
+        save_trace=save_trace,
+        trace_path=trace_path,
+        show_overlay=show_overlay,
+    )
+
+    # Determine if we should use server-side API
+    should_use_api = (
+        options.use_api if options.use_api is not None else (browser.api_key is not None)
+    )
+
+    if should_use_api and browser.api_key:
+        # Use server-side API (Pro/Enterprise tier)
+        return _snapshot_via_api(browser, options)
+    else:
+        # Use local extension (Free tier)
+        return _snapshot_via_extension(browser, options)
+
+
+def _snapshot_via_extension(
+    browser: SentienceBrowser,
+    options: SnapshotOptions,
+) -> Snapshot:
+    """Take snapshot using local extension (Free tier)"""
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
+    # The new architecture loads injected_api.js asynchronously, so window.sentience
+    # may not be immediately available after page load
+    try:
+        browser.page.wait_for_function(
+            "typeof window.sentience !== 'undefined'",
+            timeout=5000,  # 5 second timeout
+        )
+    except Exception as e:
+        # Gather diagnostics if wait fails
+        try:
+            diag = browser.page.evaluate(
+                """() => ({
+                    sentience_defined: typeof window.sentience !== 'undefined',
+                    extension_id: document.documentElement.dataset.sentienceExtensionId || 'not set',
+                    url: window.location.href
+                })"""
+            )
+        except Exception:
+            diag = {"error": "Could not gather diagnostics"}
+
+        raise RuntimeError(
+            f"Sentience extension failed to inject window.sentience API. "
+            f"Is the extension loaded? Diagnostics: {diag}"
+        ) from e
+
+    # Build options dict for extension API (exclude save_trace/trace_path)
+    ext_options: dict[str, Any] = {}
+    if options.screenshot is not False:
+        ext_options["screenshot"] = options.screenshot
+    if options.limit != 50:
+        ext_options["limit"] = options.limit
+    if options.filter is not None:
+        ext_options["filter"] = (
+            options.filter.model_dump() if hasattr(options.filter, "model_dump") else options.filter
+        )
+
+    # Call extension API
+    result = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.snapshot(options);
+        }
+        """,
+        ext_options,
+    )
+
+    # Save trace if requested
+    if options.save_trace:
+        _save_trace_to_file(result.get("raw_elements", []), options.trace_path)
+
+    # Show visual overlay if requested
+    if options.show_overlay:
+        raw_elements = result.get("raw_elements", [])
+        if raw_elements:
+            browser.page.evaluate(
+                """
+                (elements) => {
+                    if (window.sentience && window.sentience.showOverlay) {
+                        window.sentience.showOverlay(elements, null);
+                    }
+                }
+                """,
+                raw_elements,
+            )
+
+    # Validate and parse with Pydantic
+    snapshot_obj = Snapshot(**result)
+    return snapshot_obj
+
+
+def _snapshot_via_api(
+    browser: SentienceBrowser,
+    options: SnapshotOptions,
+) -> Snapshot:
+    """Take snapshot using server-side API (Pro/Enterprise tier)"""
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    if not browser.api_key:
+        raise ValueError("API key required for server-side processing")
+
+    if not browser.api_url:
+        raise ValueError("API URL required for server-side processing")
+
+    # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
+    # Even for API mode, we need the extension to collect raw data locally
+    try:
+        browser.page.wait_for_function("typeof window.sentience !== 'undefined'", timeout=5000)
+    except Exception as e:
+        raise RuntimeError(
+            "Sentience extension failed to inject. Cannot collect raw data for API processing."
+        ) from e
+
+    # Step 1: Get raw data from local extension (always happens locally)
+    raw_options: dict[str, Any] = {}
+    if options.screenshot is not False:
+        raw_options["screenshot"] = options.screenshot
+
+    raw_result = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.snapshot(options);
+        }
+        """,
+        raw_options,
+    )
+
+    # Save trace if requested (save raw data before API processing)
+    if options.save_trace:
+        _save_trace_to_file(raw_result.get("raw_elements", []), options.trace_path)
+
+    # Step 2: Send to server for smart ranking/filtering
+    # Use raw_elements (raw data) instead of elements (processed data)
+    # Server validates API key and applies proprietary ranking logic
+    payload = {
+        "raw_elements": raw_result.get("raw_elements", []),  # Raw data needed for server processing
+        "url": raw_result.get("url", ""),
+        "viewport": raw_result.get("viewport"),
+        "goal": options.goal,  # Optional goal/task description
+        "options": {
+            "limit": options.limit,
+            "filter": options.filter.model_dump() if options.filter else None,
+        },
+    }
+
+    # Check payload size before sending (server has 10MB limit)
+    payload_json = json.dumps(payload)
+    payload_size = len(payload_json.encode("utf-8"))
+    if payload_size > MAX_PAYLOAD_BYTES:
+        raise ValueError(
+            f"Payload size ({payload_size / 1024 / 1024:.2f}MB) exceeds server limit "
+            f"({MAX_PAYLOAD_BYTES / 1024 / 1024:.0f}MB). "
+            f"Try reducing the number of elements on the page or filtering elements."
+        )
+
+    headers = {
+        "Authorization": f"Bearer {browser.api_key}",
+        "Content-Type": "application/json",
+    }
+
+    try:
+        response = requests.post(
+            f"{browser.api_url}/v1/snapshot",
+            data=payload_json,  # Reuse already-serialized JSON
+            headers=headers,
+            timeout=30,
+        )
+        response.raise_for_status()
+
+        api_result = response.json()
+
+        # Merge API result with local data (screenshot, etc.)
+        snapshot_data = {
+            "status": api_result.get("status", "success"),
+            "timestamp": api_result.get("timestamp"),
+            "url": api_result.get("url", raw_result.get("url", "")),
+            "viewport": api_result.get("viewport", raw_result.get("viewport")),
+            "elements": api_result.get("elements", []),
+            "screenshot": raw_result.get("screenshot"),  # Keep local screenshot
+            "screenshot_format": raw_result.get("screenshot_format"),
+            "error": api_result.get("error"),
+        }
+
+        # Show visual overlay if requested (use API-ranked elements)
+        if options.show_overlay:
+            elements = api_result.get("elements", [])
+            if elements:
+                browser.page.evaluate(
+                    """
+                    (elements) => {
+                        if (window.sentience && window.sentience.showOverlay) {
+                            window.sentience.showOverlay(elements, null);
+                        }
+                    }
+                    """,
+                    elements,
+                )
+
+        return Snapshot(**snapshot_data)
+    except requests.exceptions.RequestException as e:
+        raise RuntimeError(f"API request failed: {e}")

sentience/text_search.py

@@ -0,0 +1,107 @@
+"""
+Text search utilities - find text and get pixel coordinates
+"""
+
+from .browser import SentienceBrowser
+from .models import TextRectSearchResult
+
+
+def find_text_rect(
+    browser: SentienceBrowser,
+    text: str,
+    case_sensitive: bool = False,
+    whole_word: bool = False,
+    max_results: int = 10,
+) -> TextRectSearchResult:
+    """
+    Find all occurrences of text on the page and get their exact pixel coordinates.
+
+    This function searches for text in all visible text nodes on the page and returns
+    the bounding rectangles for each match. Useful for:
+    - Finding specific UI elements by their text content
+    - Locating buttons, links, or labels without element IDs
+    - Getting exact coordinates for click automation
+    - Highlighting search results visually
+
+    Args:
+        browser: SentienceBrowser instance
+        text: Text to search for (required)
+        case_sensitive: If True, search is case-sensitive (default: False)
+        whole_word: If True, only match whole words surrounded by whitespace (default: False)
+        max_results: Maximum number of matches to return (default: 10, max: 100)
+
+    Returns:
+        TextRectSearchResult with:
+            - status: "success" or "error"
+            - query: The search text
+            - case_sensitive: Whether search was case-sensitive
+            - whole_word: Whether whole-word matching was used
+            - matches: Number of matches found
+            - results: List of TextMatch objects, each containing:
+                - text: The matched text
+                - rect: Absolute rectangle (with scroll offset)
+                - viewport_rect: Viewport-relative rectangle
+                - context: Surrounding text (before/after)
+                - in_viewport: Whether visible in current viewport
+            - viewport: Current viewport dimensions and scroll position
+            - error: Error message if status is "error"
+
+    Examples:
+        # Find "Sign In" button
+        result = find_text_rect(browser, "Sign In")
+        if result.status == "success" and result.results:
+            first_match = result.results[0]
+            print(f"Found at: ({first_match.rect.x}, {first_match.rect.y})")
+            print(f"Size: {first_match.rect.width}x{first_match.rect.height}")
+            print(f"In viewport: {first_match.in_viewport}")
+
+        # Case-sensitive search
+        result = find_text_rect(browser, "LOGIN", case_sensitive=True)
+
+        # Whole word only
+        result = find_text_rect(browser, "log", whole_word=True)  # Won't match "login"
+
+        # Find all matches and click the first visible one
+        result = find_text_rect(browser, "Buy Now", max_results=5)
+        if result.status == "success" and result.results:
+            for match in result.results:
+                if match.in_viewport:
+                    # Use click_rect from actions module
+                    from sentience import click_rect
+                    click_result = click_rect(browser, {
+                        "x": match.rect.x,
+                        "y": match.rect.y,
+                        "w": match.rect.width,
+                        "h": match.rect.height
+                    })
+                    break
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    if not text or not text.strip():
+        return TextRectSearchResult(
+            status="error",
+            error="Text parameter is required and cannot be empty",
+        )
+
+    # Limit max_results to prevent performance issues
+    max_results = min(max_results, 100)
+
+    # Call the extension's findTextRect method
+    result_dict = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.findTextRect(options);
+        }
+        """,
+        {
+            "text": text,
+            "caseSensitive": case_sensitive,
+            "wholeWord": whole_word,
+            "maxResults": max_results,
+        },
+    )
+
+    # Parse and validate with Pydantic
+    return TextRectSearchResult(**result_dict)

sentience/trace_indexing/__init__.py

@@ -0,0 +1,27 @@
+"""
+Trace indexing module for Sentience SDK.
+"""
+
+from .indexer import build_trace_index, write_trace_index, read_step_events
+from .index_schema import (
+    TraceIndex,
+    StepIndex,
+    TraceSummary,
+    TraceFileInfo,
+    SnapshotInfo,
+    ActionInfo,
+    StepCounters,
+)
+
+__all__ = [
+    "build_trace_index",
+    "write_trace_index",
+    "read_step_events",
+    "TraceIndex",
+    "StepIndex",
+    "TraceSummary",
+    "TraceFileInfo",
+    "SnapshotInfo",
+    "ActionInfo",
+    "StepCounters",
+]

sentience/trace_indexing/index_schema.py

@@ -0,0 +1,111 @@
+"""
+Type definitions for trace index schema using concrete classes.
+"""
+
+from dataclasses import dataclass, field, asdict
+from typing import Optional, List, Literal
+
+
+@dataclass
+class TraceFileInfo:
+    """Metadata about the trace file."""
+
+    path: str
+    size_bytes: int
+    sha256: str
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class TraceSummary:
+    """High-level summary of the trace."""
+
+    first_ts: str
+    last_ts: str
+    event_count: int
+    step_count: int
+    error_count: int
+    final_url: Optional[str]
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class SnapshotInfo:
+    """Snapshot metadata for index."""
+
+    snapshot_id: Optional[str] = None
+    digest: Optional[str] = None
+    url: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class ActionInfo:
+    """Action metadata for index."""
+
+    type: Optional[str] = None
+    target_element_id: Optional[int] = None
+    args_digest: Optional[str] = None
+    success: Optional[bool] = None
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class StepCounters:
+    """Event counters per step."""
+
+    events: int = 0
+    snapshots: int = 0
+    actions: int = 0
+    llm_calls: int = 0
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class StepIndex:
+    """Index entry for a single step."""
+
+    step_index: int
+    step_id: str
+    goal: Optional[str]
+    status: Literal["ok", "error", "partial"]
+    ts_start: str
+    ts_end: str
+    offset_start: int
+    offset_end: int
+    url_before: Optional[str]
+    url_after: Optional[str]
+    snapshot_before: SnapshotInfo
+    snapshot_after: SnapshotInfo
+    action: ActionInfo
+    counters: StepCounters
+
+    def to_dict(self) -> dict:
+        result = asdict(self)
+        return result
+
+
+@dataclass
+class TraceIndex:
+    """Complete trace index schema."""
+
+    version: int
+    run_id: str
+    created_at: str
+    trace_file: TraceFileInfo
+    summary: TraceSummary
+    steps: List[StepIndex] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for JSON serialization."""
+        return asdict(self)