sentienceapi 0.90.16__py3-none-any.whl → 0.92.2__py3-none-any.whl

sentience/snapshot.py

@@ -2,6 +2,7 @@
 Snapshot functionality - calls window.sentience.snapshot() or server-side API
 """
 
+import asyncio
 import json
 import os
 import time
@@ -9,8 +10,10 @@ from typing import Any, Optional
 
 import requests
 
-from .browser import SentienceBrowser
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+from .browser_evaluator import BrowserEvaluator
 from .models import Snapshot, SnapshotOptions
+from .sentience_methods import SentienceMethod
 
 # Maximum payload size for API requests (10MB server limit)
 MAX_PAYLOAD_BYTES = 10 * 1024 * 1024
@@ -93,33 +96,16 @@ def _snapshot_via_extension(
     # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
     # The new architecture loads injected_api.js asynchronously, so window.sentience
     # may not be immediately available after page load
-    try:
-        browser.page.wait_for_function(
-            "typeof window.sentience !== 'undefined'",
-            timeout=5000,  # 5 second timeout
-        )
-    except Exception as e:
-        # Gather diagnostics if wait fails
-        try:
-            diag = browser.page.evaluate(
-                """() => ({
-                    sentience_defined: typeof window.sentience !== 'undefined',
-                    extension_id: document.documentElement.dataset.sentienceExtensionId || 'not set',
-                    url: window.location.href
-                })"""
-            )
-        except Exception:
-            diag = {"error": "Could not gather diagnostics"}
-
-        raise RuntimeError(
-            f"Sentience extension failed to inject window.sentience API. "
-            f"Is the extension loaded? Diagnostics: {diag}"
-        ) from e
+    BrowserEvaluator.wait_for_extension(browser.page, timeout_ms=5000)
 
     # Build options dict for extension API (exclude save_trace/trace_path)
     ext_options: dict[str, Any] = {}
     if options.screenshot is not False:
-        ext_options["screenshot"] = options.screenshot
+        # Serialize ScreenshotConfig to dict if it's a Pydantic model
+        if hasattr(options.screenshot, "model_dump"):
+            ext_options["screenshot"] = options.screenshot.model_dump()
+        else:
+            ext_options["screenshot"] = options.screenshot
     if options.limit != 50:
         ext_options["limit"] = options.limit
     if options.filter is not None:
@@ -177,26 +163,14 @@ def _snapshot_via_api(
 
     # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
     # Even for API mode, we need the extension to collect raw data locally
-    try:
-        browser.page.wait_for_function("typeof window.sentience !== 'undefined'", timeout=5000)
-    except Exception as e:
-        raise RuntimeError(
-            "Sentience extension failed to inject. Cannot collect raw data for API processing."
-        ) from e
+    BrowserEvaluator.wait_for_extension(browser.page, timeout_ms=5000)
 
     # Step 1: Get raw data from local extension (always happens locally)
     raw_options: dict[str, Any] = {}
     if options.screenshot is not False:
         raw_options["screenshot"] = options.screenshot
 
-    raw_result = browser.page.evaluate(
-        """
-        (options) => {
-            return window.sentience.snapshot(options);
-        }
-        """,
-        raw_options,
-    )
+    raw_result = BrowserEvaluator.invoke(browser.page, SentienceMethod.SNAPSHOT, **raw_options)
 
     # Save trace if requested (save raw data before API processing)
     if options.save_trace:
@@ -272,3 +246,282 @@ def _snapshot_via_api(
         return Snapshot(**snapshot_data)
     except requests.exceptions.RequestException as e:
         raise RuntimeError(f"API request failed: {e}")
+
+
+# ========== Async Snapshot Functions ==========
+
+
+async def snapshot_async(
+    browser: AsyncSentienceBrowser,
+    options: SnapshotOptions | None = None,
+) -> Snapshot:
+    """
+    Take a snapshot of the current page (async)
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        options: Snapshot options (screenshot, limit, filter, etc.)
+                If None, uses default options.
+
+    Returns:
+        Snapshot object
+
+    Example:
+        # Basic snapshot with defaults
+        snap = await snapshot_async(browser)
+
+        # With options
+        snap = await snapshot_async(browser, SnapshotOptions(
+            screenshot=True,
+            limit=100,
+            show_overlay=True
+        ))
+    """
+    # Use default options if none provided
+    if options is None:
+        options = SnapshotOptions()
+
+    # Determine if we should use server-side API
+    should_use_api = (
+        options.use_api if options.use_api is not None else (browser.api_key is not None)
+    )
+
+    if should_use_api and browser.api_key:
+        # Use server-side API (Pro/Enterprise tier)
+        return await _snapshot_via_api_async(browser, options)
+    else:
+        # Use local extension (Free tier)
+        return await _snapshot_via_extension_async(browser, options)
+
+
+async def _snapshot_via_extension_async(
+    browser: AsyncSentienceBrowser,
+    options: SnapshotOptions,
+) -> Snapshot:
+    """Take snapshot using local extension (Free tier) - async"""
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    # Wait for extension injection to complete
+    try:
+        await browser.page.wait_for_function(
+            "typeof window.sentience !== 'undefined'",
+            timeout=5000,
+        )
+    except Exception as e:
+        try:
+            diag = await browser.page.evaluate(
+                """() => ({
+                    sentience_defined: typeof window.sentience !== 'undefined',
+                    extension_id: document.documentElement.dataset.sentienceExtensionId || 'not set',
+                    url: window.location.href
+                })"""
+            )
+        except Exception:
+            diag = {"error": "Could not gather diagnostics"}
+
+        raise RuntimeError(
+            f"Sentience extension failed to inject window.sentience API. "
+            f"Is the extension loaded? Diagnostics: {diag}"
+        ) from e
+
+    # Build options dict for extension API
+    ext_options: dict[str, Any] = {}
+    if options.screenshot is not False:
+        # Serialize ScreenshotConfig to dict if it's a Pydantic model
+        if hasattr(options.screenshot, "model_dump"):
+            ext_options["screenshot"] = options.screenshot.model_dump()
+        else:
+            ext_options["screenshot"] = options.screenshot
+    if options.limit != 50:
+        ext_options["limit"] = options.limit
+    if options.filter is not None:
+        ext_options["filter"] = (
+            options.filter.model_dump() if hasattr(options.filter, "model_dump") else options.filter
+        )
+
+    # Call extension API
+    result = await browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.snapshot(options);
+        }
+        """,
+        ext_options,
+    )
+    if result.get("error"):
+        print(f"      Snapshot error: {result.get('error')}")
+
+    # Save trace if requested
+    if options.save_trace:
+        _save_trace_to_file(result.get("raw_elements", []), options.trace_path)
+
+    # Show visual overlay if requested
+    if options.show_overlay:
+        raw_elements = result.get("raw_elements", [])
+        if raw_elements:
+            await browser.page.evaluate(
+                """
+                (elements) => {
+                    if (window.sentience && window.sentience.showOverlay) {
+                        window.sentience.showOverlay(elements, null);
+                    }
+                }
+                """,
+                raw_elements,
+            )
+
+    # Extract screenshot_format from data URL if not provided by extension
+    if result.get("screenshot") and not result.get("screenshot_format"):
+        screenshot_data_url = result.get("screenshot", "")
+        if screenshot_data_url.startswith("data:image/"):
+            # Extract format from "data:image/jpeg;base64,..." or "data:image/png;base64,..."
+            format_match = screenshot_data_url.split(";")[0].split("/")[-1]
+            if format_match in ["jpeg", "jpg", "png"]:
+                result["screenshot_format"] = "jpeg" if format_match in ["jpeg", "jpg"] else "png"
+
+    # Validate and parse with Pydantic
+    snapshot_obj = Snapshot(**result)
+    return snapshot_obj
+
+
+async def _snapshot_via_api_async(
+    browser: AsyncSentienceBrowser,
+    options: SnapshotOptions,
+) -> Snapshot:
+    """Take snapshot using server-side API (Pro/Enterprise tier) - async"""
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    if not browser.api_key:
+        raise ValueError("API key required for server-side processing")
+
+    if not browser.api_url:
+        raise ValueError("API URL required for server-side processing")
+
+    # Wait for extension injection
+    try:
+        await browser.page.wait_for_function(
+            "typeof window.sentience !== 'undefined'", timeout=5000
+        )
+    except Exception as e:
+        raise RuntimeError(
+            "Sentience extension failed to inject. Cannot collect raw data for API processing."
+        ) from e
+
+    # Step 1: Get raw data from local extension (including screenshot)
+    raw_options: dict[str, Any] = {}
+    screenshot_requested = False
+    if options.screenshot is not False:
+        screenshot_requested = True
+        # Serialize ScreenshotConfig to dict if it's a Pydantic model
+        if hasattr(options.screenshot, "model_dump"):
+            raw_options["screenshot"] = options.screenshot.model_dump()
+        else:
+            raw_options["screenshot"] = options.screenshot
+
+    raw_result = await browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.snapshot(options);
+        }
+        """,
+        raw_options,
+    )
+
+    # Extract screenshot from raw result (extension captures it, but API doesn't return it)
+    screenshot_data_url = raw_result.get("screenshot")
+    screenshot_format = None
+    if screenshot_data_url:
+        # Extract format from data URL
+        if screenshot_data_url.startswith("data:image/"):
+            format_match = screenshot_data_url.split(";")[0].split("/")[-1]
+            if format_match in ["jpeg", "jpg", "png"]:
+                screenshot_format = "jpeg" if format_match in ["jpeg", "jpg"] else "png"
+
+    # Save trace if requested
+    if options.save_trace:
+        _save_trace_to_file(raw_result.get("raw_elements", []), options.trace_path)
+
+    # Step 2: Send to server for smart ranking/filtering
+    payload = {
+        "raw_elements": raw_result.get("raw_elements", []),
+        "url": raw_result.get("url", ""),
+        "viewport": raw_result.get("viewport"),
+        "goal": options.goal,
+        "options": {
+            "limit": options.limit,
+            "filter": options.filter.model_dump() if options.filter else None,
+        },
+    }
+
+    # Check payload size
+    payload_json = json.dumps(payload)
+    payload_size = len(payload_json.encode("utf-8"))
+    if payload_size > MAX_PAYLOAD_BYTES:
+        raise ValueError(
+            f"Payload size ({payload_size / 1024 / 1024:.2f}MB) exceeds server limit "
+            f"({MAX_PAYLOAD_BYTES / 1024 / 1024:.0f}MB). "
+            f"Try reducing the number of elements on the page or filtering elements."
+        )
+
+    headers = {
+        "Authorization": f"Bearer {browser.api_key}",
+        "Content-Type": "application/json",
+    }
+
+    try:
+        # Lazy import httpx - only needed for async API calls
+        import httpx
+
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.post(
+                f"{browser.api_url}/v1/snapshot",
+                content=payload_json,
+                headers=headers,
+            )
+            response.raise_for_status()
+            api_result = response.json()
+
+        # Extract screenshot format from data URL if not provided
+        if screenshot_data_url and not screenshot_format:
+            if screenshot_data_url.startswith("data:image/"):
+                format_match = screenshot_data_url.split(";")[0].split("/")[-1]
+                if format_match in ["jpeg", "jpg", "png"]:
+                    screenshot_format = "jpeg" if format_match in ["jpeg", "jpg"] else "png"
+
+        # Merge API result with local data
+        snapshot_data = {
+            "status": api_result.get("status", "success"),
+            "timestamp": api_result.get("timestamp"),
+            "url": api_result.get("url", raw_result.get("url", "")),
+            "viewport": api_result.get("viewport", raw_result.get("viewport")),
+            "elements": api_result.get("elements", []),
+            "screenshot": screenshot_data_url,  # Use the extracted screenshot
+            "screenshot_format": screenshot_format,  # Use the extracted format
+            "error": api_result.get("error"),
+        }
+
+        # Show visual overlay if requested
+        if options.show_overlay:
+            elements = api_result.get("elements", [])
+            if elements:
+                await browser.page.evaluate(
+                    """
+                    (elements) => {
+                        if (window.sentience && window.sentience.showOverlay) {
+                            window.sentience.showOverlay(elements, null);
+                        }
+                    }
+                    """,
+                    elements,
+                )
+
+        return Snapshot(**snapshot_data)
+    except ImportError:
+        # Fallback to requests if httpx not available (shouldn't happen in async context)
+        raise RuntimeError(
+            "httpx is required for async API calls. Install it with: pip install httpx"
+        )
+    except Exception as e:
+        raise RuntimeError(f"API request failed: {e}")

sentience/snapshot_diff.py

@@ -0,0 +1,141 @@
+"""
+Snapshot comparison utilities for diff_status detection.
+
+Implements change detection logic for the Diff Overlay feature.
+"""
+
+from typing import Literal
+
+from .models import Element, Snapshot
+
+
+class SnapshotDiff:
+    """
+    Utility for comparing snapshots and computing diff_status for elements.
+
+    Implements the logic described in DIFF_STATUS_GAP_ANALYSIS.md:
+    - ADDED: Element exists in current but not in previous
+    - REMOVED: Element existed in previous but not in current
+    - MODIFIED: Element exists in both but has changed
+    - MOVED: Element exists in both but position changed
+    """
+
+    @staticmethod
+    def _has_bbox_changed(el1: Element, el2: Element, threshold: float = 5.0) -> bool:
+        """
+        Check if element's bounding box has changed significantly.
+
+        Args:
+            el1: First element
+            el2: Second element
+            threshold: Position change threshold in pixels (default: 5.0)
+
+        Returns:
+            True if position or size changed beyond threshold
+        """
+        return (
+            abs(el1.bbox.x - el2.bbox.x) > threshold
+            or abs(el1.bbox.y - el2.bbox.y) > threshold
+            or abs(el1.bbox.width - el2.bbox.width) > threshold
+            or abs(el1.bbox.height - el2.bbox.height) > threshold
+        )
+
+    @staticmethod
+    def _has_content_changed(el1: Element, el2: Element) -> bool:
+        """
+        Check if element's content has changed.
+
+        Args:
+            el1: First element
+            el2: Second element
+
+        Returns:
+            True if text, role, or visual properties changed
+        """
+        # Compare text content
+        if el1.text != el2.text:
+            return True
+
+        # Compare role
+        if el1.role != el2.role:
+            return True
+
+        # Compare visual cues
+        if el1.visual_cues.is_primary != el2.visual_cues.is_primary:
+            return True
+        if el1.visual_cues.is_clickable != el2.visual_cues.is_clickable:
+            return True
+
+        return False
+
+    @staticmethod
+    def compute_diff_status(
+        current: Snapshot,
+        previous: Snapshot | None,
+    ) -> list[Element]:
+        """
+        Compare current snapshot with previous and set diff_status on elements.
+
+        Args:
+            current: Current snapshot
+            previous: Previous snapshot (None if this is the first snapshot)
+
+        Returns:
+            List of elements with diff_status set (includes REMOVED elements from previous)
+        """
+        # If no previous snapshot, all current elements are ADDED
+        if previous is None:
+            result = []
+            for el in current.elements:
+                # Create a copy with diff_status set
+                el_dict = el.model_dump()
+                el_dict["diff_status"] = "ADDED"
+                result.append(Element(**el_dict))
+            return result
+
+        # Build lookup maps by element ID
+        current_by_id = {el.id: el for el in current.elements}
+        previous_by_id = {el.id: el for el in previous.elements}
+
+        current_ids = set(current_by_id.keys())
+        previous_ids = set(previous_by_id.keys())
+
+        result: list[Element] = []
+
+        # Process current elements
+        for el in current.elements:
+            el_dict = el.model_dump()
+
+            if el.id not in previous_ids:
+                # Element is new - mark as ADDED
+                el_dict["diff_status"] = "ADDED"
+            else:
+                # Element existed before - check for changes
+                prev_el = previous_by_id[el.id]
+
+                bbox_changed = SnapshotDiff._has_bbox_changed(el, prev_el)
+                content_changed = SnapshotDiff._has_content_changed(el, prev_el)
+
+                if bbox_changed and content_changed:
+                    # Both position and content changed - mark as MODIFIED
+                    el_dict["diff_status"] = "MODIFIED"
+                elif bbox_changed:
+                    # Only position changed - mark as MOVED
+                    el_dict["diff_status"] = "MOVED"
+                elif content_changed:
+                    # Only content changed - mark as MODIFIED
+                    el_dict["diff_status"] = "MODIFIED"
+                else:
+                    # No change - don't set diff_status (frontend expects undefined)
+                    el_dict["diff_status"] = None
+
+            result.append(Element(**el_dict))
+
+        # Process removed elements (existed in previous but not in current)
+        for prev_id in previous_ids - current_ids:
+            prev_el = previous_by_id[prev_id]
+            el_dict = prev_el.model_dump()
+            el_dict["diff_status"] = "REMOVED"
+            result.append(Element(**el_dict))
+
+        return result

sentience/text_search.py

@@ -2,7 +2,8 @@
 Text search utilities - find text and get pixel coordinates
 """
 
-from .browser import SentienceBrowser
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+from .browser_evaluator import BrowserEvaluator
 from .models import TextRectSearchResult
 
 
@@ -88,18 +89,131 @@ def find_text_rect(
     # Limit max_results to prevent performance issues
     max_results = min(max_results, 100)
 
+    # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
+    # The new architecture loads injected_api.js asynchronously, so window.sentience
+    # may not be immediately available after page load
+    BrowserEvaluator.wait_for_extension(browser.page, timeout_ms=5000)
+
+    # Verify findTextRect method exists (for older extension versions that don't have it)
+    if not BrowserEvaluator.verify_method_exists(browser.page, SentienceMethod.FIND_TEXT_RECT):
+        raise RuntimeError(
+            "window.sentience.findTextRect is not available. "
+            "Please update the Sentience extension to the latest version."
+        )
+
+    # Call the extension's findTextRect method
+    result_dict = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.findTextRect(options);
+        }
+        """,
+        {
+            "text": text,
+            "caseSensitive": case_sensitive,
+            "wholeWord": whole_word,
+            "maxResults": max_results,
+        },
+    )
+
+    # Parse and validate with Pydantic
+    return TextRectSearchResult(**result_dict)
+
+
+async def find_text_rect_async(
+    browser: AsyncSentienceBrowser,
+    text: str,
+    case_sensitive: bool = False,
+    whole_word: bool = False,
+    max_results: int = 10,
+) -> TextRectSearchResult:
+    """
+    Find all occurrences of text on the page and get their exact pixel coordinates (async).
+
+    This function searches for text in all visible text nodes on the page and returns
+    the bounding rectangles for each match. Useful for:
+    - Finding specific UI elements by their text content
+    - Locating buttons, links, or labels without element IDs
+    - Getting exact coordinates for click automation
+    - Highlighting search results visually
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        text: Text to search for (required)
+        case_sensitive: If True, search is case-sensitive (default: False)
+        whole_word: If True, only match whole words surrounded by whitespace (default: False)
+        max_results: Maximum number of matches to return (default: 10, max: 100)
+
+    Returns:
+        TextRectSearchResult with:
+            - status: "success" or "error"
+            - query: The search text
+            - case_sensitive: Whether search was case-sensitive
+            - whole_word: Whether whole-word matching was used
+            - matches: Number of matches found
+            - results: List of TextMatch objects, each containing:
+                - text: The matched text
+                - rect: Absolute rectangle (with scroll offset)
+                - viewport_rect: Viewport-relative rectangle
+                - context: Surrounding text (before/after)
+                - in_viewport: Whether visible in current viewport
+            - viewport: Current viewport dimensions and scroll position
+            - error: Error message if status is "error"
+
+    Examples:
+        # Find "Sign In" button
+        result = await find_text_rect_async(browser, "Sign In")
+        if result.status == "success" and result.results:
+            first_match = result.results[0]
+            print(f"Found at: ({first_match.rect.x}, {first_match.rect.y})")
+            print(f"Size: {first_match.rect.width}x{first_match.rect.height}")
+            print(f"In viewport: {first_match.in_viewport}")
+
+        # Case-sensitive search
+        result = await find_text_rect_async(browser, "LOGIN", case_sensitive=True)
+
+        # Whole word only
+        result = await find_text_rect_async(browser, "log", whole_word=True)  # Won't match "login"
+
+        # Find all matches and click the first visible one
+        result = await find_text_rect_async(browser, "Buy Now", max_results=5)
+        if result.status == "success" and result.results:
+            for match in result.results:
+                if match.in_viewport:
+                    # Use click_rect_async from actions module
+                    from sentience.actions import click_rect_async
+                    click_result = await click_rect_async(browser, {
+                        "x": match.rect.x,
+                        "y": match.rect.y,
+                        "w": match.rect.width,
+                        "h": match.rect.height
+                    })
+                    break
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    if not text or not text.strip():
+        return TextRectSearchResult(
+            status="error",
+            error="Text parameter is required and cannot be empty",
+        )
+
+    # Limit max_results to prevent performance issues
+    max_results = min(max_results, 100)
+
     # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
     # The new architecture loads injected_api.js asynchronously, so window.sentience
     # may not be immediately available after page load
     try:
-        browser.page.wait_for_function(
+        await browser.page.wait_for_function(
             "typeof window.sentience !== 'undefined'",
             timeout=5000,  # 5 second timeout
         )
     except Exception as e:
         # Gather diagnostics if wait fails
         try:
-            diag = browser.page.evaluate(
+            diag = await browser.page.evaluate(
                 """() => ({
                     sentience_defined: typeof window.sentience !== 'undefined',
                     extension_id: document.documentElement.dataset.sentienceExtensionId || 'not set',
@@ -116,7 +230,7 @@ def find_text_rect(
 
     # Verify findTextRect method exists (for older extension versions that don't have it)
     try:
-        has_find_text_rect = browser.page.evaluate(
+        has_find_text_rect = await browser.page.evaluate(
             "typeof window.sentience.findTextRect !== 'undefined'"
         )
         if not has_find_text_rect:
@@ -130,7 +244,7 @@ def find_text_rect(
         raise RuntimeError(f"Failed to verify findTextRect availability: {e}") from e
 
     # Call the extension's findTextRect method
-    result_dict = browser.page.evaluate(
+    result_dict = await browser.page.evaluate(
         """
         (options) => {
             return window.sentience.findTextRect(options);