sentienceapi 0.90.16__py3-none-any.whl → 0.98.0__py3-none-any.whl

sentience/snapshot_diff.py

@@ -0,0 +1,126 @@
+"""
+Snapshot comparison utilities for diff_status detection.
+
+Implements change detection logic for the Diff Overlay feature.
+
+Uses shared canonicalization helpers from canonicalization.py to ensure
+consistent comparison behavior with trace_indexing/indexer.py.
+"""
+
+from .canonicalization import bbox_changed, content_changed
+from .models import Element, Snapshot
+
+
+class SnapshotDiff:
+    """
+    Utility for comparing snapshots and computing diff_status for elements.
+
+    Implements the logic described in DIFF_STATUS_GAP_ANALYSIS.md:
+    - ADDED: Element exists in current but not in previous
+    - REMOVED: Element existed in previous but not in current
+    - MODIFIED: Element exists in both but has changed
+    - MOVED: Element exists in both but position changed
+
+    Uses canonicalized comparisons (normalized text, rounded bbox) to reduce
+    noise from insignificant changes like sub-pixel rendering differences
+    or whitespace variations.
+    """
+
+    @staticmethod
+    def _element_to_dict(el: Element) -> dict:
+        """Convert Element model to dict for canonicalization helpers."""
+        return {
+            "id": el.id,
+            "role": el.role,
+            "text": el.text,
+            "bbox": {
+                "x": el.bbox.x,
+                "y": el.bbox.y,
+                "width": el.bbox.width,
+                "height": el.bbox.height,
+            },
+            "visual_cues": {
+                "is_primary": el.visual_cues.is_primary,
+                "is_clickable": el.visual_cues.is_clickable,
+            },
+        }
+
+    @staticmethod
+    def compute_diff_status(
+        current: Snapshot,
+        previous: Snapshot | None,
+    ) -> list[Element]:
+        """
+        Compare current snapshot with previous and set diff_status on elements.
+
+        Uses canonicalized comparisons:
+        - Text is normalized (trimmed, collapsed whitespace, lowercased)
+        - Bbox is rounded to 2px grid to ignore sub-pixel differences
+
+        Args:
+            current: Current snapshot
+            previous: Previous snapshot (None if this is the first snapshot)
+
+        Returns:
+            List of elements with diff_status set (includes REMOVED elements from previous)
+        """
+        # If no previous snapshot, all current elements are ADDED
+        if previous is None:
+            result = []
+            for el in current.elements:
+                # Create a copy with diff_status set
+                el_dict = el.model_dump()
+                el_dict["diff_status"] = "ADDED"
+                result.append(Element(**el_dict))
+            return result
+
+        # Build lookup maps by element ID
+        current_by_id = {el.id: el for el in current.elements}
+        previous_by_id = {el.id: el for el in previous.elements}
+
+        current_ids = set(current_by_id.keys())
+        previous_ids = set(previous_by_id.keys())
+
+        result: list[Element] = []
+
+        # Process current elements
+        for el in current.elements:
+            el_dict = el.model_dump()
+
+            if el.id not in previous_ids:
+                # Element is new - mark as ADDED
+                el_dict["diff_status"] = "ADDED"
+            else:
+                # Element existed before - check for changes using canonicalized comparisons
+                prev_el = previous_by_id[el.id]
+
+                # Convert to dicts for canonicalization helpers
+                el_data = SnapshotDiff._element_to_dict(el)
+                prev_el_data = SnapshotDiff._element_to_dict(prev_el)
+
+                has_bbox_changed = bbox_changed(el_data["bbox"], prev_el_data["bbox"])
+                has_content_changed = content_changed(el_data, prev_el_data)
+
+                if has_bbox_changed and has_content_changed:
+                    # Both position and content changed - mark as MODIFIED
+                    el_dict["diff_status"] = "MODIFIED"
+                elif has_bbox_changed:
+                    # Only position changed - mark as MOVED
+                    el_dict["diff_status"] = "MOVED"
+                elif has_content_changed:
+                    # Only content changed - mark as MODIFIED
+                    el_dict["diff_status"] = "MODIFIED"
+                else:
+                    # No change - don't set diff_status (frontend expects undefined)
+                    el_dict["diff_status"] = None
+
+            result.append(Element(**el_dict))
+
+        # Process removed elements (existed in previous but not in current)
+        for prev_id in previous_ids - current_ids:
+            prev_el = previous_by_id[prev_id]
+            el_dict = prev_el.model_dump()
+            el_dict["diff_status"] = "REMOVED"
+            result.append(Element(**el_dict))
+
+        return result

sentience/text_search.py

@@ -2,8 +2,10 @@
 Text search utilities - find text and get pixel coordinates
 """
 
-from .browser import SentienceBrowser
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+from .browser_evaluator import BrowserEvaluator
 from .models import TextRectSearchResult
+from .sentience_methods import SentienceMethod
 
 
 def find_text_rect(
@@ -88,18 +90,131 @@ def find_text_rect(
     # Limit max_results to prevent performance issues
     max_results = min(max_results, 100)
 
+    # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
+    # The new architecture loads injected_api.js asynchronously, so window.sentience
+    # may not be immediately available after page load
+    BrowserEvaluator.wait_for_extension(browser.page, timeout_ms=5000)
+
+    # Verify findTextRect method exists (for older extension versions that don't have it)
+    if not BrowserEvaluator.verify_method_exists(browser.page, SentienceMethod.FIND_TEXT_RECT):
+        raise RuntimeError(
+            "window.sentience.findTextRect is not available. "
+            "Please update the Sentience extension to the latest version."
+        )
+
+    # Call the extension's findTextRect method
+    result_dict = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.findTextRect(options);
+        }
+        """,
+        {
+            "text": text,
+            "caseSensitive": case_sensitive,
+            "wholeWord": whole_word,
+            "maxResults": max_results,
+        },
+    )
+
+    # Parse and validate with Pydantic
+    return TextRectSearchResult(**result_dict)
+
+
+async def find_text_rect_async(
+    browser: AsyncSentienceBrowser,
+    text: str,
+    case_sensitive: bool = False,
+    whole_word: bool = False,
+    max_results: int = 10,
+) -> TextRectSearchResult:
+    """
+    Find all occurrences of text on the page and get their exact pixel coordinates (async).
+
+    This function searches for text in all visible text nodes on the page and returns
+    the bounding rectangles for each match. Useful for:
+    - Finding specific UI elements by their text content
+    - Locating buttons, links, or labels without element IDs
+    - Getting exact coordinates for click automation
+    - Highlighting search results visually
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        text: Text to search for (required)
+        case_sensitive: If True, search is case-sensitive (default: False)
+        whole_word: If True, only match whole words surrounded by whitespace (default: False)
+        max_results: Maximum number of matches to return (default: 10, max: 100)
+
+    Returns:
+        TextRectSearchResult with:
+            - status: "success" or "error"
+            - query: The search text
+            - case_sensitive: Whether search was case-sensitive
+            - whole_word: Whether whole-word matching was used
+            - matches: Number of matches found
+            - results: List of TextMatch objects, each containing:
+                - text: The matched text
+                - rect: Absolute rectangle (with scroll offset)
+                - viewport_rect: Viewport-relative rectangle
+                - context: Surrounding text (before/after)
+                - in_viewport: Whether visible in current viewport
+            - viewport: Current viewport dimensions and scroll position
+            - error: Error message if status is "error"
+
+    Examples:
+        # Find "Sign In" button
+        result = await find_text_rect_async(browser, "Sign In")
+        if result.status == "success" and result.results:
+            first_match = result.results[0]
+            print(f"Found at: ({first_match.rect.x}, {first_match.rect.y})")
+            print(f"Size: {first_match.rect.width}x{first_match.rect.height}")
+            print(f"In viewport: {first_match.in_viewport}")
+
+        # Case-sensitive search
+        result = await find_text_rect_async(browser, "LOGIN", case_sensitive=True)
+
+        # Whole word only
+        result = await find_text_rect_async(browser, "log", whole_word=True)  # Won't match "login"
+
+        # Find all matches and click the first visible one
+        result = await find_text_rect_async(browser, "Buy Now", max_results=5)
+        if result.status == "success" and result.results:
+            for match in result.results:
+                if match.in_viewport:
+                    # Use click_rect_async from actions module
+                    from sentience.actions import click_rect_async
+                    click_result = await click_rect_async(browser, {
+                        "x": match.rect.x,
+                        "y": match.rect.y,
+                        "w": match.rect.width,
+                        "h": match.rect.height
+                    })
+                    break
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    if not text or not text.strip():
+        return TextRectSearchResult(
+            status="error",
+            error="Text parameter is required and cannot be empty",
+        )
+
+    # Limit max_results to prevent performance issues
+    max_results = min(max_results, 100)
+
     # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
     # The new architecture loads injected_api.js asynchronously, so window.sentience
     # may not be immediately available after page load
     try:
-        browser.page.wait_for_function(
+        await browser.page.wait_for_function(
             "typeof window.sentience !== 'undefined'",
             timeout=5000,  # 5 second timeout
         )
     except Exception as e:
         # Gather diagnostics if wait fails
         try:
-            diag = browser.page.evaluate(
+            diag = await browser.page.evaluate(
                 """() => ({
                     sentience_defined: typeof window.sentience !== 'undefined',
                     extension_id: document.documentElement.dataset.sentienceExtensionId || 'not set',
@@ -116,7 +231,7 @@ def find_text_rect(
 
     # Verify findTextRect method exists (for older extension versions that don't have it)
     try:
-        has_find_text_rect = browser.page.evaluate(
+        has_find_text_rect = await browser.page.evaluate(
             "typeof window.sentience.findTextRect !== 'undefined'"
         )
         if not has_find_text_rect:
@@ -130,7 +245,7 @@ def find_text_rect(
         raise RuntimeError(f"Failed to verify findTextRect availability: {e}") from e
 
     # Call the extension's findTextRect method
-    result_dict = browser.page.evaluate(
+    result_dict = await browser.page.evaluate(
         """
         (options) => {
             return window.sentience.findTextRect(options);

sentience/trace_event_builder.py

@@ -0,0 +1,148 @@
+"""
+Trace event building utilities for agent-based tracing.
+
+This module provides centralized trace event building logic to reduce duplication
+across agent implementations.
+"""
+
+from typing import Any, Optional
+
+from .models import AgentActionResult, Element, Snapshot
+
+
+class TraceEventBuilder:
+    """
+    Helper for building trace events with consistent structure.
+
+    Provides static methods for building common trace event types:
+    - snapshot_taken events
+    - step_end events
+    """
+
+    @staticmethod
+    def build_snapshot_event(
+        snapshot: Snapshot,
+        include_all_elements: bool = True,
+    ) -> dict[str, Any]:
+        """
+        Build snapshot_taken trace event data.
+
+        Args:
+            snapshot: Snapshot to build event from
+            include_all_elements: If True, include all elements (for DOM tree display).
+                                 If False, use filtered elements only.
+
+        Returns:
+            Dictionary with snapshot event data
+        """
+        # Normalize importance values to importance_score (0-1 range) per snapshot
+        # Min-max normalization: (value - min) / (max - min)
+        importance_values = [el.importance for el in snapshot.elements]
+
+        if importance_values:
+            min_importance = min(importance_values)
+            max_importance = max(importance_values)
+            importance_range = max_importance - min_importance
+        else:
+            min_importance = 0
+            max_importance = 0
+            importance_range = 0
+
+        # Include ALL elements with full data for DOM tree display
+        # Add importance_score field normalized to [0, 1]
+        elements_data = []
+        for el in snapshot.elements:
+            el_dict = el.model_dump()
+
+            # Compute normalized importance_score
+            if importance_range > 0:
+                importance_score = (el.importance - min_importance) / importance_range
+            else:
+                # If all elements have same importance, set to 0.5
+                importance_score = 0.5
+
+            el_dict["importance_score"] = importance_score
+            elements_data.append(el_dict)
+
+        return {
+            "url": snapshot.url,
+            "element_count": len(snapshot.elements),
+            "timestamp": snapshot.timestamp,
+            "elements": elements_data,  # Full element data for DOM tree
+        }
+
+    @staticmethod
+    def build_step_end_event(
+        step_id: str,
+        step_index: int,
+        goal: str,
+        attempt: int,
+        pre_url: str,
+        post_url: str,
+        snapshot_digest: str | None,
+        llm_data: dict[str, Any],
+        exec_data: dict[str, Any],
+        verify_data: dict[str, Any],
+        pre_elements: list[dict[str, Any]] | None = None,
+        assertions: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Build step_end trace event data.
+
+        Args:
+            step_id: Unique step identifier
+            step_index: Step index (0-based)
+            goal: User's goal for this step
+            attempt: Attempt number (0-based)
+            pre_url: URL before action execution
+            post_url: URL after action execution
+            snapshot_digest: Digest of snapshot before action
+            llm_data: LLM interaction data
+            exec_data: Action execution data
+            verify_data: Verification data
+            pre_elements: Optional list of elements from pre-snapshot (with diff_status)
+            assertions: Optional list of assertion results from AgentRuntime
+
+        Returns:
+            Dictionary with step_end event data
+        """
+        pre_data: dict[str, Any] = {
+            "url": pre_url,
+            "snapshot_digest": snapshot_digest,
+        }
+
+        # Add elements to pre field if provided (for diff overlay support)
+        if pre_elements is not None:
+            pre_data["elements"] = pre_elements
+
+        # Build verify data with assertions if provided
+        final_verify_data = verify_data.copy()
+        if assertions:
+            # Ensure signals dict exists
+            if "signals" not in final_verify_data:
+                final_verify_data["signals"] = {}
+
+            # Add assertions to signals
+            final_verify_data["signals"]["assertions"] = assertions
+
+            # Check for task completion (assertions marked as required that passed)
+            for a in assertions:
+                if a.get("passed") and a.get("required"):
+                    final_verify_data["signals"]["task_done"] = True
+                    final_verify_data["signals"]["task_done_label"] = a.get("label")
+                    break
+
+        return {
+            "v": 1,
+            "step_id": step_id,
+            "step_index": step_index,
+            "goal": goal,
+            "attempt": attempt,
+            "pre": pre_data,
+            "llm": llm_data,
+            "exec": exec_data,
+            "post": {
+                "url": post_url,
+            },
+            "verify": final_verify_data,
+        }

sentience/trace_file_manager.py

@@ -0,0 +1,197 @@
+"""
+Trace file management utilities for consistent file operations.
+
+This module provides helper functions for common trace file operations
+shared between JsonlTraceSink and CloudTraceSink.
+"""
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Optional
+
+from .models import TraceStats
+
+
+class TraceFileManager:
+    """
+    Helper for common trace file operations.
+
+    Provides static methods for file operations shared across trace sinks.
+    """
+
+    @staticmethod
+    def write_event(file_handle: Any, event: dict[str, Any]) -> None:
+        """
+        Write a trace event to a file handle as JSONL.
+
+        Args:
+            file_handle: Open file handle (must be writable)
+            event: Event dictionary to write
+        """
+        json_str = json.dumps(event, ensure_ascii=False)
+        file_handle.write(json_str + "\n")
+        file_handle.flush()  # Ensure written to disk
+
+    @staticmethod
+    def ensure_directory(path: Path) -> None:
+        """
+        Ensure the parent directory of a path exists.
+
+        Args:
+            path: File path whose parent directory should exist
+        """
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+    @staticmethod
+    def read_events(path: Path) -> list[dict[str, Any]]:
+        """
+        Read all events from a JSONL trace file.
+
+        Args:
+            path: Path to JSONL trace file
+
+        Returns:
+            List of event dictionaries
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            json.JSONDecodeError: If file contains invalid JSON
+        """
+        events = []
+        with open(path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    event = json.loads(line)
+                    events.append(event)
+                except json.JSONDecodeError:
+                    # Skip invalid lines but continue reading
+                    continue
+        return events
+
+    @staticmethod
+    def extract_stats(
+        events: list[dict[str, Any]],
+        infer_status_func: None | (
+            Callable[[list[dict[str, Any]], dict[str, Any] | None], str]
+        ) = None,
+    ) -> TraceStats:
+        """
+        Extract execution statistics from trace events.
+
+        This is a common operation shared between JsonlTraceSink and CloudTraceSink.
+
+        Args:
+            events: List of trace event dictionaries
+            infer_status_func: Optional function to infer final_status from events.
+                             If None, uses default inference logic.
+
+        Returns:
+            TraceStats with execution statistics
+        """
+        if not events:
+            return TraceStats(
+                total_steps=0,
+                total_events=0,
+                duration_ms=None,
+                final_status="unknown",
+                started_at=None,
+                ended_at=None,
+            )
+
+        # Find run_start and run_end events
+        run_start = next((e for e in events if e.get("type") == "run_start"), None)
+        run_end = next((e for e in events if e.get("type") == "run_end"), None)
+
+        # Extract timestamps
+        started_at: str | None = None
+        ended_at: str | None = None
+        if run_start:
+            started_at = run_start.get("ts")
+        if run_end:
+            ended_at = run_end.get("ts")
+
+        # Calculate duration
+        duration_ms: int | None = None
+        if started_at and ended_at:
+            try:
+                from datetime import datetime
+
+                start_dt = datetime.fromisoformat(started_at.replace("Z", "+00:00"))
+                end_dt = datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
+                delta = end_dt - start_dt
+                duration_ms = int(delta.total_seconds() * 1000)
+            except Exception:
+                pass
+
+        # Count steps (from step_start events, only first attempt)
+        step_indices = set()
+        for event in events:
+            if event.get("type") == "step_start":
+                step_index = event.get("data", {}).get("step_index")
+                if step_index is not None:
+                    step_indices.add(step_index)
+        total_steps = len(step_indices) if step_indices else 0
+
+        # If run_end has steps count, use that (more accurate)
+        if run_end:
+            steps_from_end = run_end.get("data", {}).get("steps")
+            if steps_from_end is not None:
+                total_steps = max(total_steps, steps_from_end)
+
+        # Count total events
+        total_events = len(events)
+
+        # Infer final status
+        if infer_status_func:
+            final_status = infer_status_func(events, run_end)
+        else:
+            final_status = TraceFileManager._infer_final_status(events, run_end)
+
+        return TraceStats(
+            total_steps=total_steps,
+            total_events=total_events,
+            duration_ms=duration_ms,
+            final_status=final_status,
+            started_at=started_at,
+            ended_at=ended_at,
+        )
+
+    @staticmethod
+    def _infer_final_status(
+        events: list[dict[str, Any]],
+        run_end: dict[str, Any] | None,
+    ) -> str:
+        """
+        Infer final status from trace events.
+
+        Args:
+            events: List of trace event dictionaries
+            run_end: Optional run_end event dictionary
+
+        Returns:
+            Final status string: "success", "failure", "partial", or "unknown"
+        """
+        # Check for run_end event with status
+        if run_end:
+            status = run_end.get("data", {}).get("status")
+            if status in ("success", "failure", "partial", "unknown"):
+                return status
+
+        # Infer from error events
+        has_errors = any(e.get("type") == "error" for e in events)
+        if has_errors:
+            step_ends = [e for e in events if e.get("type") == "step_end"]
+            if step_ends:
+                return "partial"
+            else:
+                return "failure"
+        else:
+            step_ends = [e for e in events if e.get("type") == "step_end"]
+            if step_ends:
+                return "success"
+            else:
+                return "unknown"