sentienceapi 0.90.12__py3-none-any.whl → 0.92.2__py3-none-any.whl

sentience/text_search.py

@@ -2,7 +2,8 @@
 Text search utilities - find text and get pixel coordinates
 """
 
-from .browser import SentienceBrowser
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+from .browser_evaluator import BrowserEvaluator
 from .models import TextRectSearchResult
 
 
@@ -88,18 +89,131 @@ def find_text_rect(
     # Limit max_results to prevent performance issues
     max_results = min(max_results, 100)
 
+    # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
+    # The new architecture loads injected_api.js asynchronously, so window.sentience
+    # may not be immediately available after page load
+    BrowserEvaluator.wait_for_extension(browser.page, timeout_ms=5000)
+
+    # Verify findTextRect method exists (for older extension versions that don't have it)
+    if not BrowserEvaluator.verify_method_exists(browser.page, SentienceMethod.FIND_TEXT_RECT):
+        raise RuntimeError(
+            "window.sentience.findTextRect is not available. "
+            "Please update the Sentience extension to the latest version."
+        )
+
+    # Call the extension's findTextRect method
+    result_dict = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.findTextRect(options);
+        }
+        """,
+        {
+            "text": text,
+            "caseSensitive": case_sensitive,
+            "wholeWord": whole_word,
+            "maxResults": max_results,
+        },
+    )
+
+    # Parse and validate with Pydantic
+    return TextRectSearchResult(**result_dict)
+
+
+async def find_text_rect_async(
+    browser: AsyncSentienceBrowser,
+    text: str,
+    case_sensitive: bool = False,
+    whole_word: bool = False,
+    max_results: int = 10,
+) -> TextRectSearchResult:
+    """
+    Find all occurrences of text on the page and get their exact pixel coordinates (async).
+
+    This function searches for text in all visible text nodes on the page and returns
+    the bounding rectangles for each match. Useful for:
+    - Finding specific UI elements by their text content
+    - Locating buttons, links, or labels without element IDs
+    - Getting exact coordinates for click automation
+    - Highlighting search results visually
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        text: Text to search for (required)
+        case_sensitive: If True, search is case-sensitive (default: False)
+        whole_word: If True, only match whole words surrounded by whitespace (default: False)
+        max_results: Maximum number of matches to return (default: 10, max: 100)
+
+    Returns:
+        TextRectSearchResult with:
+            - status: "success" or "error"
+            - query: The search text
+            - case_sensitive: Whether search was case-sensitive
+            - whole_word: Whether whole-word matching was used
+            - matches: Number of matches found
+            - results: List of TextMatch objects, each containing:
+                - text: The matched text
+                - rect: Absolute rectangle (with scroll offset)
+                - viewport_rect: Viewport-relative rectangle
+                - context: Surrounding text (before/after)
+                - in_viewport: Whether visible in current viewport
+            - viewport: Current viewport dimensions and scroll position
+            - error: Error message if status is "error"
+
+    Examples:
+        # Find "Sign In" button
+        result = await find_text_rect_async(browser, "Sign In")
+        if result.status == "success" and result.results:
+            first_match = result.results[0]
+            print(f"Found at: ({first_match.rect.x}, {first_match.rect.y})")
+            print(f"Size: {first_match.rect.width}x{first_match.rect.height}")
+            print(f"In viewport: {first_match.in_viewport}")
+
+        # Case-sensitive search
+        result = await find_text_rect_async(browser, "LOGIN", case_sensitive=True)
+
+        # Whole word only
+        result = await find_text_rect_async(browser, "log", whole_word=True)  # Won't match "login"
+
+        # Find all matches and click the first visible one
+        result = await find_text_rect_async(browser, "Buy Now", max_results=5)
+        if result.status == "success" and result.results:
+            for match in result.results:
+                if match.in_viewport:
+                    # Use click_rect_async from actions module
+                    from sentience.actions import click_rect_async
+                    click_result = await click_rect_async(browser, {
+                        "x": match.rect.x,
+                        "y": match.rect.y,
+                        "w": match.rect.width,
+                        "h": match.rect.height
+                    })
+                    break
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    if not text or not text.strip():
+        return TextRectSearchResult(
+            status="error",
+            error="Text parameter is required and cannot be empty",
+        )
+
+    # Limit max_results to prevent performance issues
+    max_results = min(max_results, 100)
+
     # CRITICAL: Wait for extension injection to complete (CSP-resistant architecture)
     # The new architecture loads injected_api.js asynchronously, so window.sentience
     # may not be immediately available after page load
     try:
-        browser.page.wait_for_function(
+        await browser.page.wait_for_function(
             "typeof window.sentience !== 'undefined'",
             timeout=5000,  # 5 second timeout
         )
     except Exception as e:
         # Gather diagnostics if wait fails
         try:
-            diag = browser.page.evaluate(
+            diag = await browser.page.evaluate(
                 """() => ({
                     sentience_defined: typeof window.sentience !== 'undefined',
                     extension_id: document.documentElement.dataset.sentienceExtensionId || 'not set',
@@ -116,7 +230,7 @@ def find_text_rect(
 
     # Verify findTextRect method exists (for older extension versions that don't have it)
     try:
-        has_find_text_rect = browser.page.evaluate(
+        has_find_text_rect = await browser.page.evaluate(
             "typeof window.sentience.findTextRect !== 'undefined'"
         )
         if not has_find_text_rect:
@@ -130,7 +244,7 @@ def find_text_rect(
         raise RuntimeError(f"Failed to verify findTextRect availability: {e}") from e
 
     # Call the extension's findTextRect method
-    result_dict = browser.page.evaluate(
+    result_dict = await browser.page.evaluate(
         """
         (options) => {
             return window.sentience.findTextRect(options);

sentience/trace_event_builder.py

@@ -0,0 +1,129 @@
+"""
+Trace event building utilities for agent-based tracing.
+
+This module provides centralized trace event building logic to reduce duplication
+across agent implementations.
+"""
+
+from typing import Any, Optional
+
+from .models import AgentActionResult, Element, Snapshot
+
+
+class TraceEventBuilder:
+    """
+    Helper for building trace events with consistent structure.
+
+    Provides static methods for building common trace event types:
+    - snapshot_taken events
+    - step_end events
+    """
+
+    @staticmethod
+    def build_snapshot_event(
+        snapshot: Snapshot,
+        include_all_elements: bool = True,
+    ) -> dict[str, Any]:
+        """
+        Build snapshot_taken trace event data.
+
+        Args:
+            snapshot: Snapshot to build event from
+            include_all_elements: If True, include all elements (for DOM tree display).
+                                 If False, use filtered elements only.
+
+        Returns:
+            Dictionary with snapshot event data
+        """
+        # Normalize importance values to importance_score (0-1 range) per snapshot
+        # Min-max normalization: (value - min) / (max - min)
+        importance_values = [el.importance for el in snapshot.elements]
+
+        if importance_values:
+            min_importance = min(importance_values)
+            max_importance = max(importance_values)
+            importance_range = max_importance - min_importance
+        else:
+            min_importance = 0
+            max_importance = 0
+            importance_range = 0
+
+        # Include ALL elements with full data for DOM tree display
+        # Add importance_score field normalized to [0, 1]
+        elements_data = []
+        for el in snapshot.elements:
+            el_dict = el.model_dump()
+
+            # Compute normalized importance_score
+            if importance_range > 0:
+                importance_score = (el.importance - min_importance) / importance_range
+            else:
+                # If all elements have same importance, set to 0.5
+                importance_score = 0.5
+
+            el_dict["importance_score"] = importance_score
+            elements_data.append(el_dict)
+
+        return {
+            "url": snapshot.url,
+            "element_count": len(snapshot.elements),
+            "timestamp": snapshot.timestamp,
+            "elements": elements_data,  # Full element data for DOM tree
+        }
+
+    @staticmethod
+    def build_step_end_event(
+        step_id: str,
+        step_index: int,
+        goal: str,
+        attempt: int,
+        pre_url: str,
+        post_url: str,
+        snapshot_digest: str | None,
+        llm_data: dict[str, Any],
+        exec_data: dict[str, Any],
+        verify_data: dict[str, Any],
+        pre_elements: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Build step_end trace event data.
+
+        Args:
+            step_id: Unique step identifier
+            step_index: Step index (0-based)
+            goal: User's goal for this step
+            attempt: Attempt number (0-based)
+            pre_url: URL before action execution
+            post_url: URL after action execution
+            snapshot_digest: Digest of snapshot before action
+            llm_data: LLM interaction data
+            exec_data: Action execution data
+            verify_data: Verification data
+            pre_elements: Optional list of elements from pre-snapshot (with diff_status)
+
+        Returns:
+            Dictionary with step_end event data
+        """
+        pre_data: dict[str, Any] = {
+            "url": pre_url,
+            "snapshot_digest": snapshot_digest,
+        }
+
+        # Add elements to pre field if provided (for diff overlay support)
+        if pre_elements is not None:
+            pre_data["elements"] = pre_elements
+
+        return {
+            "v": 1,
+            "step_id": step_id,
+            "step_index": step_index,
+            "goal": goal,
+            "attempt": attempt,
+            "pre": pre_data,
+            "llm": llm_data,
+            "exec": exec_data,
+            "post": {
+                "url": post_url,
+            },
+            "verify": verify_data,
+        }

sentience/trace_file_manager.py

@@ -0,0 +1,197 @@
+"""
+Trace file management utilities for consistent file operations.
+
+This module provides helper functions for common trace file operations
+shared between JsonlTraceSink and CloudTraceSink.
+"""
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Optional
+
+from .models import TraceStats
+
+
+class TraceFileManager:
+    """
+    Helper for common trace file operations.
+
+    Provides static methods for file operations shared across trace sinks.
+    """
+
+    @staticmethod
+    def write_event(file_handle: Any, event: dict[str, Any]) -> None:
+        """
+        Write a trace event to a file handle as JSONL.
+
+        Args:
+            file_handle: Open file handle (must be writable)
+            event: Event dictionary to write
+        """
+        json_str = json.dumps(event, ensure_ascii=False)
+        file_handle.write(json_str + "\n")
+        file_handle.flush()  # Ensure written to disk
+
+    @staticmethod
+    def ensure_directory(path: Path) -> None:
+        """
+        Ensure the parent directory of a path exists.
+
+        Args:
+            path: File path whose parent directory should exist
+        """
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+    @staticmethod
+    def read_events(path: Path) -> list[dict[str, Any]]:
+        """
+        Read all events from a JSONL trace file.
+
+        Args:
+            path: Path to JSONL trace file
+
+        Returns:
+            List of event dictionaries
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            json.JSONDecodeError: If file contains invalid JSON
+        """
+        events = []
+        with open(path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    event = json.loads(line)
+                    events.append(event)
+                except json.JSONDecodeError:
+                    # Skip invalid lines but continue reading
+                    continue
+        return events
+
+    @staticmethod
+    def extract_stats(
+        events: list[dict[str, Any]],
+        infer_status_func: None | (
+            Callable[[list[dict[str, Any]], dict[str, Any] | None], str]
+        ) = None,
+    ) -> TraceStats:
+        """
+        Extract execution statistics from trace events.
+
+        This is a common operation shared between JsonlTraceSink and CloudTraceSink.
+
+        Args:
+            events: List of trace event dictionaries
+            infer_status_func: Optional function to infer final_status from events.
+                             If None, uses default inference logic.
+
+        Returns:
+            TraceStats with execution statistics
+        """
+        if not events:
+            return TraceStats(
+                total_steps=0,
+                total_events=0,
+                duration_ms=None,
+                final_status="unknown",
+                started_at=None,
+                ended_at=None,
+            )
+
+        # Find run_start and run_end events
+        run_start = next((e for e in events if e.get("type") == "run_start"), None)
+        run_end = next((e for e in events if e.get("type") == "run_end"), None)
+
+        # Extract timestamps
+        started_at: str | None = None
+        ended_at: str | None = None
+        if run_start:
+            started_at = run_start.get("ts")
+        if run_end:
+            ended_at = run_end.get("ts")
+
+        # Calculate duration
+        duration_ms: int | None = None
+        if started_at and ended_at:
+            try:
+                from datetime import datetime
+
+                start_dt = datetime.fromisoformat(started_at.replace("Z", "+00:00"))
+                end_dt = datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
+                delta = end_dt - start_dt
+                duration_ms = int(delta.total_seconds() * 1000)
+            except Exception:
+                pass
+
+        # Count steps (from step_start events, only first attempt)
+        step_indices = set()
+        for event in events:
+            if event.get("type") == "step_start":
+                step_index = event.get("data", {}).get("step_index")
+                if step_index is not None:
+                    step_indices.add(step_index)
+        total_steps = len(step_indices) if step_indices else 0
+
+        # If run_end has steps count, use that (more accurate)
+        if run_end:
+            steps_from_end = run_end.get("data", {}).get("steps")
+            if steps_from_end is not None:
+                total_steps = max(total_steps, steps_from_end)
+
+        # Count total events
+        total_events = len(events)
+
+        # Infer final status
+        if infer_status_func:
+            final_status = infer_status_func(events, run_end)
+        else:
+            final_status = TraceFileManager._infer_final_status(events, run_end)
+
+        return TraceStats(
+            total_steps=total_steps,
+            total_events=total_events,
+            duration_ms=duration_ms,
+            final_status=final_status,
+            started_at=started_at,
+            ended_at=ended_at,
+        )
+
+    @staticmethod
+    def _infer_final_status(
+        events: list[dict[str, Any]],
+        run_end: dict[str, Any] | None,
+    ) -> str:
+        """
+        Infer final status from trace events.
+
+        Args:
+            events: List of trace event dictionaries
+            run_end: Optional run_end event dictionary
+
+        Returns:
+            Final status string: "success", "failure", "partial", or "unknown"
+        """
+        # Check for run_end event with status
+        if run_end:
+            status = run_end.get("data", {}).get("status")
+            if status in ("success", "failure", "partial", "unknown"):
+                return status
+
+        # Infer from error events
+        has_errors = any(e.get("type") == "error" for e in events)
+        if has_errors:
+            step_ends = [e for e in events if e.get("type") == "step_end"]
+            if step_ends:
+                return "partial"
+            else:
+                return "failure"
+        else:
+            step_ends = [e for e in events if e.get("type") == "step_end"]
+            if step_ends:
+                return "success"
+            else:
+                return "unknown"

sentience/trace_indexing/index_schema.py

@@ -13,6 +13,7 @@ class TraceFileInfo:
     path: str
     size_bytes: int
     sha256: str
+    line_count: int | None = None  # Number of lines in the trace file
 
     def to_dict(self) -> dict:
         return asdict(self)
@@ -28,6 +29,12 @@ class TraceSummary:
     step_count: int
     error_count: int
     final_url: str | None
+    status: Literal["success", "failure", "partial", "unknown"] | None = None
+    agent_name: str | None = None  # Agent name from run_start event
+    duration_ms: int | None = None  # Calculated duration in milliseconds
+    counters: dict[str, int] | None = (
+        None  # Aggregated counters (snapshot_count, action_count, error_count)
+    )
 
     def to_dict(self) -> dict:
         return asdict(self)
@@ -78,17 +85,18 @@ class StepIndex:
     step_index: int
     step_id: str
     goal: str | None
-    status: Literal["ok", "error", "partial"]
+    status: Literal["success", "failure", "partial", "unknown"]
     ts_start: str
     ts_end: str
     offset_start: int
     offset_end: int
-    url_before: str | None
-    url_after: str | None
-    snapshot_before: SnapshotInfo
-    snapshot_after: SnapshotInfo
-    action: ActionInfo
-    counters: StepCounters
+    line_number: int | None = None  # Line number for byte-range fetching
+    url_before: str | None = None
+    url_after: str | None = None
+    snapshot_before: SnapshotInfo = field(default_factory=SnapshotInfo)
+    snapshot_after: SnapshotInfo = field(default_factory=SnapshotInfo)
+    action: ActionInfo = field(default_factory=ActionInfo)
+    counters: StepCounters = field(default_factory=StepCounters)
 
     def to_dict(self) -> dict:
         result = asdict(self)
@@ -109,3 +117,83 @@ class TraceIndex:
     def to_dict(self) -> dict:
         """Convert to dictionary for JSON serialization."""
         return asdict(self)
+
+    def to_sentience_studio_dict(self) -> dict:
+        """
+        Convert to SS-compatible format.
+
+        Maps SDK field names to frontend expectations:
+        - created_at -> generated_at
+        - first_ts -> start_time
+        - last_ts -> end_time
+        - step_index (0-based) -> step (1-based)
+        - ts_start -> timestamp
+        - Filters out "unknown" status
+        """
+        from datetime import datetime
+
+        # Calculate duration if not already set
+        duration_ms = self.summary.duration_ms
+        if duration_ms is None and self.summary.first_ts and self.summary.last_ts:
+            try:
+                start = datetime.fromisoformat(self.summary.first_ts.replace("Z", "+00:00"))
+                end = datetime.fromisoformat(self.summary.last_ts.replace("Z", "+00:00"))
+                duration_ms = int((end - start).total_seconds() * 1000)
+            except (ValueError, AttributeError):
+                duration_ms = None
+
+        # Aggregate counters if not already set
+        counters = self.summary.counters
+        if counters is None:
+            snapshot_count = sum(step.counters.snapshots for step in self.steps)
+            action_count = sum(step.counters.actions for step in self.steps)
+            counters = {
+                "snapshot_count": snapshot_count,
+                "action_count": action_count,
+                "error_count": self.summary.error_count,
+            }
+
+        return {
+            "version": self.version,
+            "run_id": self.run_id,
+            "generated_at": self.created_at,  # Renamed from created_at
+            "trace_file": {
+                "path": self.trace_file.path,
+                "size_bytes": self.trace_file.size_bytes,
+                "line_count": self.trace_file.line_count,  # Added
+            },
+            "summary": {
+                "agent_name": self.summary.agent_name,  # Added
+                "total_steps": self.summary.step_count,  # Renamed from step_count
+                "status": (
+                    self.summary.status if self.summary.status != "unknown" else None
+                ),  # Filter out unknown
+                "start_time": self.summary.first_ts,  # Renamed from first_ts
+                "end_time": self.summary.last_ts,  # Renamed from last_ts
+                "duration_ms": duration_ms,  # Added
+                "counters": counters,  # Added
+            },
+            "steps": [
+                {
+                    "step": s.step_index + 1,  # Convert 0-based to 1-based
+                    "byte_offset": s.offset_start,
+                    "line_number": s.line_number,  # Added
+                    "timestamp": s.ts_start,  # Use start time
+                    "action": {
+                        "type": s.action.type or "",
+                        "goal": s.goal,  # Move goal into action
+                        "digest": s.action.args_digest,
+                    },
+                    "snapshot": (
+                        {
+                            "url": s.snapshot_after.url,
+                            "digest": s.snapshot_after.digest,
+                        }
+                        if s.snapshot_after.url
+                        else None
+                    ),
+                    "status": s.status if s.status != "unknown" else None,  # Filter out unknown
+                }
+                for s in self.steps
+            ],
+        }