sentienceapi 0.90.16__py3-none-any.whl → 0.98.0__py3-none-any.whl

sentience/query.py

@@ -3,7 +3,7 @@ Query engine v1 - semantic selector matching
 """
 
 import re
-from typing import Any
+from typing import Any, Optional
 
 from .models import Element, Snapshot
 
@@ -52,16 +52,28 @@ def parse_selector(selector: str) -> dict[str, Any]:  # noqa: C901
                 query["visible"] = False
         elif op == "~":
             # Substring match (case-insensitive)
-            if key == "text" or key == "name":
+            if key == "text":
                 query["text_contains"] = value
+            elif key == "name":
+                query["name_contains"] = value
+            elif key == "value":
+                query["value_contains"] = value
         elif op == "^=":
             # Prefix match
-            if key == "text" or key == "name":
+            if key == "text":
                 query["text_prefix"] = value
+            elif key == "name":
+                query["name_prefix"] = value
+            elif key == "value":
+                query["value_prefix"] = value
         elif op == "$=":
             # Suffix match
-            if key == "text" or key == "name":
+            if key == "text":
                 query["text_suffix"] = value
+            elif key == "name":
+                query["name_suffix"] = value
+            elif key == "value":
+                query["value_suffix"] = value
         elif op == ">":
             # Greater than
             if is_numeric:
@@ -116,8 +128,14 @@ def parse_selector(selector: str) -> dict[str, Any]:  # noqa: C901
                 query["visible"] = value.lower() == "true"
             elif key == "tag":
                 query["tag"] = value
-            elif key == "name" or key == "text":
+            elif key == "text":
                 query["text"] = value
+            elif key == "name":
+                query["name"] = value
+            elif key == "value":
+                query["value"] = value
+            elif key in ("checked", "disabled", "expanded"):
+                query[key] = value.lower() == "true"
             elif key == "importance" and is_numeric:
                 query["importance"] = numeric_value
             elif key.startswith("attr."):
@@ -192,6 +210,50 @@ def match_element(element: Element, query: dict[str, Any]) -> bool:  # noqa: C90
         if not element.text.lower().endswith(query["text_suffix"].lower()):
             return False
 
+    # Name matching (best-effort; fallback to text for backward compatibility)
+    name_val = element.name or element.text or ""
+    if "name" in query:
+        if not name_val or name_val != query["name"]:
+            return False
+    if "name_contains" in query:
+        if not name_val or query["name_contains"].lower() not in name_val.lower():
+            return False
+    if "name_prefix" in query:
+        if not name_val or not name_val.lower().startswith(query["name_prefix"].lower()):
+            return False
+    if "name_suffix" in query:
+        if not name_val or not name_val.lower().endswith(query["name_suffix"].lower()):
+            return False
+
+    # Value matching (inputs/textarea/select)
+    if "value" in query:
+        if element.value is None or element.value != query["value"]:
+            return False
+    if "value_contains" in query:
+        if element.value is None or query["value_contains"].lower() not in element.value.lower():
+            return False
+    if "value_prefix" in query:
+        if element.value is None or not element.value.lower().startswith(
+            query["value_prefix"].lower()
+        ):
+            return False
+    if "value_suffix" in query:
+        if element.value is None or not element.value.lower().endswith(
+            query["value_suffix"].lower()
+        ):
+            return False
+
+    # State matching (best-effort)
+    if "checked" in query:
+        if (element.checked is True) != query["checked"]:
+            return False
+    if "disabled" in query:
+        if (element.disabled is True) != query["disabled"]:
+            return False
+    if "expanded" in query:
+        if (element.expanded is True) != query["expanded"]:
+            return False
+
     # Importance filtering
     if "importance" in query:
         if element.importance != query["importance"]:

sentience/read.py

@@ -4,14 +4,15 @@ Read page content - supports raw HTML, text, and markdown formats
 
 from typing import Literal
 
-from .browser import SentienceBrowser
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+from .models import ReadResult
 
 
 def read(
     browser: SentienceBrowser,
     output_format: Literal["raw", "text", "markdown"] = "raw",
     enhance_markdown: bool = True,
-) -> dict:
+) -> ReadResult:
     """
     Read page content as raw HTML, text, or markdown
 
@@ -93,4 +94,95 @@ def read(
         {"format": output_format},
     )
 
-    return result
+    # Convert dict result to ReadResult model
+    return ReadResult(**result)
+
+
+async def read_async(
+    browser: AsyncSentienceBrowser,
+    output_format: Literal["raw", "text", "markdown"] = "raw",
+    enhance_markdown: bool = True,
+) -> ReadResult:
+    """
+    Read page content as raw HTML, text, or markdown (async)
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        output_format: Output format - "raw" (default, returns HTML for external processing),
+                        "text" (plain text), or "markdown" (lightweight or enhanced markdown).
+        enhance_markdown: If True and output_format is "markdown", uses markdownify for better conversion.
+                          If False, uses the extension's lightweight markdown converter.
+
+    Returns:
+        dict with:
+            - status: "success" or "error"
+            - url: Current page URL
+            - format: "raw", "text", or "markdown"
+            - content: Page content as string
+            - length: Content length in characters
+            - error: Error message if status is "error"
+
+    Examples:
+        # Get raw HTML (default) - can be used with markdownify for better conversion
+        result = await read_async(browser)
+        html_content = result["content"]
+
+        # Get high-quality markdown (uses markdownify internally)
+        result = await read_async(browser, output_format="markdown")
+        markdown = result["content"]
+
+        # Get plain text
+        result = await read_async(browser, output_format="text")
+        text = result["content"]
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    if output_format == "markdown" and enhance_markdown:
+        # Get raw HTML from the extension first
+        raw_html_result = await browser.page.evaluate(
+            """
+            (options) => {
+                return window.sentience.read(options);
+            }
+            """,
+            {"format": "raw"},
+        )
+
+        if raw_html_result.get("status") == "success":
+            html_content = raw_html_result["content"]
+            try:
+                # Use markdownify for enhanced markdown conversion
+                from markdownify import MarkdownifyError, markdownify
+
+                markdown_content = markdownify(html_content, heading_style="ATX", wrap=True)
+                return {
+                    "status": "success",
+                    "url": raw_html_result["url"],
+                    "format": "markdown",
+                    "content": markdown_content,
+                    "length": len(markdown_content),
+                }
+            except ImportError:
+                print(
+                    "Warning: 'markdownify' not installed. Install with 'pip install markdownify' for enhanced markdown. Falling back to extension's markdown."
+                )
+            except MarkdownifyError as e:
+                print(f"Warning: markdownify failed ({e}), falling back to extension's markdown.")
+            except Exception as e:
+                print(
+                    f"Warning: An unexpected error occurred with markdownify ({e}), falling back to extension's markdown."
+                )
+
+    # If not enhanced markdown, or fallback, call extension with requested format
+    result = await browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.read(options);
+        }
+        """,
+        {"format": output_format},
+    )
+
+    # Convert dict result to ReadResult model
+    return ReadResult(**result)

sentience/recorder.py

@@ -4,11 +4,11 @@ Recorder - captures user actions into a trace
 
 import json
 from datetime import datetime
-from typing import Any
+from typing import Any, Optional
 
-from .browser import SentienceBrowser
+from .browser import AsyncSentienceBrowser, SentienceBrowser
 from .models import Element, Snapshot
-from .snapshot import snapshot
+from .snapshot import snapshot, snapshot_async
 
 
 class TraceStep:
@@ -367,3 +367,223 @@ def record(browser: SentienceBrowser, capture_snapshots: bool = False) -> Record
         Recorder instance
     """
     return Recorder(browser, capture_snapshots=capture_snapshots)
+
+
+class RecorderAsync:
+    """Recorder for capturing user actions (async)"""
+
+    def __init__(self, browser: AsyncSentienceBrowser, capture_snapshots: bool = False):
+        self.browser = browser
+        self.capture_snapshots = capture_snapshots
+        self.trace: Trace | None = None
+        self._active = False
+        self._mask_patterns: list[str] = []  # Patterns to mask (e.g., "password", "email")
+
+    async def start(self) -> None:
+        """Start recording"""
+        if not self.browser.page:
+            raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+        self._active = True
+        start_url = self.browser.page.url
+        self.trace = Trace(start_url)
+
+        # Set up event listeners in the browser
+        self._setup_listeners()
+
+    def stop(self) -> None:
+        """Stop recording"""
+        self._active = False
+        self._cleanup_listeners()
+
+    def add_mask_pattern(self, pattern: str) -> None:
+        """Add a pattern to mask in recorded text (e.g., "password", "email")"""
+        self._mask_patterns.append(pattern.lower())
+
+    def _should_mask(self, text: str) -> bool:
+        """Check if text should be masked"""
+        text_lower = text.lower()
+        return any(pattern in text_lower for pattern in self._mask_patterns)
+
+    def _setup_listeners(self) -> None:
+        """Set up event listeners to capture actions"""
+        # Note: We'll capture actions through the SDK methods rather than DOM events
+        # This is cleaner and more reliable
+        pass
+
+    def _cleanup_listeners(self) -> None:
+        """Clean up event listeners"""
+        pass
+
+    async def _infer_selector(self, element_id: int) -> str | None:  # noqa: C901
+        """
+        Infer a semantic selector for an element (async)
+
+        Uses heuristics to build a robust selector:
+        - role=... text~"..."
+        - If text empty: use name/aria-label/placeholder
+        - Include clickable=true when relevant
+        - Validate against snapshot (should match 1 element)
+        """
+        try:
+            # Take a snapshot to get element info
+            snap = await snapshot_async(self.browser)
+
+            # Find the element in the snapshot
+            element = None
+            for el in snap.elements:
+                if el.id == element_id:
+                    element = el
+                    break
+
+            if not element:
+                return None
+
+            # Build candidate selector
+            parts = []
+
+            # Add role
+            if element.role and element.role != "generic":
+                parts.append(f"role={element.role}")
+
+            # Add text if available
+            if element.text:
+                # Use contains match for text
+                text_part = element.text.replace('"', '\\"')[:50]  # Limit length
+                parts.append(f'text~"{text_part}"')
+            else:
+                # Try to get name/aria-label/placeholder from DOM
+                try:
+                    el = await self.browser.page.evaluate(
+                        f"""
+                        () => {{
+                            const el = window.sentience_registry[{element_id}];
+                            if (!el) return null;
+                            return {{
+                                name: el.name || null,
+                                ariaLabel: el.getAttribute('aria-label') || null,
+                                placeholder: el.placeholder || null
+                            }};
+                        }}
+                    """
+                    )
+
+                    if el:
+                        if el.get("name"):
+                            parts.append(f'name="{el["name"]}"')
+                        elif el.get("ariaLabel"):
+                            parts.append(f'text~"{el["ariaLabel"]}"')
+                        elif el.get("placeholder"):
+                            parts.append(f'text~"{el["placeholder"]}"')
+                except Exception:
+                    pass
+
+            # Add clickable if relevant
+            if element.visual_cues.is_clickable:
+                parts.append("clickable=true")
+
+            if not parts:
+                return None
+
+            selector = " ".join(parts)
+
+            # Validate selector - should match exactly 1 element
+            matches = [el for el in snap.elements if self._match_element(el, selector)]
+
+            if len(matches) == 1:
+                return selector
+            elif len(matches) > 1:
+                # Add more constraints (importance threshold, near-center)
+                # For now, just return the selector with a note
+                return selector
+            else:
+                # Selector doesn't match - return None (will use element_id)
+                return None
+
+        except Exception:
+            return None
+
+    def _match_element(self, element: Element, selector: str) -> bool:
+        """Simple selector matching (basic implementation)"""
+        # This is a simplified version - in production, use the full query engine
+        from .query import match_element, parse_selector
+
+        try:
+            query_dict = parse_selector(selector)
+            return match_element(element, query_dict)
+        except Exception:
+            return False
+
+    def record_navigation(self, url: str) -> None:
+        """Record a navigation event"""
+        if self._active and self.trace:
+            self.trace.add_navigation(url)
+
+    async def record_click(self, element_id: int, selector: str | None = None) -> None:
+        """Record a click event with smart selector inference (async)"""
+        if self._active and self.trace:
+            # If no selector provided, try to infer one
+            if selector is None:
+                selector = await self._infer_selector(element_id)
+
+            # Optionally capture snapshot
+            if self.capture_snapshots:
+                try:
+                    snap = await snapshot_async(self.browser)
+                    step = TraceStep(
+                        ts=int((datetime.now() - self.trace._start_time).total_seconds() * 1000),
+                        type="click",
+                        element_id=element_id,
+                        selector=selector,
+                        snapshot=snap,
+                    )
+                    self.trace.add_step(step)
+                except Exception:
+                    # If snapshot fails, just record without it
+                    self.trace.add_click(element_id, selector)
+            else:
+                self.trace.add_click(element_id, selector)
+
+    async def record_type(self, element_id: int, text: str, selector: str | None = None) -> None:
+        """Record a type event with smart selector inference (async)"""
+        if self._active and self.trace:
+            # If no selector provided, try to infer one
+            if selector is None:
+                selector = await self._infer_selector(element_id)
+
+            mask = self._should_mask(text)
+            self.trace.add_type(element_id, text, selector, mask=mask)
+
+    def record_press(self, key: str) -> None:
+        """Record a key press event"""
+        if self._active and self.trace:
+            self.trace.add_press(key)
+
+    def save(self, filepath: str) -> None:
+        """Save trace to file"""
+        if not self.trace:
+            raise RuntimeError("No trace to save. Start recording first.")
+        self.trace.save(filepath)
+
+    async def __aenter__(self):
+        """Context manager entry"""
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit"""
+        self.stop()
+
+
+def record_async(browser: AsyncSentienceBrowser, capture_snapshots: bool = False) -> RecorderAsync:
+    """
+    Create a recorder instance (async)
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        capture_snapshots: Whether to capture snapshots at each step
+
+    Returns:
+        RecorderAsync instance
+    """
+    return RecorderAsync(browser, capture_snapshots=capture_snapshots)

sentience/schemas/trace_v1.json

@@ -13,7 +13,7 @@
     },
     "type": {
       "type": "string",
-      "enum": ["run_start", "step_start", "snapshot_taken", "llm_called", "action_executed", "verification", "recovery", "step_end", "run_end", "error"],
+      "enum": ["run_start", "step_start", "snapshot", "snapshot_taken", "llm_called", "llm_response", "action", "action_executed", "verification", "recovery", "step_end", "run_end", "error"],
       "description": "Event type"
     },
     "ts": {
@@ -64,15 +64,61 @@
           }
         },
         {
-          "description": "snapshot_taken data",
-          "required": ["step_id", "snapshot_digest"],
+          "description": "snapshot or snapshot_taken data",
           "properties": {
-            "step_id": {"type": "string"},
+            "step_id": {"type": ["string", "null"]},
             "snapshot_id": {"type": ["string", "null"]},
             "snapshot_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
             "snapshot_digest_loose": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
             "url": {"type": ["string", "null"]},
-            "element_count": {"type": "integer"}
+            "element_count": {"type": "integer"},
+            "timestamp": {"type": ["string", "null"]},
+            "elements": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "id": {"type": "integer"},
+                  "role": {"type": "string"},
+                  "text": {"type": ["string", "null"]},
+                  "importance": {"type": "number"},
+                  "importance_score": {"type": "number"},
+                  "bbox": {
+                    "type": "object",
+                    "properties": {
+                      "x": {"type": "number"},
+                      "y": {"type": "number"},
+                      "width": {"type": "number"},
+                      "height": {"type": "number"}
+                    },
+                    "required": ["x", "y", "width", "height"]
+                  },
+                  "visual_cues": {
+                    "type": "object",
+                    "properties": {
+                      "is_primary": {"type": "boolean"},
+                      "is_clickable": {"type": "boolean"},
+                      "background_color_name": {"type": ["string", "null"]}
+                    }
+                  },
+                  "in_viewport": {"type": "boolean"},
+                  "is_occluded": {"type": "boolean"},
+                  "z_index": {"type": "integer"},
+                  "rerank_index": {"type": ["integer", "null"]},
+                  "heuristic_index": {"type": ["integer", "null"]},
+                  "ml_probability": {"type": ["number", "null"]},
+                  "ml_score": {"type": ["number", "null"]},
+                  "diff_status": {
+                    "type": ["string", "null"],
+                    "enum": ["ADDED", "REMOVED", "MODIFIED", "MOVED", null],
+                    "description": "Diff status for Diff Overlay feature. ADDED: new element, REMOVED: element was removed, MODIFIED: element changed, MOVED: element position changed, null: no change"
+                  }
+                },
+                "required": ["id", "role", "importance", "bbox", "visual_cues"]
+              }
+            },
+            "screenshot_base64": {"type": ["string", "null"]},
+            "screenshot_format": {"type": ["string", "null"], "enum": ["png", "jpeg", null]}
           }
         },
         {
@@ -119,7 +165,15 @@
               "required": ["response_text", "response_hash"],
               "properties": {
                 "response_text": {"type": "string"},
-                "response_hash": {"type": "string"}
+                "response_hash": {"type": "string"},
+                "usage": {
+                  "type": "object",
+                  "properties": {
+                    "prompt_tokens": {"type": "integer"},
+                    "completion_tokens": {"type": "integer"},
+                    "total_tokens": {"type": "integer"}
+                  }
+                }
               }
             },
             "action": {
@@ -145,7 +199,17 @@
                 "text": {"type": "string"},
                 "key": {"type": "string"},
                 "url_changed": {"type": ["boolean", "null"]},
-                "duration_ms": {"type": "integer"}
+                "duration_ms": {"type": "integer"},
+                "error": {"type": ["string", "null"]},
+                "bounding_box": {
+                  "type": "object",
+                  "properties": {
+                    "x": {"type": "number"},
+                    "y": {"type": "number"},
+                    "width": {"type": "number"},
+                    "height": {"type": "number"}
+                  }
+                }
               }
             },
             "post": {
@@ -162,7 +226,48 @@
               "properties": {
                 "policy": {"type": "string"},
                 "passed": {"type": "boolean"},
-                "signals": {"type": "object"}
+                "signals": {
+                  "type": "object",
+                  "properties": {
+                    "url_changed": {"type": "boolean"},
+                    "error": {"type": ["string", "null"]},
+                    "elements_found": {
+                      "type": "array",
+                      "items": {
+                        "type": "object",
+                        "properties": {
+                          "label": {"type": "string"},
+                          "bounding_box": {
+                            "type": "object",
+                            "properties": {
+                              "x": {"type": "number"},
+                              "y": {"type": "number"},
+                              "width": {"type": "number"},
+                              "height": {"type": "number"}
+                            }
+                          }
+                        }
+                      }
+                    },
+                    "assertions": {
+                      "type": "array",
+                      "description": "Assertion results from agent verification loop",
+                      "items": {
+                        "type": "object",
+                        "required": ["label", "passed"],
+                        "properties": {
+                          "label": {"type": "string", "description": "Human-readable assertion label"},
+                          "passed": {"type": "boolean", "description": "Whether the assertion passed"},
+                          "required": {"type": "boolean", "description": "If true, assertion gates step success"},
+                          "reason": {"type": "string", "description": "Explanation (especially when failed)"},
+                          "details": {"type": "object", "description": "Additional structured data for debugging"}
+                        }
+                      }
+                    },
+                    "task_done": {"type": "boolean", "description": "True if task completion assertion passed"},
+                    "task_done_label": {"type": "string", "description": "Label of the task completion assertion"}
+                  }
+                }
               }
             },
             "recovery": {
@@ -182,6 +287,15 @@
           "properties": {
             "step_id": {"type": "string"},
             "passed": {"type": "boolean"},
+            "kind": {
+              "type": "string",
+              "enum": ["assert", "task_done"],
+              "description": "Type of verification event"
+            },
+            "label": {"type": "string", "description": "Human-readable label for the assertion"},
+            "required": {"type": "boolean", "description": "If true, assertion gates step success"},
+            "reason": {"type": "string", "description": "Explanation (especially when failed)"},
+            "details": {"type": "object", "description": "Additional structured data for debugging"},
             "signals": {"type": "object"}
           }
         },
@@ -198,7 +312,12 @@
           "description": "run_end data",
           "required": ["steps"],
           "properties": {
-            "steps": {"type": "integer"}
+            "steps": {"type": "integer"},
+            "status": {
+              "type": "string",
+              "enum": ["success", "failure", "partial", "unknown"],
+              "description": "Final execution status"
+            }
           }
         },
         {