sentienceapi 0.90.11__py3-none-any.whl

sentience/recorder.py

@@ -0,0 +1,369 @@
+"""
+Recorder - captures user actions into a trace
+"""
+
+import json
+from datetime import datetime
+from typing import Any
+
+from .browser import SentienceBrowser
+from .models import Element, Snapshot
+from .snapshot import snapshot
+
+
+class TraceStep:
+    """A single step in a trace"""
+
+    def __init__(
+        self,
+        ts: int,
+        type: str,
+        selector: str | None = None,
+        element_id: int | None = None,
+        text: str | None = None,
+        key: str | None = None,
+        url: str | None = None,
+        snapshot: Snapshot | None = None,
+    ):
+        self.ts = ts
+        self.type = type
+        self.selector = selector
+        self.element_id = element_id
+        self.text = text
+        self.key = key
+        self.url = url
+        self.snapshot = snapshot
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization"""
+        result = {
+            "ts": self.ts,
+            "type": self.type,
+        }
+        if self.selector is not None:
+            result["selector"] = self.selector
+        if self.element_id is not None:
+            result["element_id"] = self.element_id
+        if self.text is not None:
+            result["text"] = self.text
+        if self.key is not None:
+            result["key"] = self.key
+        if self.url is not None:
+            result["url"] = self.url
+        if self.snapshot is not None:
+            result["snapshot"] = self.snapshot.model_dump()
+        return result
+
+
+class Trace:
+    """Trace of user actions"""
+
+    def __init__(self, start_url: str):
+        self.version = "1.0.0"
+        self.created_at = datetime.now().isoformat()
+        self.start_url = start_url
+        self.steps: list[TraceStep] = []
+        self._start_time = datetime.now()
+
+    def add_step(self, step: TraceStep) -> None:
+        """Add a step to the trace"""
+        self.steps.append(step)
+
+    def add_navigation(self, url: str) -> None:
+        """Add navigation step"""
+        ts = int((datetime.now() - self._start_time).total_seconds() * 1000)
+        step = TraceStep(ts=ts, type="navigation", url=url)
+        self.add_step(step)
+
+    def add_click(self, element_id: int, selector: str | None = None) -> None:
+        """Add click step"""
+        ts = int((datetime.now() - self._start_time).total_seconds() * 1000)
+        step = TraceStep(ts=ts, type="click", element_id=element_id, selector=selector)
+        self.add_step(step)
+
+    def add_type(
+        self,
+        element_id: int,
+        text: str,
+        selector: str | None = None,
+        mask: bool = False,
+    ) -> None:
+        """Add type step"""
+        ts = int((datetime.now() - self._start_time).total_seconds() * 1000)
+        # Mask sensitive data if requested
+        masked_text = "***" if mask else text
+        step = TraceStep(
+            ts=ts,
+            type="type",
+            element_id=element_id,
+            text=masked_text,
+            selector=selector,
+        )
+        self.add_step(step)
+
+    def add_press(self, key: str) -> None:
+        """Add press key step"""
+        ts = int((datetime.now() - self._start_time).total_seconds() * 1000)
+        step = TraceStep(ts=ts, type="press", key=key)
+        self.add_step(step)
+
+    def save(self, filepath: str) -> None:
+        """Save trace to JSON file"""
+        data = {
+            "version": self.version,
+            "created_at": self.created_at,
+            "start_url": self.start_url,
+            "steps": [step.to_dict() for step in self.steps],
+        }
+        with open(filepath, "w") as f:
+            json.dump(data, f, indent=2)
+
+    @classmethod
+    def load(cls, filepath: str) -> "Trace":
+        """Load trace from JSON file"""
+        with open(filepath) as f:
+            data = json.load(f)
+
+        trace = cls(data["start_url"])
+        trace.version = data["version"]
+        trace.created_at = data["created_at"]
+
+        for step_data in data["steps"]:
+            snapshot_data = step_data.get("snapshot")
+            snapshot_obj = None
+            if snapshot_data:
+                snapshot_obj = Snapshot(**snapshot_data)
+
+            step = TraceStep(
+                ts=step_data["ts"],
+                type=step_data["type"],
+                selector=step_data.get("selector"),
+                element_id=step_data.get("element_id"),
+                text=step_data.get("text"),
+                key=step_data.get("key"),
+                url=step_data.get("url"),
+                snapshot=snapshot_obj,
+            )
+            trace.steps.append(step)
+
+        return trace
+
+
+class Recorder:
+    """Recorder for capturing user actions"""
+
+    def __init__(self, browser: SentienceBrowser, capture_snapshots: bool = False):
+        self.browser = browser
+        self.capture_snapshots = capture_snapshots
+        self.trace: Trace | None = None
+        self._active = False
+        self._mask_patterns: list[str] = []  # Patterns to mask (e.g., "password", "email")
+
+    def start(self) -> None:
+        """Start recording"""
+        if not self.browser.page:
+            raise RuntimeError("Browser not started. Call browser.start() first.")
+
+        self._active = True
+        start_url = self.browser.page.url
+        self.trace = Trace(start_url)
+
+        # Set up event listeners in the browser
+        self._setup_listeners()
+
+    def stop(self) -> None:
+        """Stop recording"""
+        self._active = False
+        self._cleanup_listeners()
+
+    def add_mask_pattern(self, pattern: str) -> None:
+        """Add a pattern to mask in recorded text (e.g., "password", "email")"""
+        self._mask_patterns.append(pattern.lower())
+
+    def _should_mask(self, text: str) -> bool:
+        """Check if text should be masked"""
+        text_lower = text.lower()
+        return any(pattern in text_lower for pattern in self._mask_patterns)
+
+    def _setup_listeners(self) -> None:
+        """Set up event listeners to capture actions"""
+        # Note: We'll capture actions through the SDK methods rather than DOM events
+        # This is cleaner and more reliable
+        pass
+
+    def _cleanup_listeners(self) -> None:
+        """Clean up event listeners"""
+        pass
+
+    def _infer_selector(self, element_id: int) -> str | None:  # noqa: C901
+        """
+        Infer a semantic selector for an element
+
+        Uses heuristics to build a robust selector:
+        - role=... text~"..."
+        - If text empty: use name/aria-label/placeholder
+        - Include clickable=true when relevant
+        - Validate against snapshot (should match 1 element)
+        """
+        try:
+            # Take a snapshot to get element info
+            snap = snapshot(self.browser)
+
+            # Find the element in the snapshot
+            element = None
+            for el in snap.elements:
+                if el.id == element_id:
+                    element = el
+                    break
+
+            if not element:
+                return None
+
+            # Build candidate selector
+            parts = []
+
+            # Add role
+            if element.role and element.role != "generic":
+                parts.append(f"role={element.role}")
+
+            # Add text if available
+            if element.text:
+                # Use contains match for text
+                text_part = element.text.replace('"', '\\"')[:50]  # Limit length
+                parts.append(f'text~"{text_part}"')
+            else:
+                # Try to get name/aria-label/placeholder from DOM
+                try:
+                    el = self.browser.page.evaluate(
+                        f"""
+                        () => {{
+                            const el = window.sentience_registry[{element_id}];
+                            if (!el) return null;
+                            return {{
+                                name: el.name || null,
+                                ariaLabel: el.getAttribute('aria-label') || null,
+                                placeholder: el.placeholder || null
+                            }};
+                        }}
+                    """
+                    )
+
+                    if el:
+                        if el.get("name"):
+                            parts.append(f'name="{el["name"]}"')
+                        elif el.get("ariaLabel"):
+                            parts.append(f'text~"{el["ariaLabel"]}"')
+                        elif el.get("placeholder"):
+                            parts.append(f'text~"{el["placeholder"]}"')
+                except Exception:
+                    pass
+
+            # Add clickable if relevant
+            if element.visual_cues.is_clickable:
+                parts.append("clickable=true")
+
+            if not parts:
+                return None
+
+            selector = " ".join(parts)
+
+            # Validate selector - should match exactly 1 element
+            matches = [el for el in snap.elements if self._match_element(el, selector)]
+
+            if len(matches) == 1:
+                return selector
+            elif len(matches) > 1:
+                # Add more constraints (importance threshold, near-center)
+                # For now, just return the selector with a note
+                return selector
+            else:
+                # Selector doesn't match - return None (will use element_id)
+                return None
+
+        except Exception:
+            return None
+
+    def _match_element(self, element: Element, selector: str) -> bool:
+        """Simple selector matching (basic implementation)"""
+        # This is a simplified version - in production, use the full query engine
+        from .query import match_element, parse_selector
+
+        try:
+            query_dict = parse_selector(selector)
+            return match_element(element, query_dict)
+        except Exception:
+            return False
+
+    def record_navigation(self, url: str) -> None:
+        """Record a navigation event"""
+        if self._active and self.trace:
+            self.trace.add_navigation(url)
+
+    def record_click(self, element_id: int, selector: str | None = None) -> None:
+        """Record a click event with smart selector inference"""
+        if self._active and self.trace:
+            # If no selector provided, try to infer one
+            if selector is None:
+                selector = self._infer_selector(element_id)
+
+            # Optionally capture snapshot
+            if self.capture_snapshots:
+                try:
+                    snap = snapshot(self.browser)
+                    step = TraceStep(
+                        ts=int((datetime.now() - self.trace._start_time).total_seconds() * 1000),
+                        type="click",
+                        element_id=element_id,
+                        selector=selector,
+                        snapshot=snap,
+                    )
+                    self.trace.add_step(step)
+                except Exception:
+                    # If snapshot fails, just record without it
+                    self.trace.add_click(element_id, selector)
+            else:
+                self.trace.add_click(element_id, selector)
+
+    def record_type(self, element_id: int, text: str, selector: str | None = None) -> None:
+        """Record a type event with smart selector inference"""
+        if self._active and self.trace:
+            # If no selector provided, try to infer one
+            if selector is None:
+                selector = self._infer_selector(element_id)
+
+            mask = self._should_mask(text)
+            self.trace.add_type(element_id, text, selector, mask=mask)
+
+    def record_press(self, key: str) -> None:
+        """Record a key press event"""
+        if self._active and self.trace:
+            self.trace.add_press(key)
+
+    def save(self, filepath: str) -> None:
+        """Save trace to file"""
+        if not self.trace:
+            raise RuntimeError("No trace to save. Start recording first.")
+        self.trace.save(filepath)
+
+    def __enter__(self):
+        """Context manager entry"""
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit"""
+        self.stop()
+
+
+def record(browser: SentienceBrowser, capture_snapshots: bool = False) -> Recorder:
+    """
+    Create a recorder instance
+
+    Args:
+        browser: SentienceBrowser instance
+        capture_snapshots: Whether to capture snapshots at each step
+
+    Returns:
+        Recorder instance
+    """
+    return Recorder(browser, capture_snapshots=capture_snapshots)

sentience/schemas/trace_v1.json

@@ -0,0 +1,216 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://sentience.ai/schemas/trace/v1",
+  "title": "Sentience Agent Trace Event",
+  "description": "Schema for Sentience agent trace events in JSONL format",
+  "type": "object",
+  "required": ["v", "type", "ts", "run_id", "seq", "data"],
+  "properties": {
+    "v": {
+      "type": "integer",
+      "const": 1,
+      "description": "Schema version"
+    },
+    "type": {
+      "type": "string",
+      "enum": ["run_start", "step_start", "snapshot_taken", "llm_called", "action_executed", "verification", "recovery", "step_end", "run_end", "error"],
+      "description": "Event type"
+    },
+    "ts": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp"
+    },
+    "ts_ms": {
+      "type": "integer",
+      "description": "Unix timestamp in milliseconds"
+    },
+    "run_id": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
+      "description": "UUID for the agent run"
+    },
+    "seq": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Monotonically increasing sequence number"
+    },
+    "step_id": {
+      "type": ["string", "null"],
+      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
+      "description": "UUID for the step (present for step-scoped events)"
+    },
+    "data": {
+      "type": "object",
+      "description": "Event-specific payload",
+      "oneOf": [
+        {
+          "description": "run_start data",
+          "properties": {
+            "agent": {"type": "string"},
+            "llm_model": {"type": ["string", "null"]},
+            "config": {"type": "object"}
+          }
+        },
+        {
+          "description": "step_start data",
+          "required": ["step_id", "step_index", "goal", "attempt"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "step_index": {"type": "integer"},
+            "goal": {"type": "string"},
+            "attempt": {"type": "integer"},
+            "pre_url": {"type": ["string", "null"]}
+          }
+        },
+        {
+          "description": "snapshot_taken data",
+          "required": ["step_id", "snapshot_digest"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "snapshot_id": {"type": ["string", "null"]},
+            "snapshot_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "snapshot_digest_loose": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "url": {"type": ["string", "null"]},
+            "element_count": {"type": "integer"}
+          }
+        },
+        {
+          "description": "llm_called data",
+          "required": ["step_id", "response_text", "response_hash"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "model": {"type": ["string", "null"]},
+            "temperature": {"type": "number"},
+            "system_prompt_hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "user_prompt_hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "response_text": {"type": "string"},
+            "response_hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "usage": {
+              "type": "object",
+              "properties": {
+                "prompt_tokens": {"type": "integer"},
+                "completion_tokens": {"type": "integer"},
+                "total_tokens": {"type": "integer"}
+              }
+            }
+          }
+        },
+        {
+          "description": "step_end data (StepResult)",
+          "required": ["step_id", "step_index", "goal", "attempt", "pre", "llm", "exec", "post", "verify"],
+          "properties": {
+            "v": {"type": "integer", "const": 1},
+            "step_id": {"type": "string"},
+            "step_index": {"type": "integer"},
+            "goal": {"type": "string"},
+            "attempt": {"type": "integer"},
+            "pre": {
+              "type": "object",
+              "required": ["snapshot_digest"],
+              "properties": {
+                "url": {"type": ["string", "null"]},
+                "snapshot_digest": {"type": "string"},
+                "snapshot_digest_loose": {"type": "string"}
+              }
+            },
+            "llm": {
+              "type": "object",
+              "required": ["response_text", "response_hash"],
+              "properties": {
+                "response_text": {"type": "string"},
+                "response_hash": {"type": "string"}
+              }
+            },
+            "action": {
+              "type": "object",
+              "required": ["kind"],
+              "properties": {
+                "kind": {"type": "string", "enum": ["click", "type", "press", "finish", "navigate"]},
+                "element_id": {"type": "integer"},
+                "text": {"type": "string"},
+                "key": {"type": "string"},
+                "url": {"type": "string"},
+                "raw": {"type": "string"}
+              }
+            },
+            "exec": {
+              "type": "object",
+              "required": ["success", "outcome", "duration_ms"],
+              "properties": {
+                "success": {"type": "boolean"},
+                "outcome": {"type": "string"},
+                "action": {"type": "string"},
+                "element_id": {"type": "integer"},
+                "text": {"type": "string"},
+                "key": {"type": "string"},
+                "url_changed": {"type": ["boolean", "null"]},
+                "duration_ms": {"type": "integer"}
+              }
+            },
+            "post": {
+              "type": "object",
+              "properties": {
+                "url": {"type": ["string", "null"]},
+                "snapshot_digest": {"type": "string"},
+                "snapshot_digest_loose": {"type": "string"}
+              }
+            },
+            "verify": {
+              "type": "object",
+              "required": ["passed"],
+              "properties": {
+                "policy": {"type": "string"},
+                "passed": {"type": "boolean"},
+                "signals": {"type": "object"}
+              }
+            },
+            "recovery": {
+              "type": ["object", "null"],
+              "properties": {
+                "attempted": {"type": "boolean"},
+                "success": {"type": "boolean"},
+                "strategy": {"type": "string"},
+                "attempts": {"type": "array"}
+              }
+            }
+          }
+        },
+        {
+          "description": "verification data",
+          "required": ["step_id", "passed"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "passed": {"type": "boolean"},
+            "signals": {"type": "object"}
+          }
+        },
+        {
+          "description": "recovery data",
+          "required": ["step_id", "strategy"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "strategy": {"type": "string"},
+            "attempt": {"type": "integer"}
+          }
+        },
+        {
+          "description": "run_end data",
+          "required": ["steps"],
+          "properties": {
+            "steps": {"type": "integer"}
+          }
+        },
+        {
+          "description": "error data",
+          "required": ["step_id", "error"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "attempt": {"type": "integer"},
+            "error": {"type": "string"}
+          }
+        }
+      ]
+    }
+  }
+}

sentience/screenshot.py

@@ -0,0 +1,54 @@
+"""
+Screenshot functionality - standalone screenshot capture
+"""
+
+from typing import Any, Literal
+
+from .browser import SentienceBrowser
+
+
+def screenshot(
+    browser: SentienceBrowser,
+    format: Literal["png", "jpeg"] = "png",
+    quality: int | None = None,
+) -> str:
+    """
+    Capture screenshot of current page
+
+    Args:
+        browser: SentienceBrowser instance
+        format: Image format - "png" or "jpeg"
+        quality: JPEG quality (1-100), only used for JPEG format
+
+    Returns:
+        Base64-encoded screenshot data URL (e.g., "data:image/png;base64,...")
+
+    Raises:
+        RuntimeError: If browser not started
+        ValueError: If quality is invalid for JPEG
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    if format == "jpeg" and quality is not None:
+        if not (1 <= quality <= 100):
+            raise ValueError("Quality must be between 1 and 100 for JPEG format")
+
+    # Use Playwright's screenshot with base64 encoding
+    screenshot_options: dict[str, Any] = {
+        "type": format,
+    }
+
+    if format == "jpeg" and quality is not None:
+        screenshot_options["quality"] = quality
+
+    # Capture screenshot as base64
+    # Playwright returns bytes when encoding is not specified, so we encode manually
+    import base64
+
+    image_bytes = browser.page.screenshot(**screenshot_options)
+    base64_data = base64.b64encode(image_bytes).decode("utf-8")
+
+    # Return as data URL
+    mime_type = "image/png" if format == "png" else "image/jpeg"
+    return f"data:{mime_type};base64,{base64_data}"