sentienceapi 0.95.0__py3-none-any.whl

sentience/schemas/trace_v1.json

@@ -0,0 +1,335 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://sentience.ai/schemas/trace/v1",
+  "title": "Sentience Agent Trace Event",
+  "description": "Schema for Sentience agent trace events in JSONL format",
+  "type": "object",
+  "required": ["v", "type", "ts", "run_id", "seq", "data"],
+  "properties": {
+    "v": {
+      "type": "integer",
+      "const": 1,
+      "description": "Schema version"
+    },
+    "type": {
+      "type": "string",
+      "enum": ["run_start", "step_start", "snapshot", "snapshot_taken", "llm_called", "llm_response", "action", "action_executed", "verification", "recovery", "step_end", "run_end", "error"],
+      "description": "Event type"
+    },
+    "ts": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp"
+    },
+    "ts_ms": {
+      "type": "integer",
+      "description": "Unix timestamp in milliseconds"
+    },
+    "run_id": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
+      "description": "UUID for the agent run"
+    },
+    "seq": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Monotonically increasing sequence number"
+    },
+    "step_id": {
+      "type": ["string", "null"],
+      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
+      "description": "UUID for the step (present for step-scoped events)"
+    },
+    "data": {
+      "type": "object",
+      "description": "Event-specific payload",
+      "oneOf": [
+        {
+          "description": "run_start data",
+          "properties": {
+            "agent": {"type": "string"},
+            "llm_model": {"type": ["string", "null"]},
+            "config": {"type": "object"}
+          }
+        },
+        {
+          "description": "step_start data",
+          "required": ["step_id", "step_index", "goal", "attempt"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "step_index": {"type": "integer"},
+            "goal": {"type": "string"},
+            "attempt": {"type": "integer"},
+            "pre_url": {"type": ["string", "null"]}
+          }
+        },
+        {
+          "description": "snapshot or snapshot_taken data",
+          "properties": {
+            "step_id": {"type": ["string", "null"]},
+            "snapshot_id": {"type": ["string", "null"]},
+            "snapshot_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "snapshot_digest_loose": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "url": {"type": ["string", "null"]},
+            "element_count": {"type": "integer"},
+            "timestamp": {"type": ["string", "null"]},
+            "elements": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "id": {"type": "integer"},
+                  "role": {"type": "string"},
+                  "text": {"type": ["string", "null"]},
+                  "importance": {"type": "number"},
+                  "importance_score": {"type": "number"},
+                  "bbox": {
+                    "type": "object",
+                    "properties": {
+                      "x": {"type": "number"},
+                      "y": {"type": "number"},
+                      "width": {"type": "number"},
+                      "height": {"type": "number"}
+                    },
+                    "required": ["x", "y", "width", "height"]
+                  },
+                  "visual_cues": {
+                    "type": "object",
+                    "properties": {
+                      "is_primary": {"type": "boolean"},
+                      "is_clickable": {"type": "boolean"},
+                      "background_color_name": {"type": ["string", "null"]}
+                    }
+                  },
+                  "in_viewport": {"type": "boolean"},
+                  "is_occluded": {"type": "boolean"},
+                  "z_index": {"type": "integer"},
+                  "rerank_index": {"type": ["integer", "null"]},
+                  "heuristic_index": {"type": ["integer", "null"]},
+                  "ml_probability": {"type": ["number", "null"]},
+                  "ml_score": {"type": ["number", "null"]},
+                  "diff_status": {
+                    "type": ["string", "null"],
+                    "enum": ["ADDED", "REMOVED", "MODIFIED", "MOVED", null],
+                    "description": "Diff status for Diff Overlay feature. ADDED: new element, REMOVED: element was removed, MODIFIED: element changed, MOVED: element position changed, null: no change"
+                  }
+                },
+                "required": ["id", "role", "importance", "bbox", "visual_cues"]
+              }
+            },
+            "screenshot_base64": {"type": ["string", "null"]},
+            "screenshot_format": {"type": ["string", "null"], "enum": ["png", "jpeg", null]}
+          }
+        },
+        {
+          "description": "llm_called data",
+          "required": ["step_id", "response_text", "response_hash"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "model": {"type": ["string", "null"]},
+            "temperature": {"type": "number"},
+            "system_prompt_hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "user_prompt_hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "response_text": {"type": "string"},
+            "response_hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+            "usage": {
+              "type": "object",
+              "properties": {
+                "prompt_tokens": {"type": "integer"},
+                "completion_tokens": {"type": "integer"},
+                "total_tokens": {"type": "integer"}
+              }
+            }
+          }
+        },
+        {
+          "description": "step_end data (StepResult)",
+          "required": ["step_id", "step_index", "goal", "attempt", "pre", "llm", "exec", "post", "verify"],
+          "properties": {
+            "v": {"type": "integer", "const": 1},
+            "step_id": {"type": "string"},
+            "step_index": {"type": "integer"},
+            "goal": {"type": "string"},
+            "attempt": {"type": "integer"},
+            "pre": {
+              "type": "object",
+              "required": ["snapshot_digest"],
+              "properties": {
+                "url": {"type": ["string", "null"]},
+                "snapshot_digest": {"type": "string"},
+                "snapshot_digest_loose": {"type": "string"}
+              }
+            },
+            "llm": {
+              "type": "object",
+              "required": ["response_text", "response_hash"],
+              "properties": {
+                "response_text": {"type": "string"},
+                "response_hash": {"type": "string"},
+                "usage": {
+                  "type": "object",
+                  "properties": {
+                    "prompt_tokens": {"type": "integer"},
+                    "completion_tokens": {"type": "integer"},
+                    "total_tokens": {"type": "integer"}
+                  }
+                }
+              }
+            },
+            "action": {
+              "type": "object",
+              "required": ["kind"],
+              "properties": {
+                "kind": {"type": "string", "enum": ["click", "type", "press", "finish", "navigate"]},
+                "element_id": {"type": "integer"},
+                "text": {"type": "string"},
+                "key": {"type": "string"},
+                "url": {"type": "string"},
+                "raw": {"type": "string"}
+              }
+            },
+            "exec": {
+              "type": "object",
+              "required": ["success", "outcome", "duration_ms"],
+              "properties": {
+                "success": {"type": "boolean"},
+                "outcome": {"type": "string"},
+                "action": {"type": "string"},
+                "element_id": {"type": "integer"},
+                "text": {"type": "string"},
+                "key": {"type": "string"},
+                "url_changed": {"type": ["boolean", "null"]},
+                "duration_ms": {"type": "integer"},
+                "error": {"type": ["string", "null"]},
+                "bounding_box": {
+                  "type": "object",
+                  "properties": {
+                    "x": {"type": "number"},
+                    "y": {"type": "number"},
+                    "width": {"type": "number"},
+                    "height": {"type": "number"}
+                  }
+                }
+              }
+            },
+            "post": {
+              "type": "object",
+              "properties": {
+                "url": {"type": ["string", "null"]},
+                "snapshot_digest": {"type": "string"},
+                "snapshot_digest_loose": {"type": "string"}
+              }
+            },
+            "verify": {
+              "type": "object",
+              "required": ["passed"],
+              "properties": {
+                "policy": {"type": "string"},
+                "passed": {"type": "boolean"},
+                "signals": {
+                  "type": "object",
+                  "properties": {
+                    "url_changed": {"type": "boolean"},
+                    "error": {"type": ["string", "null"]},
+                    "elements_found": {
+                      "type": "array",
+                      "items": {
+                        "type": "object",
+                        "properties": {
+                          "label": {"type": "string"},
+                          "bounding_box": {
+                            "type": "object",
+                            "properties": {
+                              "x": {"type": "number"},
+                              "y": {"type": "number"},
+                              "width": {"type": "number"},
+                              "height": {"type": "number"}
+                            }
+                          }
+                        }
+                      }
+                    },
+                    "assertions": {
+                      "type": "array",
+                      "description": "Assertion results from agent verification loop",
+                      "items": {
+                        "type": "object",
+                        "required": ["label", "passed"],
+                        "properties": {
+                          "label": {"type": "string", "description": "Human-readable assertion label"},
+                          "passed": {"type": "boolean", "description": "Whether the assertion passed"},
+                          "required": {"type": "boolean", "description": "If true, assertion gates step success"},
+                          "reason": {"type": "string", "description": "Explanation (especially when failed)"},
+                          "details": {"type": "object", "description": "Additional structured data for debugging"}
+                        }
+                      }
+                    },
+                    "task_done": {"type": "boolean", "description": "True if task completion assertion passed"},
+                    "task_done_label": {"type": "string", "description": "Label of the task completion assertion"}
+                  }
+                }
+              }
+            },
+            "recovery": {
+              "type": ["object", "null"],
+              "properties": {
+                "attempted": {"type": "boolean"},
+                "success": {"type": "boolean"},
+                "strategy": {"type": "string"},
+                "attempts": {"type": "array"}
+              }
+            }
+          }
+        },
+        {
+          "description": "verification data",
+          "required": ["step_id", "passed"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "passed": {"type": "boolean"},
+            "kind": {
+              "type": "string",
+              "enum": ["assert", "task_done"],
+              "description": "Type of verification event"
+            },
+            "label": {"type": "string", "description": "Human-readable label for the assertion"},
+            "required": {"type": "boolean", "description": "If true, assertion gates step success"},
+            "reason": {"type": "string", "description": "Explanation (especially when failed)"},
+            "details": {"type": "object", "description": "Additional structured data for debugging"},
+            "signals": {"type": "object"}
+          }
+        },
+        {
+          "description": "recovery data",
+          "required": ["step_id", "strategy"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "strategy": {"type": "string"},
+            "attempt": {"type": "integer"}
+          }
+        },
+        {
+          "description": "run_end data",
+          "required": ["steps"],
+          "properties": {
+            "steps": {"type": "integer"},
+            "status": {
+              "type": "string",
+              "enum": ["success", "failure", "partial", "unknown"],
+              "description": "Final execution status"
+            }
+          }
+        },
+        {
+          "description": "error data",
+          "required": ["step_id", "error"],
+          "properties": {
+            "step_id": {"type": "string"},
+            "attempt": {"type": "integer"},
+            "error": {"type": "string"}
+          }
+        }
+      ]
+    }
+  }
+}

sentience/screenshot.py

@@ -0,0 +1,100 @@
+"""
+Screenshot functionality - standalone screenshot capture
+"""
+
+import base64
+from typing import Any, Literal, Optional
+
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+
+
+def screenshot(
+    browser: SentienceBrowser,
+    format: Literal["png", "jpeg"] = "png",
+    quality: int | None = None,
+) -> str:
+    """
+    Capture screenshot of current page
+
+    Args:
+        browser: SentienceBrowser instance
+        format: Image format - "png" or "jpeg"
+        quality: JPEG quality (1-100), only used for JPEG format
+
+    Returns:
+        Base64-encoded screenshot data URL (e.g., "data:image/png;base64,...")
+
+    Raises:
+        RuntimeError: If browser not started
+        ValueError: If quality is invalid for JPEG
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    if format == "jpeg" and quality is not None:
+        if not (1 <= quality <= 100):
+            raise ValueError("Quality must be between 1 and 100 for JPEG format")
+
+    # Use Playwright's screenshot with base64 encoding
+    screenshot_options: dict[str, Any] = {
+        "type": format,
+    }
+
+    if format == "jpeg" and quality is not None:
+        screenshot_options["quality"] = quality
+
+    # Capture screenshot as base64
+    # Playwright returns bytes when encoding is not specified, so we encode manually
+    import base64
+
+    image_bytes = browser.page.screenshot(**screenshot_options)
+    base64_data = base64.b64encode(image_bytes).decode("utf-8")
+
+    # Return as data URL
+    mime_type = "image/png" if format == "png" else "image/jpeg"
+    return f"data:{mime_type};base64,{base64_data}"
+
+
+async def screenshot_async(
+    browser: AsyncSentienceBrowser,
+    format: Literal["png", "jpeg"] = "png",
+    quality: int | None = None,
+) -> str:
+    """
+    Capture screenshot of current page (async)
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        format: Image format - "png" or "jpeg"
+        quality: JPEG quality (1-100), only used for JPEG format
+
+    Returns:
+        Base64-encoded screenshot data URL (e.g., "data:image/png;base64,...")
+
+    Raises:
+        RuntimeError: If browser not started
+        ValueError: If quality is invalid for JPEG
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    if format == "jpeg" and quality is not None:
+        if not (1 <= quality <= 100):
+            raise ValueError("Quality must be between 1 and 100 for JPEG format")
+
+    # Use Playwright's screenshot with base64 encoding
+    screenshot_options: dict[str, Any] = {
+        "type": format,
+    }
+
+    if format == "jpeg" and quality is not None:
+        screenshot_options["quality"] = quality
+
+    # Capture screenshot as base64
+    # Playwright returns bytes when encoding is not specified, so we encode manually
+    image_bytes = await browser.page.screenshot(**screenshot_options)
+    base64_data = base64.b64encode(image_bytes).decode("utf-8")
+
+    # Return as data URL
+    mime_type = "image/png" if format == "png" else "image/jpeg"
+    return f"data:{mime_type};base64,{base64_data}"

sentience/sentience_methods.py

@@ -0,0 +1,86 @@
+"""
+Enums for Sentience API methods and agent actions.
+
+This module provides type-safe enums for:
+1. window.sentience API methods (extension-level)
+2. Agent action types (high-level automation commands)
+"""
+
+from enum import Enum
+
+
+class SentienceMethod(str, Enum):
+    """
+    Enum for window.sentience API methods.
+
+    These are the actual methods available on the window.sentience object
+    injected by the Chrome extension.
+    """
+
+    # Core snapshot and element discovery
+    SNAPSHOT = "snapshot"
+    """Take a snapshot of the current page with element geometry and metadata."""
+
+    # Element interaction
+    CLICK = "click"
+    """Click an element by its ID from the snapshot registry."""
+
+    # Content extraction
+    READ = "read"
+    """Read page content as raw HTML, text, or markdown."""
+
+    FIND_TEXT_RECT = "findTextRect"
+    """Find exact pixel coordinates of text occurrences on the page."""
+
+    # Visual overlay
+    SHOW_OVERLAY = "showOverlay"
+    """Show visual overlay highlighting elements with importance scores."""
+
+    CLEAR_OVERLAY = "clearOverlay"
+    """Clear the visual overlay."""
+
+    # Developer tools
+    START_RECORDING = "startRecording"
+    """Start recording mode for golden set collection (developer tool)."""
+
+    def __str__(self) -> str:
+        """Return the method name as a string."""
+        return self.value
+
+
+class AgentAction(str, Enum):
+    """
+    Enum for high-level agent action types.
+
+    These are the action commands that agents can execute. They may use
+    one or more window.sentience methods or Playwright APIs directly.
+    """
+
+    # Element interaction
+    CLICK = "click"
+    """Click an element by ID. Uses window.sentience.click() or Playwright mouse.click()."""
+
+    TYPE = "type"
+    """Type text into an input element. Uses Playwright keyboard.type() directly."""
+
+    PRESS = "press"
+    """Press a keyboard key (Enter, Escape, Tab, etc.). Uses Playwright keyboard.press()."""
+
+    # Navigation
+    NAVIGATE = "navigate"
+    """Navigate to a URL. Uses Playwright page.goto() directly."""
+
+    SCROLL = "scroll"
+    """Scroll the page or an element. Uses Playwright page.mouse.wheel() or element.scrollIntoView()."""
+
+    # Completion
+    FINISH = "finish"
+    """Signal that the agent task is complete. No browser action, just status update."""
+
+    # Wait/verification
+    WAIT = "wait"
+    """Wait for a condition or duration. Uses Playwright wait_for_* methods."""
+
+    def __str__(self) -> str:
+        """Return the action name as a string."""
+        return self.value