sentienceapi 0.92.2__py3-none-any.whl → 0.98.0__py3-none-any.whl

sentience/actions.py

@@ -4,10 +4,12 @@ from typing import Optional
 Actions v1 - click, type, press
 """
 
+import asyncio
 import time
 
 from .browser import AsyncSentienceBrowser, SentienceBrowser
 from .browser_evaluator import BrowserEvaluator
+from .cursor_policy import CursorPolicy, build_human_cursor_path
 from .models import ActionResult, BBox, Snapshot
 from .sentience_methods import SentienceMethod
 from .snapshot import snapshot, snapshot_async
@@ -18,6 +20,7 @@ def click(  # noqa: C901
     element_id: int,
     use_mouse: bool = True,
     take_snapshot: bool = False,
+    cursor_policy: CursorPolicy | None = None,
 ) -> ActionResult:
     """
     Click an element by ID using hybrid approach (mouse simulation by default)
@@ -37,6 +40,7 @@ def click(  # noqa: C901
 
     start_time = time.time()
     url_before = browser.page.url
+    cursor_meta: dict | None = None
 
     if use_mouse:
         # Hybrid approach: Get element bbox from snapshot, calculate center, use mouse.click()
@@ -52,9 +56,49 @@ def click(  # noqa: C901
                 # Calculate center of element bbox
                 center_x = element.bbox.x + element.bbox.width / 2
                 center_y = element.bbox.y + element.bbox.height / 2
-                # Use Playwright's native mouse click for realistic simulation
+                # Optional: human-like cursor movement (opt-in)
                 try:
-                    browser.page.mouse.click(center_x, center_y)
+                    if cursor_policy is not None and cursor_policy.mode == "human":
+                        # Best-effort cursor state on browser instance
+                        pos = getattr(browser, "_sentience_cursor_pos", None)
+                        if not isinstance(pos, tuple) or len(pos) != 2:
+                            try:
+                                vp = browser.page.viewport_size or {}
+                                pos = (
+                                    float(vp.get("width", 0)) / 2.0,
+                                    float(vp.get("height", 0)) / 2.0,
+                                )
+                            except Exception:
+                                pos = (0.0, 0.0)
+
+                        cursor_meta = build_human_cursor_path(
+                            start=(float(pos[0]), float(pos[1])),
+                            target=(float(center_x), float(center_y)),
+                            policy=cursor_policy,
+                        )
+                        pts = cursor_meta.get("path", [])
+                        steps = int(cursor_meta.get("steps") or max(1, len(pts)))
+                        duration_ms = int(cursor_meta.get("duration_ms") or 0)
+                        per_step_s = (
+                            (duration_ms / max(1, len(pts))) / 1000.0 if duration_ms > 0 else 0.0
+                        )
+                        for p in pts:
+                            browser.page.mouse.move(float(p["x"]), float(p["y"]))
+                            if per_step_s > 0:
+                                time.sleep(per_step_s)
+                        pause_ms = int(cursor_meta.get("pause_before_click_ms") or 0)
+                        if pause_ms > 0:
+                            time.sleep(pause_ms / 1000.0)
+                        browser.page.mouse.click(center_x, center_y)
+                        setattr(
+                            browser, "_sentience_cursor_pos", (float(center_x), float(center_y))
+                        )
+                    else:
+                        # Default behavior (no regression)
+                        browser.page.mouse.click(center_x, center_y)
+                        setattr(
+                            browser, "_sentience_cursor_pos", (float(center_x), float(center_y))
+                        )
                     success = True
                 except Exception:
                     # If navigation happens, mouse.click might fail, but that's OK
@@ -122,6 +166,7 @@ def click(  # noqa: C901
         outcome=outcome,
         url_changed=url_changed,
         snapshot_after=snapshot_after,
+        cursor=cursor_meta,
         error=(
             None
             if success
@@ -134,7 +179,11 @@ def click(  # noqa: C901
 
 
 def type_text(
-    browser: SentienceBrowser, element_id: int, text: str, take_snapshot: bool = False
+    browser: SentienceBrowser,
+    element_id: int,
+    text: str,
+    take_snapshot: bool = False,
+    delay_ms: float = 0,
 ) -> ActionResult:
     """
     Type text into an element (focus then input)
@@ -144,9 +193,16 @@ def type_text(
         element_id: Element ID from snapshot
         text: Text to type
         take_snapshot: Whether to take snapshot after action
+        delay_ms: Delay between keystrokes in milliseconds for human-like typing (default: 0)
 
     Returns:
         ActionResult
+
+    Example:
+        >>> # Type instantly (default behavior)
+        >>> type_text(browser, element_id, "Hello World")
+        >>> # Type with human-like delay (~10ms between keystrokes)
+        >>> type_text(browser, element_id, "Hello World", delay_ms=10)
     """
     if not browser.page:
         raise RuntimeError("Browser not started. Call browser.start() first.")
@@ -177,8 +233,8 @@ def type_text(
             error={"code": "focus_failed", "reason": "Element not found"},
         )
 
-    # Type using Playwright keyboard
-    browser.page.keyboard.type(text)
+    # Type using Playwright keyboard with optional delay between keystrokes
+    browser.page.keyboard.type(text, delay=delay_ms)
 
     duration_ms = int((time.time() - start_time) * 1000)
     url_after = browser.page.url
@@ -242,6 +298,94 @@ def press(browser: SentienceBrowser, key: str, take_snapshot: bool = False) -> A
     )
 
 
+def scroll_to(
+    browser: SentienceBrowser,
+    element_id: int,
+    behavior: str = "smooth",
+    block: str = "center",
+    take_snapshot: bool = False,
+) -> ActionResult:
+    """
+    Scroll an element into view
+
+    Scrolls the page so that the specified element is visible in the viewport.
+    Uses the element registry to find the element and scrollIntoView() to scroll it.
+
+    Args:
+        browser: SentienceBrowser instance
+        element_id: Element ID from snapshot to scroll into view
+        behavior: Scroll behavior - 'smooth', 'instant', or 'auto' (default: 'smooth')
+        block: Vertical alignment - 'start', 'center', 'end', or 'nearest' (default: 'center')
+        take_snapshot: Whether to take snapshot after action
+
+    Returns:
+        ActionResult
+
+    Example:
+        >>> snap = snapshot(browser)
+        >>> button = find(snap, 'role=button[name="Submit"]')
+        >>> if button:
+        >>>     # Scroll element into view with smooth animation
+        >>>     scroll_to(browser, button.id)
+        >>>     # Scroll instantly to top of viewport
+        >>>     scroll_to(browser, button.id, behavior='instant', block='start')
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    # Scroll element into view using the element registry
+    scrolled = browser.page.evaluate(
+        """
+        (args) => {
+            const el = window.sentience_registry[args.id];
+            if (el && el.scrollIntoView) {
+                el.scrollIntoView({
+                    behavior: args.behavior,
+                    block: args.block,
+                    inline: 'nearest'
+                });
+                return true;
+            }
+            return false;
+        }
+        """,
+        {"id": element_id, "behavior": behavior, "block": block},
+    )
+
+    if not scrolled:
+        return ActionResult(
+            success=False,
+            duration_ms=int((time.time() - start_time) * 1000),
+            outcome="error",
+            error={"code": "scroll_failed", "reason": "Element not found or not scrollable"},
+        )
+
+    # Wait a bit for scroll to complete (especially for smooth scrolling)
+    wait_time = 500 if behavior == "smooth" else 100
+    browser.page.wait_for_timeout(wait_time)
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = snapshot(browser)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
 def _highlight_rect(
     browser: SentienceBrowser, rect: dict[str, float], duration_sec: float = 2.0
 ) -> None:
@@ -315,6 +459,7 @@ def click_rect(
     highlight: bool = True,
     highlight_duration: float = 2.0,
     take_snapshot: bool = False,
+    cursor_policy: CursorPolicy | None = None,
 ) -> ActionResult:
     """
     Click at the center of a rectangle using Playwright's native mouse simulation.
@@ -370,6 +515,7 @@ def click_rect(
     # Calculate center of rectangle
     center_x = x + w / 2
     center_y = y + h / 2
+    cursor_meta: dict | None = None
 
     # Show highlight before clicking (if enabled)
     if highlight:
@@ -380,7 +526,35 @@ def click_rect(
     # Use Playwright's native mouse click for realistic simulation
     # This triggers hover, focus, mousedown, mouseup sequences
     try:
+        if cursor_policy is not None and cursor_policy.mode == "human":
+            pos = getattr(browser, "_sentience_cursor_pos", None)
+            if not isinstance(pos, tuple) or len(pos) != 2:
+                try:
+                    vp = browser.page.viewport_size or {}
+                    pos = (float(vp.get("width", 0)) / 2.0, float(vp.get("height", 0)) / 2.0)
+                except Exception:
+                    pos = (0.0, 0.0)
+
+            cursor_meta = build_human_cursor_path(
+                start=(float(pos[0]), float(pos[1])),
+                target=(float(center_x), float(center_y)),
+                policy=cursor_policy,
+            )
+            pts = cursor_meta.get("path", [])
+            duration_ms_move = int(cursor_meta.get("duration_ms") or 0)
+            per_step_s = (
+                (duration_ms_move / max(1, len(pts))) / 1000.0 if duration_ms_move > 0 else 0.0
+            )
+            for p in pts:
+                browser.page.mouse.move(float(p["x"]), float(p["y"]))
+                if per_step_s > 0:
+                    time.sleep(per_step_s)
+            pause_ms = int(cursor_meta.get("pause_before_click_ms") or 0)
+            if pause_ms > 0:
+                time.sleep(pause_ms / 1000.0)
+
         browser.page.mouse.click(center_x, center_y)
+        setattr(browser, "_sentience_cursor_pos", (float(center_x), float(center_y)))
         success = True
     except Exception as e:
         success = False
@@ -413,6 +587,7 @@ def click_rect(
         outcome=outcome,
         url_changed=url_changed,
         snapshot_after=snapshot_after,
+        cursor=cursor_meta,
         error=(
             None
             if success
@@ -432,6 +607,7 @@ async def click_async(
     element_id: int,
     use_mouse: bool = True,
     take_snapshot: bool = False,
+    cursor_policy: CursorPolicy | None = None,
 ) -> ActionResult:
     """
     Click an element by ID using hybrid approach (async)
@@ -450,6 +626,7 @@ async def click_async(
 
     start_time = time.time()
     url_before = browser.page.url
+    cursor_meta: dict | None = None
 
     if use_mouse:
         try:
@@ -464,7 +641,44 @@ async def click_async(
                 center_x = element.bbox.x + element.bbox.width / 2
                 center_y = element.bbox.y + element.bbox.height / 2
                 try:
-                    await browser.page.mouse.click(center_x, center_y)
+                    if cursor_policy is not None and cursor_policy.mode == "human":
+                        pos = getattr(browser, "_sentience_cursor_pos", None)
+                        if not isinstance(pos, tuple) or len(pos) != 2:
+                            try:
+                                vp = browser.page.viewport_size or {}
+                                pos = (
+                                    float(vp.get("width", 0)) / 2.0,
+                                    float(vp.get("height", 0)) / 2.0,
+                                )
+                            except Exception:
+                                pos = (0.0, 0.0)
+
+                        cursor_meta = build_human_cursor_path(
+                            start=(float(pos[0]), float(pos[1])),
+                            target=(float(center_x), float(center_y)),
+                            policy=cursor_policy,
+                        )
+                        pts = cursor_meta.get("path", [])
+                        duration_ms = int(cursor_meta.get("duration_ms") or 0)
+                        per_step_s = (
+                            (duration_ms / max(1, len(pts))) / 1000.0 if duration_ms > 0 else 0.0
+                        )
+                        for p in pts:
+                            await browser.page.mouse.move(float(p["x"]), float(p["y"]))
+                            if per_step_s > 0:
+                                await asyncio.sleep(per_step_s)
+                        pause_ms = int(cursor_meta.get("pause_before_click_ms") or 0)
+                        if pause_ms > 0:
+                            await asyncio.sleep(pause_ms / 1000.0)
+                        await browser.page.mouse.click(center_x, center_y)
+                        setattr(
+                            browser, "_sentience_cursor_pos", (float(center_x), float(center_y))
+                        )
+                    else:
+                        await browser.page.mouse.click(center_x, center_y)
+                        setattr(
+                            browser, "_sentience_cursor_pos", (float(center_x), float(center_y))
+                        )
                     success = True
                 except Exception:
                     success = True
@@ -541,6 +755,7 @@ async def click_async(
         outcome=outcome,
         url_changed=url_changed,
         snapshot_after=snapshot_after,
+        cursor=cursor_meta,
         error=(
             None
             if success
@@ -553,7 +768,11 @@ async def click_async(
 
 
 async def type_text_async(
-    browser: AsyncSentienceBrowser, element_id: int, text: str, take_snapshot: bool = False
+    browser: AsyncSentienceBrowser,
+    element_id: int,
+    text: str,
+    take_snapshot: bool = False,
+    delay_ms: float = 0,
 ) -> ActionResult:
     """
     Type text into an element (async)
@@ -563,9 +782,16 @@ async def type_text_async(
         element_id: Element ID from snapshot
         text: Text to type
         take_snapshot: Whether to take snapshot after action
+        delay_ms: Delay between keystrokes in milliseconds for human-like typing (default: 0)
 
     Returns:
         ActionResult
+
+    Example:
+        >>> # Type instantly (default behavior)
+        >>> await type_text_async(browser, element_id, "Hello World")
+        >>> # Type with human-like delay (~10ms between keystrokes)
+        >>> await type_text_async(browser, element_id, "Hello World", delay_ms=10)
     """
     if not browser.page:
         raise RuntimeError("Browser not started. Call await browser.start() first.")
@@ -596,8 +822,8 @@ async def type_text_async(
             error={"code": "focus_failed", "reason": "Element not found"},
         )
 
-    # Type using Playwright keyboard
-    await browser.page.keyboard.type(text)
+    # Type using Playwright keyboard with optional delay between keystrokes
+    await browser.page.keyboard.type(text, delay=delay_ms)
 
     duration_ms = int((time.time() - start_time) * 1000)
     url_after = browser.page.url
@@ -663,6 +889,94 @@ async def press_async(
     )
 
 
+async def scroll_to_async(
+    browser: AsyncSentienceBrowser,
+    element_id: int,
+    behavior: str = "smooth",
+    block: str = "center",
+    take_snapshot: bool = False,
+) -> ActionResult:
+    """
+    Scroll an element into view (async)
+
+    Scrolls the page so that the specified element is visible in the viewport.
+    Uses the element registry to find the element and scrollIntoView() to scroll it.
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        element_id: Element ID from snapshot to scroll into view
+        behavior: Scroll behavior - 'smooth', 'instant', or 'auto' (default: 'smooth')
+        block: Vertical alignment - 'start', 'center', 'end', or 'nearest' (default: 'center')
+        take_snapshot: Whether to take snapshot after action
+
+    Returns:
+        ActionResult
+
+    Example:
+        >>> snap = await snapshot_async(browser)
+        >>> button = find(snap, 'role=button[name="Submit"]')
+        >>> if button:
+        >>>     # Scroll element into view with smooth animation
+        >>>     await scroll_to_async(browser, button.id)
+        >>>     # Scroll instantly to top of viewport
+        >>>     await scroll_to_async(browser, button.id, behavior='instant', block='start')
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    # Scroll element into view using the element registry
+    scrolled = await browser.page.evaluate(
+        """
+        (args) => {
+            const el = window.sentience_registry[args.id];
+            if (el && el.scrollIntoView) {
+                el.scrollIntoView({
+                    behavior: args.behavior,
+                    block: args.block,
+                    inline: 'nearest'
+                });
+                return true;
+            }
+            return false;
+        }
+        """,
+        {"id": element_id, "behavior": behavior, "block": block},
+    )
+
+    if not scrolled:
+        return ActionResult(
+            success=False,
+            duration_ms=int((time.time() - start_time) * 1000),
+            outcome="error",
+            error={"code": "scroll_failed", "reason": "Element not found or not scrollable"},
+        )
+
+    # Wait a bit for scroll to complete (especially for smooth scrolling)
+    wait_time = 500 if behavior == "smooth" else 100
+    await browser.page.wait_for_timeout(wait_time)
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = await snapshot_async(browser)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
 async def _highlight_rect_async(
     browser: AsyncSentienceBrowser, rect: dict[str, float], duration_sec: float = 2.0
 ) -> None:
@@ -724,6 +1038,7 @@ async def click_rect_async(
     highlight: bool = True,
     highlight_duration: float = 2.0,
     take_snapshot: bool = False,
+    cursor_policy: CursorPolicy | None = None,
 ) -> ActionResult:
     """
     Click at the center of a rectangle (async)
@@ -770,6 +1085,7 @@ async def click_rect_async(
     # Calculate center of rectangle
     center_x = x + w / 2
     center_y = y + h / 2
+    cursor_meta: dict | None = None
 
     # Show highlight before clicking
     if highlight:
@@ -778,7 +1094,35 @@ async def click_rect_async(
 
     # Use Playwright's native mouse click
     try:
+        if cursor_policy is not None and cursor_policy.mode == "human":
+            pos = getattr(browser, "_sentience_cursor_pos", None)
+            if not isinstance(pos, tuple) or len(pos) != 2:
+                try:
+                    vp = browser.page.viewport_size or {}
+                    pos = (float(vp.get("width", 0)) / 2.0, float(vp.get("height", 0)) / 2.0)
+                except Exception:
+                    pos = (0.0, 0.0)
+
+            cursor_meta = build_human_cursor_path(
+                start=(float(pos[0]), float(pos[1])),
+                target=(float(center_x), float(center_y)),
+                policy=cursor_policy,
+            )
+            pts = cursor_meta.get("path", [])
+            duration_ms_move = int(cursor_meta.get("duration_ms") or 0)
+            per_step_s = (
+                (duration_ms_move / max(1, len(pts))) / 1000.0 if duration_ms_move > 0 else 0.0
+            )
+            for p in pts:
+                await browser.page.mouse.move(float(p["x"]), float(p["y"]))
+                if per_step_s > 0:
+                    await asyncio.sleep(per_step_s)
+            pause_ms = int(cursor_meta.get("pause_before_click_ms") or 0)
+            if pause_ms > 0:
+                await asyncio.sleep(pause_ms / 1000.0)
+
         await browser.page.mouse.click(center_x, center_y)
+        setattr(browser, "_sentience_cursor_pos", (float(center_x), float(center_y)))
         success = True
     except Exception as e:
         success = False
@@ -811,6 +1155,7 @@ async def click_rect_async(
         outcome=outcome,
         url_changed=url_changed,
         snapshot_after=snapshot_after,
+        cursor=cursor_meta,
         error=(
             None
             if success

sentience/agent.py

@@ -355,6 +355,7 @@ class SentienceAgent(BaseAgent):
                     url_changed=result_dict.get("url_changed"),
                     error=result_dict.get("error"),
                     message=result_dict.get("message"),
+                    cursor=result_dict.get("cursor"),
                 )
 
                 # Emit action execution trace event if tracer is enabled
@@ -391,6 +392,7 @@ class SentienceAgent(BaseAgent):
                             "post_url": post_url,
                             "elements": elements_data,  # Add element data for overlay
                             "target_element_id": result.element_id,  # Highlight target in red
+                            "cursor": result.cursor,
                         },
                         step_id=step_id,
                     )
@@ -445,6 +447,8 @@ class SentienceAgent(BaseAgent):
                         ),
                         "duration_ms": duration_ms,
                     }
+                    if result.cursor is not None:
+                        exec_data["cursor"] = result.cursor
 
                     # Add optional exec fields
                     if result.element_id is not None: