browserrig 0.1.0__py3-none-any.whl

browserrig/tools/javascript.py

@@ -0,0 +1,37 @@
+from typing import Annotated
+
+from harnessed import Inject, Plugin, ToolResult
+from pydantic import Field
+
+from browserrig.browser import Tab
+from browserrig.core import ActionKind
+from browserrig.core import browser_tool as tool
+
+from .effects import Effect
+from .execution import Actions
+
+
+class JavaScriptTools(Plugin, needs=(Actions,)):
+    @tool(
+        kind=ActionKind.JAVASCRIPT,
+        description=(
+            "Execute JavaScript in the active tab's main page context via Runtime.evaluate. "
+            "Supports top-level await and retains variables until navigation. Use await for "
+            "Promise results. Returns the last "
+            "expression's value as JSON, or a JavaScript value such as undefined or NaN. "
+            "Return serializable data when inspecting objects. Errors are reported in the tool "
+            "result. timeout is in seconds."
+        ),
+    )
+    async def execute_javascript(
+        self,
+        code: str,
+        timeout: Annotated[float, Field(gt=0, le=30)] = 10,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def execute(tab: Tab) -> Effect:
+            result = await tab.evaluate(code, timeout=timeout)
+            return Effect(f"JavaScript result:\n{result}")
+
+        return await actions.run(kind=ActionKind.JAVASCRIPT, operation=execute)

browserrig/tools/keyboard.py

@@ -0,0 +1,126 @@
+from harnessed import Inject, Plugin, ToolError, ToolResult
+from harnessed.kernel import dependency
+
+from browserrig.browser import Browser, Tab
+from browserrig.core import ActionKind
+from browserrig.core import browser_tool as tool
+from browserrig.input import TypedText
+from browserrig.observation import Observe
+from browserrig.secrets import Secrets
+
+from .effects import Effect
+from .execution import Actions
+
+
+class KeyboardTools(Plugin, needs=(Actions,)):
+    """Keyboard input to the current focus, plus typing into a DOM ref when refs exist."""
+
+    _browser = dependency(Browser)
+    _observe = dependency(Observe)
+
+    @tool(
+        kind=ActionKind.PRESS,
+        chainable=True,
+        description=(
+            "Send keyboard input to the current page focus. Provide exactly one of "
+            "text for literal text, or key for one character or a named key such as "
+            "Escape, Enter or Tab. Use hold_control, hold_shift, hold_alt and hold_meta "
+            "for shortcuts, e.g. key='a', hold_control=true selects all."
+        ),
+    )
+    async def send_keys(
+        self,
+        *,
+        text: str | None = None,
+        key: str | None = None,
+        hold_control: bool = False,
+        hold_shift: bool = False,
+        hold_alt: bool = False,
+        hold_meta: bool = False,
+        actions: Inject[Actions],
+        secrets: Inject[Secrets | None] = None,
+    ) -> ToolResult:
+        if (text is None) == (key is None):
+            raise ToolError("Provide exactly one of text or key")
+        if text is not None and any((hold_control, hold_shift, hold_alt, hold_meta)):
+            raise ToolError("Modifier flags require key")
+
+        async def send(_tab: Tab) -> Effect:
+            if key is not None:
+                await self._browser.keyboard.press_key(
+                    key,
+                    hold_control=hold_control,
+                    hold_shift=hold_shift,
+                    hold_alt=hold_alt,
+                    hold_meta=hold_meta,
+                )
+                return Effect("Key dispatched.")
+            assert text is not None
+            if secrets is not None:
+                entry = await secrets.prepare(text, browser=self._browser)
+                if entry is not None:
+                    return _secret_entered(
+                        await secrets.enter(entry, self._browser.keyboard)
+                    )
+            return _entered(await self._browser.keyboard.type_text(text))
+
+        return await actions.run(kind=ActionKind.PRESS, operation=send)
+
+    @tool(
+        kind=ActionKind.TYPE,
+        chainable=True,
+        description="Type exact text into a DOM ref; clear replaces its value by default.",
+    )
+    async def type_text(
+        self,
+        ref: str,
+        text: str,
+        clear: bool = True,
+        *,
+        actions: Inject[Actions],
+        secrets: Inject[Secrets | None] = None,
+    ) -> ToolResult:
+        async def check(tab: Tab) -> None:
+            await self._observe.dom_element(tab, ref)
+
+        async def type_text(tab: Tab) -> Effect:
+            point = await actions.element_point(tab=tab, ref=ref)
+            await self._browser.mouse.move_to(point)
+            # Hover may reveal an overlay over the field; hit-test before focusing it.
+            await actions.element_point(tab=tab, ref=ref, scroll=False)
+            field = await self._observe.dom_element(tab, ref)
+            # Check the secrets' domains before the field is touched.
+            entry = (
+                await secrets.prepare(text, browser=self._browser, field=field)
+                if secrets is not None
+                else None
+            )
+            await field.focus(select_all=clear)
+            if clear:
+                await self._browser.keyboard.press_key("Backspace")
+            if entry is not None and secrets is not None:
+                return _secret_entered(
+                    await secrets.enter(entry, self._browser.keyboard)
+                )
+            return _entered(await self._browser.keyboard.type_text(text))
+
+        return await actions.run(kind=ActionKind.TYPE, operation=type_text, check=check)
+
+
+def _entered(typed: TypedText) -> Effect:
+    if typed.exact:
+        return Effect("Text entered.")
+    return Effect(
+        f"Text entered with a slip: typed {typed.entered!r} instead of "
+        f"{typed.intended!r}. Check the field before continuing."
+    )
+
+
+def _secret_entered(exact: bool) -> Effect:
+    if exact:
+        return Effect("Text entered.")
+    # Never echo text that surrounds a secret.
+    return Effect(
+        "Text entered with a slip in the text around the secret. "
+        "Check the field before continuing."
+    )

browserrig/tools/mouse.py

@@ -0,0 +1,224 @@
+from typing import Annotated, Literal
+
+from harnessed import Inject, Instructions, Plugin, SystemPrompt, ToolResult
+from harnessed.kernel import dependency
+from pydantic import Field
+
+from browserrig.browser import Browser, Tab
+from browserrig.core import ActionKind, BrowserError, Point
+from browserrig.core import browser_tool as tool
+from browserrig.observation import CapturedImage, Observe, Snapshot
+
+from .effects import Effect
+from .execution import Actions
+
+type _ClickCount = Annotated[
+    Literal[1, 2],
+    Field(description="2 = native double click (open, edit, select word)"),
+]
+type _ClickButton = Annotated[
+    Literal["left", "right"], Field(description="Mouse button to press.")
+]
+
+
+_INSTRUCTIONS = Instructions(
+    title="Screenshot coordinates",
+    text=(
+        "Coordinates are pixels from the top-left of the newest shown image for this document; "
+        "region and canvas images are mapped back to the viewport automatically. "
+        "A red cursor marker shows the pointer position when the screenshot was captured; "
+        "it is omitted when outside the image. "
+        "Observe again if the tab or document changes before pointing."
+    ),
+)
+
+
+class PointerTools(Plugin, needs=(Actions,)):
+    """Mouse input by screenshot coordinates, plus by observed ref when refs exist.
+
+    Coordinates also reach unreferenced regions, so text-only agents keep them too.
+    """
+
+    _browser = dependency(Browser)
+    _observe = dependency(Observe)
+
+    def instructions(self) -> SystemPrompt:
+        return SystemPrompt(_INSTRUCTIONS)
+
+    @tool(
+        kind=ActionKind.MOVE_MOUSE,
+        description=(
+            "Move the cursor to image or viewport coordinates without clicking. "
+            "Use before scroll to target a particular scrollable area, or to hover."
+        ),
+    )
+    async def move_mouse(
+        self,
+        x: Annotated[float, Field(ge=0)],
+        y: Annotated[float, Field(ge=0)],
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def move(_tab: Tab) -> Effect:
+            point = _image_point(self._observe.coordinate_source(), Point(x=x, y=y))
+            await self._browser.mouse.move_to(point)
+            return Effect("Cursor moved without clicking.")
+
+        return await actions.run(kind=ActionKind.MOVE_MOUSE, operation=move)
+
+    @tool(
+        kind=ActionKind.CLICK_AT,
+        description=(
+            "Automatically move the cursor to image or viewport coordinates and click there; "
+            "count=2 performs a native double click; button=right performs a right click."
+        ),
+    )
+    async def click_at(
+        self,
+        x: Annotated[float, Field(ge=0)],
+        y: Annotated[float, Field(ge=0)],
+        count: _ClickCount = 1,
+        button: _ClickButton = "left",
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def click(_tab: Tab) -> Effect:
+            point = _image_point(self._observe.coordinate_source(), Point(x=x, y=y))
+            await self._browser.mouse.click(point, count=count, button=button)
+            return Effect(_click_message(count, button), break_chain=button == "right")
+
+        return await actions.run(
+            kind=ActionKind.CLICK_AT, operation=click, expects_change=True
+        )
+
+    @tool(
+        kind=ActionKind.SCROLL,
+        description=(
+            "Scroll vertically by CSS pixels at the current cursor position. "
+            "Use move_mouse first to target a particular scrollable area."
+        ),
+    )
+    async def scroll(
+        self,
+        delta_y: Annotated[float, Field(ge=-10000, le=10000)],
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def scroll(_tab: Tab) -> Effect:
+            await self._browser.mouse.scroll(delta_y=delta_y)
+            return Effect("Scroll dispatched.")
+
+        return await actions.run(
+            kind=ActionKind.SCROLL,
+            operation=scroll,
+        )
+
+    @tool(
+        kind=ActionKind.DRAG_AT,
+        description="Drag between screenshot points.",
+    )
+    async def drag_at(
+        self,
+        start: Point,
+        end: Point,
+        duration: Annotated[float, Field(ge=0.05, le=5)] = 0.5,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def drag(_tab: Tab) -> Effect:
+            await self._browser.mouse.drag(
+                start=_image_point(self._observe.coordinate_source(), start),
+                end=_image_point(self._observe.coordinate_source(), end),
+                duration=duration,
+            )
+            return Effect("Drag dispatched; inspect the destination.")
+
+        return await actions.run(
+            kind=ActionKind.DRAG_AT, operation=drag, expects_change=True
+        )
+
+    @tool(
+        kind=ActionKind.CLICK,
+        chainable=True,
+        description=(
+            "Automatically move the cursor to a current observed ref and click it; "
+            "count=2 performs a native double click; "
+            "button=right performs a right click."
+        ),
+    )
+    async def click(
+        self,
+        ref: str,
+        count: _ClickCount = 1,
+        button: _ClickButton = "left",
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def click(tab: Tab) -> Effect:
+            point = await actions.element_point(tab=tab, ref=ref)
+            await self._browser.mouse.move_to(point)
+            if change := await actions.page_change(tab, after_input=False):
+                return Effect(
+                    f"Hover changed the page: {change} The click was not dispatched. "
+                    "Inspect the next observation before choosing a target.",
+                    prepared_scroll=point.scrolled,
+                    break_chain=True,
+                )
+            # Hover may reveal an overlay or move the target; hit-test again.
+            current = await actions.element_point(tab=tab, ref=ref, scroll=False)
+            await self._browser.mouse.click(current, count=count, button=button)
+            return Effect(
+                _click_message(count, button),
+                prepared_scroll=point.scrolled,
+                break_chain=button == "right",
+            )
+
+        return await actions.run(
+            kind=ActionKind.CLICK, operation=click, expects_change=True
+        )
+
+    @tool(
+        kind=ActionKind.DRAG,
+        description=(
+            "Drag between observed refs, scrolling them into view when supported. "
+            "Both endpoints must fit in the viewport together."
+        ),
+    )
+    async def drag(
+        self,
+        source_ref: str,
+        target_ref: str,
+        duration: Annotated[float, Field(ge=0.05, le=5)] = 0.5,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def drag(tab: Tab) -> Effect:
+            prepared = await actions.element_point(tab=tab, ref=source_ref)
+            target = await actions.element_point(tab=tab, ref=target_ref)
+            start = await actions.element_point(tab=tab, ref=source_ref, scroll=False)
+            end = await actions.element_point(tab=tab, ref=target_ref, scroll=False)
+            await self._browser.mouse.drag(start=start, end=end, duration=duration)
+            return Effect(
+                "Drag dispatched; inspect the destination.",
+                prepared_scroll=prepared.scrolled or target.scrolled,
+            )
+
+        return await actions.run(
+            kind=ActionKind.DRAG, operation=drag, expects_change=True
+        )
+
+
+def _click_message(count: Literal[1, 2], button: Literal["left", "right"]) -> str:
+    gesture = "right click" if button == "right" else "click"
+    if count == 2:
+        gesture = "double " + gesture
+    return gesture.capitalize() + " dispatched."
+
+
+def _image_point(snapshot: Snapshot | CapturedImage, point: Point) -> Point:
+    if snapshot.geometry is not None:
+        x, y = snapshot.geometry.point(point.x, point.y)
+        return Point(x=x, y=y)
+    if point.x >= snapshot.width or point.y >= snapshot.height:
+        raise BrowserError("Point is outside the observed viewport")
+    return point

browserrig/tools/navigation.py

@@ -0,0 +1,189 @@
+import asyncio
+from typing import Annotated
+from urllib.parse import quote_plus, urljoin, urlsplit
+
+from harnessed import Inject, Plugin, ToolResult
+from harnessed.kernel import dependency
+from pydantic import Field
+
+from browserrig.browser import Browser, BrowserEvents, Tab, validated_url
+from browserrig.core import ActionKind, BrowserError, ReadinessPhase
+from browserrig.core import browser_tool as tool
+from browserrig.events import NavigationStartedEvent
+from browserrig.readiness import Readiness
+
+from .effects import Effect
+from .execution import Actions
+from .intents import NavigateIntent, WaitIntent
+
+# Where a new tab lands when the agent gives no URL.
+NEW_TAB_URL = "https://duckduckgo.com/"
+
+
+class NavigationTools(Plugin, needs=(Actions,)):
+    _browser = dependency(Browser)
+    _events = dependency(BrowserEvents)
+    _readiness = dependency(Readiness)
+
+    @tool(
+        kind=ActionKind.NAVIGATE,
+        description=(
+            "Navigate to an HTTP(S) URL in the active tab, or in a new tab with new_tab=true. "
+            "A path starting with / opens on the active page's site."
+        ),
+    )
+    async def navigate(
+        self,
+        url: str,
+        new_tab: bool = False,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        url = site_url(url, self._browser.active_tab.url)
+        return await self._navigate(
+            url, actions, ActionKind.NAVIGATE, typed=typed_address(url), new_tab=new_tab
+        )
+
+    @tool(
+        kind=ActionKind.SEARCH,
+        description="Search the web with DuckDuckGo in the active tab.",
+    )
+    async def search(
+        self,
+        query: str,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        return await self._navigate(
+            f"https://duckduckgo.com/?q={quote_plus(query)}",
+            actions,
+            ActionKind.SEARCH,
+            typed=query,
+        )
+
+    @tool(
+        kind=ActionKind.GO_BACK, description="Go to the previous browser history entry."
+    )
+    async def go_back(self, *, actions: Inject[Actions]) -> ToolResult:
+        async def back(tab: Tab) -> Effect:
+            if not await tab.go_back():
+                raise BrowserError("No previous history entry")
+            return Effect("Navigated back.")
+
+        return await actions.run(
+            kind=ActionKind.GO_BACK,
+            operation=back,
+        )
+
+    @tool(
+        kind=ActionKind.GO_FORWARD,
+        description=(
+            "Go to the next browser history entry. Reports an error if forward navigation "
+            "is unavailable."
+        ),
+    )
+    async def go_forward(self, *, actions: Inject[Actions]) -> ToolResult:
+        async def check(tab: Tab) -> None:
+            if not await tab.can_go_forward():
+                raise BrowserError("Cannot go forward: no next history entry")
+
+        async def forward(tab: Tab) -> Effect:
+            if not await tab.go_forward():
+                raise BrowserError("Cannot go forward: no next history entry")
+            return Effect("Navigated forward.")
+
+        return await actions.run(
+            kind=ActionKind.GO_FORWARD,
+            operation=forward,
+            check=check,
+        )
+
+    @tool(kind=ActionKind.RELOAD, description="Reload the active tab.")
+    async def reload(self, *, actions: Inject[Actions]) -> ToolResult:
+        async def reload(tab: Tab) -> Effect:
+            await tab.reload()
+            return Effect("Reload executed.")
+
+        return await actions.run(
+            kind=ActionKind.RELOAD,
+            operation=reload,
+        )
+
+    @tool(
+        kind=ActionKind.WAIT,
+        readonly=True,
+        description=(
+            "Wait briefly for an expected asynchronous page update; the next observation shows it."
+        ),
+    )
+    async def wait(
+        self,
+        seconds: Annotated[float, Field(ge=0, le=10)] = 1,
+        *,
+        actions: Inject[Actions],
+    ) -> str:
+        async with self._browser.action_lock:
+
+            async def pause() -> None:
+                await asyncio.sleep(seconds)
+
+            await actions.stage(WaitIntent(seconds), self._browser.active_tab, pause)
+        return "Wait completed; the next observation shows the current state."
+
+    async def _navigate(
+        self,
+        url: str,
+        actions: Actions,
+        kind: ActionKind,
+        *,
+        typed: str,
+        new_tab: bool = False,
+    ) -> ToolResult:
+        validated_url(url)
+
+        async def navigate(tab: Tab) -> Effect:
+            async def load() -> None:
+                await self._events.emit(
+                    NavigationStartedEvent(target_id=tab.id, url=url, typed=typed)
+                )
+                await tab.navigate(url)
+
+            await actions.stage(NavigateIntent(url, typed), tab, load)
+            return Effect("Navigation executed.")
+
+        async def navigate_new(_tab: Tab) -> Effect:
+            # An empty tab first, so the address can be entered in a fresh tab.
+            opened = await self._browser.tabs.open()
+
+            async def dispatch() -> None:
+                await navigate(opened)
+
+            await self._readiness.wait(
+                opened, phase=ReadinessPhase.NAVIGATION, dispatch=dispatch
+            )
+            return Effect(f"Opened tab {opened.id} and navigated.")
+
+        if new_tab:
+            return await actions.run(kind=kind, operation=navigate_new, settle=False)
+        return await actions.run(kind=kind, operation=navigate)
+
+
+def site_url(  # smelt: ignore[SMT429] -- pure, table-tested
+    url: str, current: str
+) -> str:
+    """Resolve a site path such as ``/wiki/Page`` against the current HTTP(S) page."""
+    site_path = url.startswith("/") and not url.startswith("//")
+    if site_path and urlsplit(current).scheme in {"http", "https"}:
+        return urljoin(current, url)
+    return url
+
+
+def typed_address(url: str) -> str:  # smelt: ignore[SMT429] -- pure, table-tested
+    """What a person types into the address bar for `url`: no scheme, no lone trailing slash."""
+    parts = urlsplit(url)
+    if parts.scheme not in {"http", "https"}:
+        return url
+    typed = url.split("://", 1)[1]
+    if parts.path == "/" and not parts.query and not parts.fragment:
+        typed = typed.removesuffix("/")
+    return typed

browserrig/tools/scrolling.py

@@ -0,0 +1,39 @@
+import asyncio
+
+from browserrig.browser import Tab
+from browserrig.core import BrowserError, StaleElement
+from browserrig.input import Mouse
+from browserrig.observation import ElementHandle, ScrollableElement
+
+
+async def prepare_element(
+    tab: Tab, handle: ElementHandle, mouse: Mouse, *, document_id: str
+) -> bool:
+    """Execute a backend's wheel plan through normal mouse input, when supported."""
+    if not isinstance(handle, ScrollableElement):
+        return False
+    scrolled = False
+    previous = None
+    stalled = 0
+    async with asyncio.timeout(20):
+        for _ in range(32):
+            await tab.sync_document()
+            if tab.document_id != document_id:
+                raise StaleElement(
+                    "Document changed while scrolling; use refs from the next observation"
+                )
+            step = await handle.scroll_step()
+            if step is None:
+                return scrolled
+            stalled = stalled + 1 if step.key == previous else 0
+            if stalled >= 2:
+                break
+            previous = step.key
+            await mouse.scroll(
+                point=step.point, delta_x=step.delta_x, delta_y=step.delta_y
+            )
+            scrolled = True
+            await asyncio.sleep(0.05)
+    raise BrowserError(
+        "Element could not be brought into view using wheel input; try another way to reach it"
+    )

browserrig/tools/tabs.py

@@ -0,0 +1,81 @@
+from harnessed import Inject, Plugin, ToolResult
+from harnessed.kernel import dependency
+
+from browserrig.browser import Browser, Tab, TabInfo
+from browserrig.core import ActionKind, ReadinessPhase
+from browserrig.core import browser_tool as tool
+from browserrig.readiness import Readiness
+
+from .effects import Effect
+from .execution import Actions
+from .navigation import NEW_TAB_URL
+
+
+class TabTools(Plugin, needs=(Actions,)):
+    _browser = dependency(Browser)
+    _readiness = dependency(Readiness)
+
+    @tool(
+        kind=ActionKind.LIST_TABS,
+        chainable=True,
+        readonly=True,
+        description="List controlled tabs and identify the active tab.",
+    )
+    async def list_tabs(self) -> tuple[TabInfo, ...]:
+        return await self._browser.tabs.list()
+
+    @tool(
+        kind=ActionKind.OPEN_TAB,
+        description="Open and activate a new tab; without a URL it starts on DuckDuckGo.",
+    )
+    async def open_tab(
+        self,
+        url: str = NEW_TAB_URL,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def open_tab(_tab: Tab) -> Effect:
+            opened = await self._browser.tabs.open(url)
+            await self._readiness.wait(opened, phase=ReadinessPhase.NAVIGATION)
+            return Effect(f"Opened tab {opened.id}.")
+
+        return await actions.run(
+            kind=ActionKind.OPEN_TAB, operation=open_tab, settle=False
+        )
+
+    @tool(
+        kind=ActionKind.SWITCH_TAB,
+        description="Activate one listed tab by its stable target ID.",
+    )
+    async def switch_tab(
+        self,
+        target_id: str,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def switch(_tab: Tab) -> Effect:
+            selected = await self._browser.tabs.activate(target_id)
+            await self._readiness.wait(selected)
+            return Effect(f"Activated tab {selected.id}.")
+
+        return await actions.run(
+            kind=ActionKind.SWITCH_TAB, operation=switch, settle=False
+        )
+
+    @tool(
+        kind=ActionKind.CLOSE_TAB,
+        description="Close a controlled tab. A remaining tab is explicitly selected if needed.",
+    )
+    async def close_tab(
+        self,
+        target_id: str,
+        *,
+        actions: Inject[Actions],
+    ) -> ToolResult:
+        async def close(_tab: Tab) -> Effect:
+            selected = await self._browser.tabs.close(target_id)
+            return Effect(f"Tab closed. Active tab: {selected.id}.")
+
+        return await actions.run(
+            kind=ActionKind.CLOSE_TAB, operation=close, settle=False
+        )