inspect-ai 0.3.70__py3-none-any.whl → 0.3.72__py3-none-any.whl

inspect_ai/tool/_tools/_computer/test_args.py

@@ -0,0 +1,151 @@
+import pytest
+
+from ._resources.tool._args import parse_arguments
+
+
+def test_parse_args_screenshot() -> None:
+    args = parse_arguments(["screenshot"])
+    assert args.action == "screenshot"
+
+
+def test_parse_args_cursor_position() -> None:
+    args = parse_arguments(["cursor_position"])
+    assert args.action == "cursor_position"
+
+
+def test_parse_args_type() -> None:
+    args = parse_arguments(["type", "--text", "hello"])
+    assert args.action == "type"
+    assert args.text == "hello"
+
+
+def test_parse_args_mouse_move() -> None:
+    args = parse_arguments(["mouse_move", "--coordinate", "100", "200"])
+    assert args.action == "mouse_move"
+    assert args.coordinate == [100, 200]
+
+
+def test_parse_args_left_click() -> None:
+    args = parse_arguments(["left_click", "--coordinate", "100", "200"])
+    assert args.action == "left_click"
+    assert args.coordinate == [100, 200]
+
+
+def test_parse_args_right_click() -> None:
+    args = parse_arguments(["right_click", "--coordinate", "100", "200"])
+    assert args.action == "right_click"
+    assert args.coordinate == [100, 200]
+
+
+def test_parse_args_middle_click() -> None:
+    args = parse_arguments(["middle_click", "--coordinate", "100", "200"])
+    assert args.action == "middle_click"
+    assert args.coordinate == [100, 200]
+
+
+def test_parse_args_double_click() -> None:
+    args = parse_arguments(["double_click", "--coordinate", "100", "200"])
+    assert args.action == "double_click"
+    assert args.coordinate == [100, 200]
+
+
+def test_parse_args_triple_click() -> None:
+    args = parse_arguments(["triple_click", "--coordinate", "100", "200"])
+    assert args.action == "triple_click"
+    assert args.coordinate == [100, 200]
+
+
+def test_parse_args_hold_key() -> None:
+    args = parse_arguments(["hold_key", "--text", "a", "--duration", "5"])
+    assert args.action == "hold_key"
+    assert args.text == "a"
+    assert args.duration == 5
+
+
+def test_parse_args_left_click_drag() -> None:
+    args = parse_arguments(
+        [
+            "left_click_drag",
+            "--start_coordinate",
+            "100",
+            "200",
+            "--coordinate",
+            "300",
+            "400",
+            "--text",
+            "drag",
+        ]
+    )
+    assert args.action == "left_click_drag"
+    assert args.start_coordinate == [100, 200]
+    assert args.coordinate == [300, 400]
+    assert args.text == "drag"
+
+
+def test_parse_args_scroll() -> None:
+    args = parse_arguments(
+        [
+            "scroll",
+            "--scroll_direction",
+            "up",
+            "--scroll_amount",
+            "10",
+            "--coordinate",
+            "100",
+            "200",
+        ]
+    )
+    assert args.action == "scroll"
+    assert args.scroll_direction == "up"
+    assert args.scroll_amount == 10
+    assert args.coordinate == [100, 200]
+
+
+def test_parse_args_wait() -> None:
+    args = parse_arguments(["wait", "--duration", "5"])
+    assert args.action == "wait"
+    assert args.duration == 5
+
+
+def test_parse_args_type_missing_text() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(["type"])
+
+
+def test_parse_args_invalid_action() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(["invalid_action"])
+
+
+def test_parse_args_mouse_move_missing_coordinate() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(["mouse_move"])
+
+
+def test_parse_args_click_invalid_coordinate() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(["left_click", "--coordinate", "100"])
+
+
+def test_parse_args_hold_key_missing_duration() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(["hold_key", "--text", "a"])
+
+
+def test_parse_args_left_click_drag_missing_start_coordinate() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(
+            ["left_click_drag", "--coordinate", "300", "400", "--text", "drag"]
+        )
+
+
+def test_parse_args_scroll_missing_scroll_direction() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(
+            ["scroll", "--scroll_amount", "10", "--coordinate", "100", "200"]
+        )
+
+
+def test_parse_args_wait_missing_duration() -> None:
+    with pytest.raises(SystemExit):
+        parse_arguments(["wait"])

inspect_ai/tool/_tools/_web_browser/_resources/.pylintrc

@@ -0,0 +1,8 @@
+[MASTER]
+; R - Refactorings
+; C - Convention
+; W - Warning
+; E - Error
+enable=C,R,W,E
+disable=R0903,C0114,C0115,C0116,C0301,C0411,C1804,C1805,W0120,W0511,E0401,E1101,E0611,E1128
+score=no

inspect_ai/tool/_tools/_web_browser/_resources/.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "debugpy",
+      "request": "launch",
+      "name": "Debug Web Server",
+      "program": "${workspaceFolder}/web_server.py"
+    },
+    {
+      "type": "debugpy",
+      "request": "launch",
+      "name": "Debug Web Client interactive mode",
+      "program": "${workspaceFolder}/web_client.py"
+    },
+    {
+      "type": "debugpy",
+      "request": "launch",
+      "name": "Debug Web Client w/arguments",
+      "program": "${workspaceFolder}/web_client.py",
+      "args": ["${command:pickArgs}"]
+    }
+  ]
+}

inspect_ai/tool/_tools/_web_browser/_resources/.vscode/settings.json

@@ -0,0 +1,25 @@
+{
+  "cSpell.words": [
+    "activedescendant",
+    "describedby",
+    "domcontentloaded",
+    "figcaption",
+    "flowto",
+    "framenavigated",
+    "headful",
+    "idref",
+    "jsonrpcclient",
+    "jsonrpcserver",
+    "keepalive",
+    "keyshortcuts",
+    "labelfor",
+    "labelledby",
+    "labelwrapped",
+    "multiselectable",
+    "Rects",
+    "roledescription",
+    "rubyannotation",
+    "tablecaption",
+    "valuetext"
+  ]
+}

inspect_ai/tool/_tools/_web_browser/_resources/Dockerfile

@@ -8,16 +8,15 @@ RUN apt-get update
 
 RUN pip install --upgrade pip
 
+RUN pip install playwright jsonrpcclient jsonrpcserver httpx aiohttp pillow pydantic tenacity
+
 # Install playwright
-RUN pip install playwright 
 RUN playwright install
 RUN playwright install-deps 
 
-# Install other dependancies
-RUN pip install dm-env-rpc pillow bs4 lxml
-
 # Copy Python files alongside the Dockerfile
-COPY *.py ./
+COPY . .
 
 # Run the server
-CMD ["python3", "/app/web_browser/web_server.py"]
+CMD ["python3", "/app/web_browser/web_server.py"]
+# CMD ["tail", "-f", "/dev/null"]

inspect_ai/tool/_tools/_web_browser/_resources/README.md

@@ -1,7 +1,6 @@
 ## Headless Browser Tool
 
-This directory contains an implementation for the Headless Browser Tool which can be used to test web browsing agents. 
-
+This directory contains an implementation for the Headless Browser Tool which can be used to test web browsing agents.
 
 ### Usage
 
@@ -37,27 +36,27 @@ The result will be printed out in _stdout_ in the following format:
 
 ```
 # Inside the Docker container
-error: <an ERROR message if one occured>
+error: <an ERROR message if one occurred>
 info: <general info about the container>
 web_url: <the URL of the page the browser is currently at>
 web_at: <accessibility tree of the visible elements of the page>
-```   
-
+```
 
 ### Design
 
 The following diagram describes the design and the intended usage of the tool:
 
-![diagram](images/usage_diagram.png)
+![diagram](images/usage_diagram.svg)
 
 The tool consists of the following components:
 
-* [WebServer](web_server.py) - a server which launches a stateful session with the headless chromium browser and interracts with it through the [Playwright API](https://playwright.dev/python/docs/intro) upon receiving client commands. The server components are: 
-  * _dm_env_servicer.py_ - an implementation for the gRPC Service based on [dm_env_rpc protocol](https://github.com/google-deepmind/dm_env_rpc).
-  * _web_environment.py_ - an environment which gets instantiated by the servicer and which launches the browser, stores its state and maps client commands to Playwright API. 
-  * _playwright_crawler.py_ - a wrapper over the sync Playwright API. 
+- [WebServer](web_server.py) - a server which launches a stateful session with the headless chromium browser and interacts with it through the [Playwright API](https://playwright.dev/python/docs/intro) upon receiving client commands. The server components are:
+
+  - _dm_env_servicer.py_ - an implementation for the gRPC Service based on [dm_env_rpc protocol](https://github.com/google-deepmind/dm_env_rpc).
+  - _web_environment.py_ - an environment which gets instantiated by the servicer and which launches the browser, stores its state and maps client commands to Playwright API.
+  - _playwright_crawler.py_ - a wrapper over the sync Playwright API.
 
-* [WebClient](web_client.py) - a simple stateless client to interact with the server. When launched, the client:
+- [WebClient](web_client.py) - a simple stateless client to interact with the server. When launched, the client:
   1. creates a connection with the server;
   2. sends user command to the server;
   3. receives the response in the form of observations and prints them to stdout;

inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree.py

@@ -0,0 +1,71 @@
+from functools import reduce
+from typing import Iterable, TypedDict
+
+from accessibility_tree_node import AccessibilityTreeNode
+from cdp.a11y import AXNode, AXNodeId
+from cdp.dom_snapshot import DOMSnapshot, create_snapshot_context
+from rectangle import Rectangle
+
+_AccType = tuple[
+    AXNode | None,
+    dict[AXNodeId, AXNode],
+]
+
+
+class AccessibilityTree(TypedDict):
+    root: AccessibilityTreeNode
+    nodes: dict[AXNodeId, AccessibilityTreeNode]
+
+
+def create_accessibility_tree(
+    *,
+    ax_nodes: Iterable[AXNode],
+    dom_snapshot: DOMSnapshot,
+    device_scale_factor: float,
+    window_bounds: Rectangle,
+) -> AccessibilityTree | None:
+    """
+    Creates an accessibility tree from the given Chrome DevTools Protocol AX nodes and DOM snapshot.
+
+    Args:
+      ax_nodes (Iterable[AXNode]): An iterable of AXNode objects representing the accessibility nodes.
+      dom_snapshot (DOMSnapshot): A snapshot of the DOM at the time of accessibility tree creation.
+      device_scale_factor (float): The scale factor of the device.
+      window_bounds (Bounds): The bounds of the window.
+
+    Returns:
+      AccessibilityTree: The accessibility tree.
+    """
+
+    # first make a dict of AXNodeId's to AXNode's and find the root on the way
+    def reducer(acc: _AccType, ax_node: AXNode) -> _AccType:
+        root_node, nodes = acc
+        nodes[ax_node.nodeId] = ax_node
+        return (
+            # TODO: What do we want for multiple roots?
+            root_node or (ax_node if ax_node.parentId is None else None),
+            nodes,
+        )
+
+    initial_acc: _AccType = (None, {})  # The inference engine is weak
+    root_node, nodes = reduce(reducer, ax_nodes, initial_acc)
+
+    if not root_node:
+        return None
+
+    # Now create the AccessibilityTreeNode hierarchy
+    snapshot_context = create_snapshot_context(dom_snapshot)
+    all_accessibility_tree_nodes: dict[AXNodeId, AccessibilityTreeNode] = {}
+
+    return AccessibilityTree(
+        root=AccessibilityTreeNode(
+            ax_node=root_node,
+            ax_nodes=nodes,
+            parent=None,
+            all_accessibility_tree_nodes=all_accessibility_tree_nodes,
+            snapshot_context=snapshot_context,
+            device_scale_factor=device_scale_factor,
+            window_bounds=window_bounds,
+        ),
+        nodes=all_accessibility_tree_nodes,
+    )

inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree_node.py

@@ -0,0 +1,323 @@
+from functools import reduce
+from typing import Optional
+
+from cdp.a11y import (
+    AXNode,
+    AXNodeId,
+    AXProperty,
+    AXPropertyName,
+    node_has_property,
+    string_from_ax_value,
+)
+from cdp.dom_snapshot import (
+    DOMSnapshotContext,
+    bounds_for_node_index,
+    current_url_src_for_node_index,
+    index_for_node_id,
+    text_value_for_node_index,
+)
+from rectangle import Rectangle
+
+# Properties to ignore when printing out the accessibility tree.
+_IGNORED_AT_PROPERTIES: set[AXPropertyName] = {
+    "focusable",
+    "readonly",
+    "level",
+    "settable",
+    "multiline",
+    "invalid",
+}
+
+# The AXRole enum is here (https://chromium.googlesource.com/chromium/chromium/+/HEAD/ui/accessibility/ax_enums.idl#62)
+_ROLES_IGNORED = {"horizontal_rule", "ignored"}
+
+# These are noisy and provide no real benefit unless there's a name associated with them
+_ROLES_REQUIRING_A_NAME = {"generic", "paragraph", "div"}
+
+# Used for debugging, include the elements bounding rect during output.
+_INCLUDE_BOUNDS_IN_OUTPUT = False
+
+
+class AccessibilityTreeNode:
+    def __init__(
+        self,
+        *,
+        ax_node: AXNode,
+        ax_nodes: dict[AXNodeId, AXNode],
+        parent: Optional["AccessibilityTreeNode"],
+        all_accessibility_tree_nodes: dict[AXNodeId, "AccessibilityTreeNode"],
+        snapshot_context: DOMSnapshotContext,
+        device_scale_factor: float,
+        window_bounds: Rectangle,
+    ):
+        all_accessibility_tree_nodes[ax_node.nodeId] = self
+
+        self._is_expanded = node_has_property(ax_node, "expanded")
+        self._ax_node = ax_node
+        self._parent = parent
+        self._is_ignored: bool | None = None
+        self._bounds: Rectangle | None = None
+        self._is_visible: bool | None = None
+        self._current_src_url: str | None = None
+        self._input: str | None = None
+        self.__closest_non_ignored_parent: AccessibilityTreeNode | None = None
+
+        dom_node_index = (
+            index_for_node_id(ax_node.backendDOMNodeId, snapshot_context)
+            if ax_node.backendDOMNodeId
+            else None
+        )
+
+        # The following three variables (_bounds, _is_visible, and _current_src_url) depend on
+        # info from the DOM element associated with this AXNode
+        if dom_node_index:
+            self._bounds = (
+                Rectangle(*layout_bounds).scale(1 / device_scale_factor)
+                if (
+                    layout_bounds := bounds_for_node_index(
+                        dom_node_index, snapshot_context
+                    )
+                )
+                is not None
+                else None
+            )
+            self._is_visible = (
+                (self._bounds.has_area and self._bounds.overlaps(window_bounds))
+                if self._bounds
+                else None
+            )
+            self._current_src_url = (
+                current_url_src_for_node_index(dom_node_index, snapshot_context)
+                if self.role == "image"
+                else None
+            )
+
+        if node_has_property(ax_node, "editable"):
+            # Sometimes a node will have it's input stored as 'value' other times we
+            # need to go looking through the DOM tree to find its matching string.
+            self._input = (
+                string_from_ax_value(ax_node.value)
+                if ax_node.value
+                else text_value_for_node_index(dom_node_index, snapshot_context)
+                if dom_node_index is not None
+                else None
+            )
+
+        # this is a little awkward, but the old code ensured that menuitems were visible in given the following condition
+        if self.role == "menuitem" and parent is not None and parent.is_expanded:
+            self._is_visible = True
+
+        # It's important to set all other properties before creating children in case
+        # the children sample parent.xxx
+        self.children = (
+            [
+                AccessibilityTreeNode(
+                    ax_node=ax_nodes[child_id],
+                    ax_nodes=ax_nodes,
+                    parent=self,
+                    all_accessibility_tree_nodes=all_accessibility_tree_nodes,
+                    snapshot_context=snapshot_context,
+                    device_scale_factor=device_scale_factor,
+                    window_bounds=window_bounds,
+                )
+                for child_id in ax_node.childIds
+            ]
+            if ax_node.childIds
+            else None
+        )
+
+    def __str__(self) -> str:
+        bounds_string = str(self.bounds) if _INCLUDE_BOUNDS_IN_OUTPUT else None
+
+        input_string = f'Current input: "{self._input}"' if self._input else None
+
+        name_string = f'"{self.name}"' if self.name else None
+        # Without bounds we can't click on the element, and therefore the ID
+        # can not be used. Still useful to show these elements though (e.g.)
+        # to let user know about options.
+        id_string = f"[{'*' if not self._bounds or not self._bounds.has_area else self._ax_node.nodeId}]"
+        url_string = self._current_src_url if self._current_src_url else None
+
+        url_string = _maybe_redact_base64_url(self._current_src_url)
+
+        return " ".join(
+            [
+                item
+                for item in [
+                    id_string,
+                    self.role,
+                    name_string,
+                    url_string,
+                    input_string,
+                    self._property_string(),
+                    bounds_string,
+                ]
+                if item is not None
+            ]
+        )
+
+    @property
+    def node_id(self) -> AXNodeId:
+        return self._ax_node.nodeId
+
+    @property
+    def name(self) -> str:
+        return string_from_ax_value(self._ax_node.name)
+
+    @property
+    def is_expanded(self) -> bool:
+        return self._is_expanded
+
+    @property
+    def is_visible(self) -> bool:
+        return self._is_visible or False
+
+    @property
+    def bounds(self) -> Rectangle | None:
+        return self._bounds
+
+    @property
+    def current_src_url(self) -> str | None:
+        return self._current_src_url
+
+    @property
+    def input(self) -> str | None:
+        return self._input
+
+    @property
+    def is_ignored(self) -> bool:
+        # computing _is_ignored is very expensive, do it on demand and remember the result
+        if self._is_ignored is None:
+            self._is_ignored = self._compute_is_ignored()
+        return self._is_ignored
+
+    @property
+    def role(self) -> str:
+        return string_from_ax_value(self._ax_node.role)
+
+    @property
+    def _closest_non_ignored_parent(self) -> Optional["AccessibilityTreeNode"]:
+        if self._parent and self.__closest_non_ignored_parent is None:
+            self.__closest_non_ignored_parent = (
+                None
+                if not self._parent
+                else self._parent
+                if not self._parent.is_ignored
+                else self._parent._closest_non_ignored_parent  # pylint: disable=protected-access
+            )
+        return self.__closest_non_ignored_parent
+
+    def render_accessibility_tree(self, indent: int = 0) -> str:
+        """Returns the string representation of the node and its children."""
+        result = ""
+        if not self.is_ignored:
+            result = "  " * indent + str(self) + "\n"
+            indent += 1
+        children_strings = [
+            child.render_accessibility_tree(indent) for child in self.children or []
+        ]
+        result = result + "".join([s for s in children_strings if s])
+        return result
+
+    def render_main_content(self) -> str | None:
+        return (
+            main_node._extract_text()  # pylint: disable=protected-access
+            if (main_node := self._get_main_content_node())
+            else None
+        )
+
+    def _compute_is_ignored(self) -> bool:
+        if self._ax_node.ignored or not self.is_visible or self.role in _ROLES_IGNORED:
+            return True
+
+        # if the parent is a link/button with a name, and this node is within the parent's bounds, we can ignore it.
+        if (
+            (closest_parent := self._closest_non_ignored_parent)
+            and closest_parent.bounds
+            and self.bounds
+            and (closest_parent.role == "link" or closest_parent.role == "button")
+            and closest_parent.name
+            and self.bounds.within(closest_parent.bounds)
+        ):
+            return True
+
+        if bool(self.name):  # excludes None or ""
+            # This simplifies things like links/buttons whose contents are sub-elements
+            # e.g.
+            #   [158] button "Menu"
+            #     [1166] StaticText "Menu"
+            # becomes
+            #   [158] button "Menu"
+            return self._parent is not None and self._parent.name == self.name
+
+        return self.role in _ROLES_REQUIRING_A_NAME
+
+    def _get_main_content_node(self) -> Optional["AccessibilityTreeNode"]:
+        if self.role == "main":
+            return self
+
+        return next(
+            (
+                result
+                for child in self.children or []
+                if (result := child._get_main_content_node())  # pylint: disable=protected-access
+            ),
+            None,
+        )
+
+    def _extract_text(self):
+        """
+        Recursively extracts and concatenates text from the accessibility tree nodes.
+
+        This method traverses the accessibility tree starting from the current node,
+        concatenating the text of each node.
+
+        Returns:
+          str: The concatenated text from the accessibility tree nodes, with
+             appropriate spacing based on node roles.
+        """
+        if not self.is_visible:
+            return ""
+
+        def reducer(acc, child):
+            child_text = child._extract_text()  # pylint: disable=protected-access
+            return (
+                acc
+                + (
+                    "\n\n"
+                    if child.role
+                    in {"paragraph", "note", "row", "div", "heading", "region"}
+                    else " "
+                )
+                + child_text
+            )
+
+        return reduce(reducer, self.children or [], self.name).strip()
+
+    def _property_string(self) -> str:
+        properties = [
+            _prop_name_value_str(node_property)
+            if node_property.value.value
+            else f"{node_property.name}"
+            for node_property in self._ax_node.properties or ()
+            if node_property.name not in _IGNORED_AT_PROPERTIES
+        ]
+        return " [" + ", ".join(properties) + "]" if properties else ""
+
+
+def _prop_name_value_str(node_property: AXProperty) -> str:
+    # pre-commit hook (debug-statements) crashes if value is inlined below
+    value = (
+        _maybe_redact_base64_url(node_property.value.value)
+        if node_property.name == "url" and isinstance(node_property.value.value, str)
+        else node_property.value.value
+    )
+    return f"{node_property.name}: {value}"
+
+
+def _maybe_redact_base64_url(url: str | None) -> str | None:
+    return (
+        "data:<base64-data-removed>"
+        if url and url.startswith("data:") and "base64" in url
+        else url
+    )

inspect_ai/tool/_tools/_web_browser/_resources/cdp/__init__.py

@@ -0,0 +1,5 @@
+"""
+Types and pure functional helpers associated with Chrome DevTools Protocol
+
+https://chromedevtools.github.io/devtools-protocol/
+"""