inspect-ai 0.3.58__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/tool/beta/_computer/_computer.py

@@ -0,0 +1,155 @@
+from typing import Awaitable, Callable
+
+from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai.tool import Tool, ToolResult, tool
+from inspect_ai.tool._tool import (
+    TOOL_INIT_MODEL_INPUT,
+    ToolParsingError,
+)
+from inspect_ai.tool._tool_call import ToolCallModelInput
+
+from . import _common as common
+from ._common import Action
+
+ActionFunction = Callable[[str], ToolResult | Awaitable[ToolResult]]
+
+
+@tool
+def computer(max_screenshots: int | None = 1, timeout: int | None = 180) -> Tool:
+    async def execute(
+        action: Action,
+        text: str | None = None,
+        coordinate: list[int] | None = None,
+    ) -> ToolResult:
+        """
+        Use this tool to interact with a computer.
+
+        Use a mouse and keyboard to interact with a computer's desktop GUI.
+
+        Keep in mind that icons require double clicks to open while other UI affordances like menu items and buttons require a single click.
+
+        Args:
+          action (Action): The action to perform.
+              - `key`: Press a key or key-combination on the keyboard.
+                  - Example: execute(action="key", text="ctrl+s")
+                  - Text can be any key name supported by xdotool's `key` such as:
+                      "Return", "Escape", "alt+Tab", "BackSpace", "Tab", "alt+Tab", "ctrl+s", "Up", "KP_0" (for the numpad 0 key),
+                      "Insert", "Delete", "Home", "End", "Prior", "Next", "Left", "Up", "Right", "Down",
+                      "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12",
+                      "Shift_L", "Shift_R", "Control_L", "Control_R", "Alt_L", "Alt_R", "Scroll_Lock", "Num_Lock", "Caps_Lock", "Pause",
+                      "KP_Multiply", "KP_Home", "KP_Up", "KP_Prior", "KP_Subtract", "KP_Left", "KP_Begin", "KP_Right", "KP_Add", "KP_End","KP_Down",
+                      "KP_Next", "KP_Insert", "KP_Delete", "KP_Enter", "KP_Divide", "KP_Equal", "KP_Decimal",
+              - `type`: Type a string of text on the keyboard. If the text contains spaces, enclose it in quotes.
+                  - Example: execute(action="type", text="The crux of the biscuit is the apostrophe!")
+              - `cursor_position`: Get the current (x, y) pixel coordinate of the cursor on the screen.
+              - `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.
+                  - Example: execute(action="mouse_move", coordinate=(100, 200))
+              - `left_click`: Click the left mouse button.
+              - `left_click_drag`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
+                  - Example: execute(action="left_click_drag", coordinate=(150, 250))
+              - `right_click`: Click the right mouse button.
+              - `middle_click`: Click the middle mouse button.
+              - `double_click`: Double-click the left mouse button.
+              - `screenshot`: Take a screenshot.
+          text (str | None): The text to type or the key to press. Required when action is "key" or "type".
+          coordinate (tuple[int, int] | None): The (x, y) pixel coordinate on the screen to which to move or drag. Required when action is "mouse_move" or "left_click_drag".
+
+        Returns:
+          The output of the command. Many commands will include a screenshot reflecting the result of the command in their output.
+        """
+        if action in ("mouse_move", "left_click_drag"):
+            if coordinate is None:
+                raise ToolParsingError(f"coordinate is required for {action}")
+            if text is not None:
+                raise ToolParsingError(f"text is not accepted for {action}")
+            if not isinstance(coordinate, list) or len(coordinate) != 2:
+                raise ToolParsingError(f"{coordinate} must be a tuple of length 2")
+            if not all(isinstance(i, int) and i >= 0 for i in coordinate):
+                raise ToolParsingError(
+                    f"{coordinate} must be a tuple of non-negative ints"
+                )
+
+            if action == "mouse_move":
+                return await common.mouse_move(
+                    coordinate[0], coordinate[1], timeout=timeout
+                )
+            elif action == "left_click_drag":
+                return await common.left_click_drag(
+                    coordinate[0], coordinate[1], timeout=timeout
+                )
+
+        if action in ("key", "type"):
+            if text is None:
+                raise ToolParsingError(f"text is required for {action}")
+            if coordinate is not None:
+                raise ToolParsingError(f"coordinate is not accepted for {action}")
+            if not isinstance(text, str):
+                raise ToolParsingError(output=f"{text} must be a string")
+
+            if action == "key":
+                return await common.press_key(text, timeout=timeout)
+            elif action == "type":
+                return await common.type(text, timeout=timeout)
+
+        if action in (
+            "left_click",
+            "right_click",
+            "double_click",
+            "middle_click",
+            "screenshot",
+            "cursor_position",
+        ):
+            if text is not None:
+                raise ToolParsingError(f"text is not accepted for {action}")
+            if coordinate is not None:
+                raise ToolParsingError(f"coordinate is not accepted for {action}")
+
+            if action == "screenshot":
+                return await common.screenshot(timeout=timeout)
+            elif action == "cursor_position":
+                return await common.cursor_position(timeout=timeout)
+            elif action == "left_click":
+                return await common.left_click(timeout=timeout)
+            elif action == "right_click":
+                return await common.right_click(timeout=timeout)
+            elif action == "middle_click":
+                return await common.middle_click(timeout=timeout)
+            elif action == "double_click":
+                return await common.double_click(timeout=timeout)
+
+        raise ToolParsingError(f"Invalid action: {action}")
+
+    # if max_screenshots is specified then polk model input into where @tool can find it
+    if max_screenshots is not None:
+        setattr(execute, TOOL_INIT_MODEL_INPUT, _computer_model_input(max_screenshots))
+
+    return execute
+
+
+def _computer_model_input(max_screenshots: int) -> ToolCallModelInput:
+    def model_input(
+        message_index: int, message_total: int, content: str | list[Content]
+    ) -> str | list[Content]:
+        # nothing to do for scalars
+        if isinstance(content, str):
+            return content
+
+        # if we are inside max_screenshots then return as is
+        elif (message_total - message_index) <= max_screenshots:
+            return content
+
+        # otherwise convert images to text placeholdrs
+        else:
+            input_content: list[Content] = []
+            for c in content:
+                if isinstance(c, ContentImage):
+                    input_content.append(
+                        ContentText(
+                            text="Screenshot removed to reduce size of input. Please consult the latest screenshots for the most up to date state of the screen."
+                        )
+                    )
+                else:
+                    input_content.append(c)
+            return input_content
+
+    return model_input

inspect_ai/tool/beta/_computer/_computer_split.py

@@ -0,0 +1,198 @@
+"""
+This module provides the same functionality as the computer tool but via a list of per-action tools . e.g. computer_mouse_move(100, 100).
+
+The split version is not publicly exported, but is retained until we decide if it performs better than the monolithic computer tool.
+"""
+
+from typing import Awaitable, Callable
+
+from inspect_ai.tool import Tool, ToolResult, tool
+
+from . import _common as common
+
+ActionFunction = Callable[[str], ToolResult | Awaitable[ToolResult]]
+
+
+def computer_split(timeout: int | None = None) -> list[Tool]:
+    """
+    Computer interaction tools.
+
+    Args:
+      timeout (int | None): Timeout (in seconds) for command.
+
+    Returns:
+       List of computer interaction tools.
+    """
+    return [
+        computer_cursor_position(),
+        computer_screenshot(),
+        computer_mouse_move(),
+        computer_left_click(),
+        computer_double_click(),
+        computer_left_click_drag(),
+        computer_right_click(),
+        computer_key(),
+        computer_type(),
+    ]
+
+
+@tool()
+def computer_cursor_position(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Get the current (x, y) pixel coordinate of the cursor on the screen.
+
+        Args:
+          None
+
+        Returns:
+          A `str` of the form "x y" where x and y are the current mouse coordinates.
+        """
+        return await common.cursor_position(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_screenshot(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Take a screenshot.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.screenshot(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_mouse_move(timeout: int | None = None) -> Tool:
+    async def execute(x: int, y: int) -> ToolResult:
+        """
+        Move the cursor to a specified (x, y) pixel coordinate on the screen.
+
+        Args:
+          x: X coordinate of the mouse destination.
+          y: Y coordinate of the mouse destination.
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.mouse_move(x, y, timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_left_click(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Click the left mouse button.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.left_click(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_double_click(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Double-click the left mouse button.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.double_click(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_left_click_drag(timeout: int | None = None) -> Tool:
+    async def execute(x: int, y: int) -> ToolResult:
+        """
+        Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
+
+        Args:
+          x: X coordinate of the mouse destination.
+          y: Y coordinate of the mouse destination.
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.left_click_drag(x, y, timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_right_click(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Click the right mouse button.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.right_click(timeout=timeout)
+
+    return execute
+
+
+# keysm list is from https://gist.github.com/rvaiya/be31f42049a4b5ad46666a8e120d9843
+@tool()
+def computer_key(timeout: int | None = None) -> Tool:
+    async def execute(key: str) -> ToolResult:
+        """
+        Press a key or key-combination on the keyboard.
+
+        Args:
+          key: The key or key-combination to press. Can be any key name supported by xdotool's `key` such as:
+            "Return", "Escape", "alt+Tab", "BackSpace", "Tab", "alt+Tab", "ctrl+s", "Up", "KP_0" (for the numpad 0 key),
+            "Insert", "Delete", "Home", "End", "Prior", "Next", "Left", "Up", "Right", "Down",
+            "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12",
+            "Shift_L", "Shift_R", "Control_L", "Control_R", "Alt_L", "Alt_R", "Scroll_Lock", "Num_Lock", "Caps_Lock", "Pause",
+            "KP_Multiply", "KP_Home", "KP_Up", "KP_Prior", "KP_Subtract", "KP_Left", "KP_Begin", "KP_Right", "KP_Add", "KP_End","KP_Down",
+            "KP_Next", "KP_Insert", "KP_Delete", "KP_Enter", "KP_Divide", "KP_Equal", "KP_Decimal"
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.press_key(key, timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_type(timeout: int | None = None) -> Tool:
+    async def execute(text: str) -> ToolResult:
+        """
+        Type a string of text on the keyboard.
+
+        Args:
+          text: The text to type. If the text contains spaces, enclose it in quotes.
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.type(text, timeout=timeout)
+
+    return execute

inspect_ai/tool/beta/_computer/_resources/Dockerfile

@@ -0,0 +1,100 @@
+FROM docker.io/ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_PRIORITY=high
+
+# Core/system layer
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install \
+      # A virtual framebuffer for running GUI applications without a physical display.
+      xvfb \    
+      # A lightweight desktop environment for UNIX-like operating systems.
+      xfce4 \
+      # The terminal emulator for the xfce4 desktop environment.
+      xfce4-terminal\
+      # A VNC server for sharing X11 desktops.
+      x11vnc \
+      # A web based VNC client
+      novnc \
+      # A WebSocket to TCP proxy/bridge for noVNC
+      websockify \
+      # The Python programming language interpreter.
+      python3 \
+      # The package installer for Python.
+      python3-pip \
+      # A command-line tool for automating X11 applications (e.g., simulating keyboard/mouse inputs).
+      xdotool \
+      # A command-line tool for taking screenshots.
+      scrot \
+      # A suite for image manipulation — needed for scaling images.
+      imagemagick && \
+    apt-get clean
+
+# Userland apt-get'able apps
+RUN apt-get install -y --no-install-recommends \
+      # A simple image viewer.
+      xpaint \
+      # A calculator application.
+      galculator && \
+    apt-get clean
+
+# install Firefox
+RUN apt-get install -y software-properties-common && \
+    add-apt-repository ppa:mozillateam/ppa && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends firefox-esr && \
+    apt-get clean
+
+# install VS Code
+RUN apt-get install -y \
+      gpg \
+      wget \
+      apt-transport-https \
+      software-properties-common && \
+    wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg && \
+    install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg && \
+    sh -c 'echo "deb [arch=amd64,arm64 signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list' && \
+    apt-get update && \
+    apt-get install -y code && \
+    apt-get clean
+
+# configure noVNC
+RUN ln -s /usr/share/novnc/vnc.html /usr/share/novnc/index.html
+
+# We copy requirements.txt by itself so that changes to the scripts will be in a later layer
+# and we only pip install if requirements.txt changes
+COPY tool/requirements.txt /opt/inspect/tool/requirements.txt
+RUN cd /opt/inspect/tool && pip3 install --no-cache-dir -r requirements.txt
+
+COPY tool/ /opt/inspect/tool
+COPY entrypoint/ /opt/inspect/entrypoint
+RUN chmod -R 755 /opt/inspect
+
+# setup user
+ENV USERNAME=user
+ENV HOME=/home/$USERNAME
+RUN useradd -m -s /bin/bash -d $HOME $USERNAME
+RUN echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER ${USERNAME}
+WORKDIR $HOME
+COPY --chown=$USERNAME:$USERNAME image_home_dir/ $HOME
+
+# configure Firefox to skip all 'first run' UI
+RUN mkdir -p $HOME/.mozilla/firefox-esr/profile.default && \
+    echo 'user_pref("browser.startup.homepage_override.mstone", "ignore");' >> $HOME/.mozilla/firefox-esr/profile.default/user.js && \
+    echo 'user_pref("browser.aboutwelcome.enabled", false);' >> $HOME/.mozilla/firefox-esr/profile.default/user.js && \
+    echo 'user_pref("datareporting.policy.firstRunURL", "");' >> $HOME/.mozilla/firefox-esr/profile.default/user.js 
+    
+EXPOSE 5900
+EXPOSE 6080
+
+ARG DISPLAY_NUM=1
+ARG WIDTH=1920
+ARG HEIGHT=1080
+ENV DISPLAY_NUM=$DISPLAY_NUM
+ENV DISPLAY=:${DISPLAY_NUM}
+ENV HEIGHT=$HEIGHT
+ENV WIDTH=$WIDTH
+
+ENTRYPOINT [ "/opt/inspect/entrypoint/entrypoint.sh" ]

inspect_ai/tool/beta/_computer/_resources/README.md

@@ -0,0 +1,30 @@
+# About This Image
+
+This image was inspired by Anthropic's Computer Use Demo [here](https://github.com/anthropics/anthropic-quickstarts/tree/main/computer-use-demo/image).
+
+Its goal is to provide the minimum infrastructure to support the use of Inspect's `computer_tool` to interact with the computer via X11 and `xdotool`, while also providing observability and interaction via VNC and noVNC.
+
+The image extends this minimal functionality by adding a few basic applications — VS Code, Firefox, XPaint, and galculator.
+
+## Entrypoint Directory
+
+1. **Xvfb (X Virtual Framebuffer)**
+   - **Script:** `xvfb_startup.sh`
+   - **Description:** Xvfb is a display server that implements the X11 display server protocol. It runs in memory and does not require a physical display, useful for running graphical applications in a headless environment.
+
+1. **xfce4**
+   - **Script:** `xfce4_startup.sh`
+   - **Description:** xfce4 is a lightweight desktop environment for UNIX-like operating systems. It aims to be fast, low on system resources, and user-friendly.
+
+1. **x11vnc**
+   - **Script:** `x11vnc_startup.sh`
+   - **Description:** x11vnc is a VNC server that allows remote access to the X11 display. It enables users to connect to the virtual display environment from a remote machine using a VNC client.
+
+1. **noVNC**
+   - **Script:** `novnc_startup.sh`
+   - **Description:** noVNC is a VNC client that runs in a web browser. It allows users to access the virtual display environment through a web interface without needing a separate VNC client application.
+
+## Desktop Directory
+
+The `Desktop` directory contains launchers for VS Code, Firefox and XPaint.
+

inspect_ai/tool/beta/_computer/_resources/entrypoint/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# remove marker files
+rm -f /tmp/.X${DISPLAY_NUM}-lock 
+rm -f /tmp/xfce_started
+
+/opt/inspect/entrypoint/xvfb_startup.sh
+/opt/inspect/entrypoint/xfce_startup.sh
+/opt/inspect/entrypoint/x11vnc_startup.sh
+/opt/inspect/entrypoint/novnc_startup.sh
+
+# Run CMD if provided
+echo "Executing CMD from derived Dockerfile: $@"
+exec "$@"
+
+# Keep the container running
+tail -f /dev/null

inspect_ai/tool/beta/_computer/_resources/entrypoint/novnc_startup.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+echo "starting noVNC"
+
+# Start noVNC with explicit websocket settings
+websockify \
+    --web=/usr/share/novnc/ \
+    6080 localhost:5900 \
+    > /tmp/novnc.log 2>&1 &
+
+# Wait for noVNC to start
+timeout=10
+while [ $timeout -gt 0 ]; do
+    if netstat -tuln | grep -q ":6080 "; then
+        break
+    fi
+    sleep 1
+    ((timeout--))
+done
+
+echo "noVNC started successfully"

inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+echo "starting vnc"
+
+(x11vnc -display $DISPLAY \
+    -forever \
+    -shared \
+    -wait 50 \
+    -cursor most \
+    -cursor arrow \
+    -rfbport 5900 \
+    -nopw \
+    2>/tmp/x11vnc_stderr.log) &
+
+x11vnc_pid=$!
+
+# Wait for x11vnc to start
+timeout=10
+while [ $timeout -gt 0 ]; do
+    if netstat -tuln | grep -q ":5900 "; then
+        break
+    fi
+    sleep 1
+    ((timeout--))
+done
+
+if [ $timeout -eq 0 ]; then
+    echo "x11vnc failed to start, stderr output:" >&2
+    cat /tmp/x11vnc_stderr.log >&2
+    exit 1
+fi
+
+: > /tmp/x11vnc_stderr.log
+
+# Monitor x11vnc process in the background
+(
+    while true; do
+        if ! kill -0 $x11vnc_pid 2>/dev/null; then
+            echo "x11vnc process crashed, restarting..." >&2
+            if [ -f /tmp/x11vnc_stderr.log ]; then
+                echo "x11vnc stderr output:" >&2
+                cat /tmp/x11vnc_stderr.log >&2
+                rm /tmp/x11vnc_stderr.log
+            fi
+            exec "$0"
+        fi
+        sleep 5
+    done
+) &

inspect_ai/tool/beta/_computer/_resources/entrypoint/xfce_startup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "starting XFCE4"
+startxfce4 &
+
+while ! pgrep -x "xfce4-session" > /dev/null; do
+  echo "Waiting for XFCE4 to start..."
+  sleep 1
+done
+
+echo "XFCE4 is fully started!"
+touch /tmp/xfce_started
+

inspect_ai/tool/beta/_computer/_resources/entrypoint/xvfb_startup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -e  # Exit on error
+
+DPI=96
+RES_AND_DEPTH=${WIDTH}x${HEIGHT}x24
+
+# Function to check if Xvfb is already running
+check_xvfb_running() {
+    if [ -e /tmp/.X${DISPLAY_NUM}-lock ]; then
+        return 0  # Xvfb is already running
+    else
+        return 1  # Xvfb is not running
+    fi
+}
+
+# Function to check if Xvfb is ready
+wait_for_xvfb() {
+    local timeout=10
+    local start_time=$(date +%s)
+    while ! xdpyinfo >/dev/null 2>&1; do
+        if [ $(($(date +%s) - start_time)) -gt $timeout ]; then
+            echo "Xvfb failed to start within $timeout seconds" >&2
+            return 1
+        fi
+        sleep 0.1
+    done
+    return 0
+}
+
+# Check if Xvfb is already running
+if check_xvfb_running; then
+    echo "Xvfb is already running on display ${DISPLAY}"
+    exit 0
+fi
+
+# Start Xvfb
+Xvfb $DISPLAY -ac -screen 0 $RES_AND_DEPTH -retro -dpi $DPI -nolisten tcp -nolisten unix &
+XVFB_PID=$!
+
+# Wait for Xvfb to start
+if wait_for_xvfb; then
+    echo "Xvfb started successfully on display ${DISPLAY}"
+    echo "Xvfb PID: $XVFB_PID"
+else
+    echo "Xvfb failed to start"
+    kill $XVFB_PID
+    exit 1
+fi

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=Firefox Web Browser
+Comment=Browse the World Wide Web
+Exec=firefox-esr %u
+Icon=firefox-esr
+Path=
+Terminal=false
+StartupNotify=true

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=Visual Studio Code
+Comment=Code Editing. Redefined.
+Exec=/usr/share/code/code %F
+Icon=vscode
+Path=
+Terminal=false
+StartupNotify=false

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/XPaint.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=XPaint
+Comment=Xpaint painting application
+Exec=xpaint
+Icon=xpaint
+Path=
+Terminal=false
+StartupNotify=false