inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl

inspect_ai/tool/beta/_computer/_computer_split.py

@@ -0,0 +1,198 @@
+"""
+This module provides the same functionality as the computer tool but via a list of per-action tools . e.g. computer_mouse_move(100, 100).
+
+The split version is not publicly exported, but is retained until we decide if it performs better than the monolithic computer tool.
+"""
+
+from typing import Awaitable, Callable
+
+from inspect_ai.tool import Tool, ToolResult, tool
+
+from . import _common as common
+
+ActionFunction = Callable[[str], ToolResult | Awaitable[ToolResult]]
+
+
+def computer_split(timeout: int | None = None) -> list[Tool]:
+    """
+    Computer interaction tools.
+
+    Args:
+      timeout (int | None): Timeout (in seconds) for command.
+
+    Returns:
+       List of computer interaction tools.
+    """
+    return [
+        computer_cursor_position(),
+        computer_screenshot(),
+        computer_mouse_move(),
+        computer_left_click(),
+        computer_double_click(),
+        computer_left_click_drag(),
+        computer_right_click(),
+        computer_key(),
+        computer_type(),
+    ]
+
+
+@tool()
+def computer_cursor_position(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Get the current (x, y) pixel coordinate of the cursor on the screen.
+
+        Args:
+          None
+
+        Returns:
+          A `str` of the form "x y" where x and y are the current mouse coordinates.
+        """
+        return await common.cursor_position(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_screenshot(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Take a screenshot.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.screenshot(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_mouse_move(timeout: int | None = None) -> Tool:
+    async def execute(x: int, y: int) -> ToolResult:
+        """
+        Move the cursor to a specified (x, y) pixel coordinate on the screen.
+
+        Args:
+          x: X coordinate of the mouse destination.
+          y: Y coordinate of the mouse destination.
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.mouse_move(x, y, timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_left_click(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Click the left mouse button.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.left_click(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_double_click(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Double-click the left mouse button.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.double_click(timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_left_click_drag(timeout: int | None = None) -> Tool:
+    async def execute(x: int, y: int) -> ToolResult:
+        """
+        Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
+
+        Args:
+          x: X coordinate of the mouse destination.
+          y: Y coordinate of the mouse destination.
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.left_click_drag(x, y, timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_right_click(timeout: int | None = None) -> Tool:
+    async def execute() -> ToolResult:
+        """
+        Click the right mouse button.
+
+        Args:
+          None
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.right_click(timeout=timeout)
+
+    return execute
+
+
+# keysm list is from https://gist.github.com/rvaiya/be31f42049a4b5ad46666a8e120d9843
+@tool()
+def computer_key(timeout: int | None = None) -> Tool:
+    async def execute(key: str) -> ToolResult:
+        """
+        Press a key or key-combination on the keyboard.
+
+        Args:
+          key: The key or key-combination to press. Can be any key name supported by xdotool's `key` such as:
+            "Return", "Escape", "alt+Tab", "BackSpace", "Tab", "alt+Tab", "ctrl+s", "Up", "KP_0" (for the numpad 0 key),
+            "Insert", "Delete", "Home", "End", "Prior", "Next", "Left", "Up", "Right", "Down",
+            "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12",
+            "Shift_L", "Shift_R", "Control_L", "Control_R", "Alt_L", "Alt_R", "Scroll_Lock", "Num_Lock", "Caps_Lock", "Pause",
+            "KP_Multiply", "KP_Home", "KP_Up", "KP_Prior", "KP_Subtract", "KP_Left", "KP_Begin", "KP_Right", "KP_Add", "KP_End","KP_Down",
+            "KP_Next", "KP_Insert", "KP_Delete", "KP_Enter", "KP_Divide", "KP_Equal", "KP_Decimal"
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.press_key(key, timeout=timeout)
+
+    return execute
+
+
+@tool()
+def computer_type(timeout: int | None = None) -> Tool:
+    async def execute(text: str) -> ToolResult:
+        """
+        Type a string of text on the keyboard.
+
+        Args:
+          text: The text to type. If the text contains spaces, enclose it in quotes.
+
+        Returns:
+          A `list` with a single `ContentImage` of the screen.
+        """
+        return await common.type(text, timeout=timeout)
+
+    return execute

inspect_ai/tool/beta/_computer/_resources/Dockerfile

@@ -0,0 +1,100 @@
+FROM docker.io/ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_PRIORITY=high
+
+# Core/system layer
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install \
+      # A virtual framebuffer for running GUI applications without a physical display.
+      xvfb \    
+      # A lightweight desktop environment for UNIX-like operating systems.
+      xfce4 \
+      # The terminal emulator for the xfce4 desktop environment.
+      xfce4-terminal\
+      # A VNC server for sharing X11 desktops.
+      x11vnc \
+      # A web based VNC client
+      novnc \
+      # A WebSocket to TCP proxy/bridge for noVNC
+      websockify \
+      # The Python programming language interpreter.
+      python3 \
+      # The package installer for Python.
+      python3-pip \
+      # A command-line tool for automating X11 applications (e.g., simulating keyboard/mouse inputs).
+      xdotool \
+      # A command-line tool for taking screenshots.
+      scrot \
+      # A suite for image manipulation — needed for scaling images.
+      imagemagick && \
+    apt-get clean
+
+# Userland apt-get'able apps
+RUN apt-get install -y --no-install-recommends \
+      # A simple image viewer.
+      xpaint \
+      # A calculator application.
+      galculator && \
+    apt-get clean
+
+# install Firefox
+RUN apt-get install -y software-properties-common && \
+    add-apt-repository ppa:mozillateam/ppa && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends firefox-esr && \
+    apt-get clean
+
+# install VS Code
+RUN apt-get install -y \
+      gpg \
+      wget \
+      apt-transport-https \
+      software-properties-common && \
+    wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg && \
+    install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg && \
+    sh -c 'echo "deb [arch=amd64,arm64 signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list' && \
+    apt-get update && \
+    apt-get install -y code && \
+    apt-get clean
+
+# configure noVNC
+RUN ln -s /usr/share/novnc/vnc.html /usr/share/novnc/index.html
+
+# We copy requirements.txt by itself so that changes to the scripts will be in a later layer
+# and we only pip install if requirements.txt changes
+COPY tool/requirements.txt /opt/inspect/tool/requirements.txt
+RUN cd /opt/inspect/tool && pip3 install --no-cache-dir -r requirements.txt
+
+COPY tool/ /opt/inspect/tool
+COPY entrypoint/ /opt/inspect/entrypoint
+RUN chmod -R 755 /opt/inspect
+
+# setup user
+ENV USERNAME=user
+ENV HOME=/home/$USERNAME
+RUN useradd -m -s /bin/bash -d $HOME $USERNAME
+RUN echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER ${USERNAME}
+WORKDIR $HOME
+COPY --chown=$USERNAME:$USERNAME image_home_dir/ $HOME
+
+# configure Firefox to skip all 'first run' UI
+RUN mkdir -p $HOME/.mozilla/firefox-esr/profile.default && \
+    echo 'user_pref("browser.startup.homepage_override.mstone", "ignore");' >> $HOME/.mozilla/firefox-esr/profile.default/user.js && \
+    echo 'user_pref("browser.aboutwelcome.enabled", false);' >> $HOME/.mozilla/firefox-esr/profile.default/user.js && \
+    echo 'user_pref("datareporting.policy.firstRunURL", "");' >> $HOME/.mozilla/firefox-esr/profile.default/user.js 
+    
+EXPOSE 5900
+EXPOSE 6080
+
+ARG DISPLAY_NUM=1
+ARG WIDTH=1920
+ARG HEIGHT=1080
+ENV DISPLAY_NUM=$DISPLAY_NUM
+ENV DISPLAY=:${DISPLAY_NUM}
+ENV HEIGHT=$HEIGHT
+ENV WIDTH=$WIDTH
+
+ENTRYPOINT [ "/opt/inspect/entrypoint/entrypoint.sh" ]

inspect_ai/tool/beta/_computer/_resources/README.md

@@ -0,0 +1,30 @@
+# About This Image
+
+This image was inspired by Anthropic's Computer Use Demo [here](https://github.com/anthropics/anthropic-quickstarts/tree/main/computer-use-demo/image).
+
+Its goal is to provide the minimum infrastructure to support the use of Inspect's `computer_tool` to interact with the computer via X11 and `xdotool`, while also providing observability and interaction via VNC and noVNC.
+
+The image extends this minimal functionality by adding a few basic applications — VS Code, Firefox, XPaint, and galculator.
+
+## Entrypoint Directory
+
+1. **Xvfb (X Virtual Framebuffer)**
+   - **Script:** `xvfb_startup.sh`
+   - **Description:** Xvfb is a display server that implements the X11 display server protocol. It runs in memory and does not require a physical display, useful for running graphical applications in a headless environment.
+
+1. **xfce4**
+   - **Script:** `xfce4_startup.sh`
+   - **Description:** xfce4 is a lightweight desktop environment for UNIX-like operating systems. It aims to be fast, low on system resources, and user-friendly.
+
+1. **x11vnc**
+   - **Script:** `x11vnc_startup.sh`
+   - **Description:** x11vnc is a VNC server that allows remote access to the X11 display. It enables users to connect to the virtual display environment from a remote machine using a VNC client.
+
+1. **noVNC**
+   - **Script:** `novnc_startup.sh`
+   - **Description:** noVNC is a VNC client that runs in a web browser. It allows users to access the virtual display environment through a web interface without needing a separate VNC client application.
+
+## Desktop Directory
+
+The `Desktop` directory contains launchers for VS Code, Firefox and XPaint.
+

inspect_ai/tool/beta/_computer/_resources/entrypoint/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# remove marker files
+rm -f /tmp/.X${DISPLAY_NUM}-lock 
+rm -f /tmp/xfce_started
+
+/opt/inspect/entrypoint/xvfb_startup.sh
+/opt/inspect/entrypoint/xfce_startup.sh
+/opt/inspect/entrypoint/x11vnc_startup.sh
+/opt/inspect/entrypoint/novnc_startup.sh
+
+# Run CMD if provided
+echo "Executing CMD from derived Dockerfile: $@"
+exec "$@"
+
+# Keep the container running
+tail -f /dev/null

inspect_ai/tool/beta/_computer/_resources/entrypoint/novnc_startup.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+echo "starting noVNC"
+
+# Start noVNC with explicit websocket settings
+websockify \
+    --web=/usr/share/novnc/ \
+    6080 localhost:5900 \
+    > /tmp/novnc.log 2>&1 &
+
+# Wait for noVNC to start
+timeout=10
+while [ $timeout -gt 0 ]; do
+    if netstat -tuln | grep -q ":6080 "; then
+        break
+    fi
+    sleep 1
+    ((timeout--))
+done
+
+echo "noVNC started successfully"

inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+echo "starting vnc"
+
+(x11vnc -display $DISPLAY \
+    -forever \
+    -shared \
+    -wait 50 \
+    -cursor most \
+    -cursor arrow \
+    -rfbport 5900 \
+    -nopw \
+    2>/tmp/x11vnc_stderr.log) &
+
+x11vnc_pid=$!
+
+# Wait for x11vnc to start
+timeout=10
+while [ $timeout -gt 0 ]; do
+    if netstat -tuln | grep -q ":5900 "; then
+        break
+    fi
+    sleep 1
+    ((timeout--))
+done
+
+if [ $timeout -eq 0 ]; then
+    echo "x11vnc failed to start, stderr output:" >&2
+    cat /tmp/x11vnc_stderr.log >&2
+    exit 1
+fi
+
+: > /tmp/x11vnc_stderr.log
+
+# Monitor x11vnc process in the background
+(
+    while true; do
+        if ! kill -0 $x11vnc_pid 2>/dev/null; then
+            echo "x11vnc process crashed, restarting..." >&2
+            if [ -f /tmp/x11vnc_stderr.log ]; then
+                echo "x11vnc stderr output:" >&2
+                cat /tmp/x11vnc_stderr.log >&2
+                rm /tmp/x11vnc_stderr.log
+            fi
+            exec "$0"
+        fi
+        sleep 5
+    done
+) &

inspect_ai/tool/beta/_computer/_resources/entrypoint/xfce_startup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "starting XFCE4"
+startxfce4 &
+
+while ! pgrep -x "xfce4-session" > /dev/null; do
+  echo "Waiting for XFCE4 to start..."
+  sleep 1
+done
+
+echo "XFCE4 is fully started!"
+touch /tmp/xfce_started
+

inspect_ai/tool/beta/_computer/_resources/entrypoint/xvfb_startup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -e  # Exit on error
+
+DPI=96
+RES_AND_DEPTH=${WIDTH}x${HEIGHT}x24
+
+# Function to check if Xvfb is already running
+check_xvfb_running() {
+    if [ -e /tmp/.X${DISPLAY_NUM}-lock ]; then
+        return 0  # Xvfb is already running
+    else
+        return 1  # Xvfb is not running
+    fi
+}
+
+# Function to check if Xvfb is ready
+wait_for_xvfb() {
+    local timeout=10
+    local start_time=$(date +%s)
+    while ! xdpyinfo >/dev/null 2>&1; do
+        if [ $(($(date +%s) - start_time)) -gt $timeout ]; then
+            echo "Xvfb failed to start within $timeout seconds" >&2
+            return 1
+        fi
+        sleep 0.1
+    done
+    return 0
+}
+
+# Check if Xvfb is already running
+if check_xvfb_running; then
+    echo "Xvfb is already running on display ${DISPLAY}"
+    exit 0
+fi
+
+# Start Xvfb
+Xvfb $DISPLAY -ac -screen 0 $RES_AND_DEPTH -retro -dpi $DPI -nolisten tcp -nolisten unix &
+XVFB_PID=$!
+
+# Wait for Xvfb to start
+if wait_for_xvfb; then
+    echo "Xvfb started successfully on display ${DISPLAY}"
+    echo "Xvfb PID: $XVFB_PID"
+else
+    echo "Xvfb failed to start"
+    kill $XVFB_PID
+    exit 1
+fi

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=Firefox Web Browser
+Comment=Browse the World Wide Web
+Exec=firefox-esr %u
+Icon=firefox-esr
+Path=
+Terminal=false
+StartupNotify=true

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=Visual Studio Code
+Comment=Code Editing. Redefined.
+Exec=/usr/share/code/code %F
+Icon=vscode
+Path=
+Terminal=false
+StartupNotify=false

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/XPaint.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=XPaint
+Comment=Xpaint painting application
+Exec=xpaint
+Icon=xpaint
+Path=
+Terminal=false
+StartupNotify=false

inspect_ai/tool/beta/_computer/_resources/tool/_logger.py

@@ -0,0 +1,22 @@
+import logging
+
+
+def setup_logger(level=logging.INFO):
+    """
+    This logger emits all of its output to PID 1's stdout.
+
+    This makes it so that logging from invocations of the computer_tool cli show up in `docker logs` output.
+    """
+    new_logger = logging.getLogger("computer_tool")
+    new_logger.setLevel(level)
+
+    stdout_handler = logging.FileHandler("/proc/1/fd/1", mode="w")
+    stdout_handler.setLevel(level)
+    stdout_handler.setFormatter(
+        logging.Formatter("%(name)s(pid=%(process)d) - %(levelname)s - %(message)s")
+    )
+
+    if not new_logger.handlers:
+        new_logger.addHandler(stdout_handler)
+
+    return new_logger

inspect_ai/tool/beta/_computer/_resources/tool/_run.py

@@ -0,0 +1,42 @@
+"""Utility to run shell commands asynchronously with a timeout."""
+
+import asyncio
+
+TRUNCATED_MESSAGE: str = "<response clipped><NOTE>To save on context only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for.</NOTE>"
+MAX_RESPONSE_LEN: int = 16000
+
+
+def maybe_truncate(content: str, truncate_after: int | None = MAX_RESPONSE_LEN):
+    """Truncate content and append a notice if content exceeds the specified length."""
+    return (
+        content
+        if not truncate_after or len(content) <= truncate_after
+        else content[:truncate_after] + TRUNCATED_MESSAGE
+    )
+
+
+async def run(
+    cmd: str,
+    timeout: float | None = 120.0,  # seconds
+    truncate_after: int | None = MAX_RESPONSE_LEN,
+):
+    """Run a shell command asynchronously with a timeout."""
+    process = await asyncio.create_subprocess_shell(
+        cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+    )
+
+    try:
+        stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
+        return (
+            process.returncode or 0,
+            maybe_truncate(stdout.decode(), truncate_after=truncate_after),
+            maybe_truncate(stderr.decode(), truncate_after=truncate_after),
+        )
+    except asyncio.TimeoutError as exc:
+        try:
+            process.kill()
+        except ProcessLookupError:
+            pass
+        raise TimeoutError(
+            f"Command '{cmd}' timed out after {timeout} seconds"
+        ) from exc

inspect_ai/tool/beta/_computer/_resources/tool/_tool_result.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass, fields, replace
+
+
+@dataclass(kw_only=True, frozen=True)
+class ToolResult:
+    """Represents the result of a tool execution."""
+
+    output: str | None = None
+    error: str | None = None
+    base64_image: str | None = None
+
+    def __bool__(self):
+        return any(getattr(self, field.name) for field in fields(self))
+
+    def __add__(self, other: "ToolResult"):
+        def combine_fields(
+            field: str | None, other_field: str | None, concatenate: bool = True
+        ):
+            if field and other_field:
+                if concatenate:
+                    return field + other_field
+                raise ValueError("Cannot combine tool results")
+            return field or other_field
+
+        return ToolResult(
+            output=combine_fields(self.output, other.output),
+            error=combine_fields(self.error, other.error),
+            base64_image=combine_fields(self.base64_image, other.base64_image, False),
+        )
+
+    def replace(self, **kwargs):
+        """Returns a new ToolResult with the given fields replaced."""
+        return replace(self, **kwargs)