inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/tool/beta/_computer/_resources/entrypoint/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# remove marker files
+rm -f /tmp/.X${DISPLAY_NUM}-lock 
+rm -f /tmp/xfce_started
+
+/opt/inspect/entrypoint/xvfb_startup.sh
+/opt/inspect/entrypoint/xfce_startup.sh
+/opt/inspect/entrypoint/x11vnc_startup.sh
+/opt/inspect/entrypoint/novnc_startup.sh
+
+# Run CMD if provided
+echo "Executing CMD from derived Dockerfile: $@"
+exec "$@"
+
+# Keep the container running
+tail -f /dev/null

inspect_ai/tool/beta/_computer/_resources/entrypoint/novnc_startup.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+echo "starting noVNC"
+
+# Start noVNC with explicit websocket settings
+websockify \
+    --web=/usr/share/novnc/ \
+    6080 localhost:5900 \
+    > /tmp/novnc.log 2>&1 &
+
+# Wait for noVNC to start
+timeout=10
+while [ $timeout -gt 0 ]; do
+    if netstat -tuln | grep -q ":6080 "; then
+        break
+    fi
+    sleep 1
+    ((timeout--))
+done
+
+echo "noVNC started successfully"

inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+echo "starting vnc"
+
+(x11vnc -display $DISPLAY \
+    -forever \
+    -shared \
+    -wait 50 \
+    -cursor most \
+    -cursor arrow \
+    -rfbport 5900 \
+    -nopw \
+    2>/tmp/x11vnc_stderr.log) &
+
+x11vnc_pid=$!
+
+# Wait for x11vnc to start
+timeout=10
+while [ $timeout -gt 0 ]; do
+    if netstat -tuln | grep -q ":5900 "; then
+        break
+    fi
+    sleep 1
+    ((timeout--))
+done
+
+if [ $timeout -eq 0 ]; then
+    echo "x11vnc failed to start, stderr output:" >&2
+    cat /tmp/x11vnc_stderr.log >&2
+    exit 1
+fi
+
+: > /tmp/x11vnc_stderr.log
+
+# Monitor x11vnc process in the background
+(
+    while true; do
+        if ! kill -0 $x11vnc_pid 2>/dev/null; then
+            echo "x11vnc process crashed, restarting..." >&2
+            if [ -f /tmp/x11vnc_stderr.log ]; then
+                echo "x11vnc stderr output:" >&2
+                cat /tmp/x11vnc_stderr.log >&2
+                rm /tmp/x11vnc_stderr.log
+            fi
+            exec "$0"
+        fi
+        sleep 5
+    done
+) &

inspect_ai/tool/beta/_computer/_resources/entrypoint/xfce_startup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "starting XFCE4"
+startxfce4 &
+
+while ! pgrep -x "xfce4-session" > /dev/null; do
+  echo "Waiting for XFCE4 to start..."
+  sleep 1
+done
+
+echo "XFCE4 is fully started!"
+touch /tmp/xfce_started
+

inspect_ai/tool/beta/_computer/_resources/entrypoint/xvfb_startup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -e  # Exit on error
+
+DPI=96
+RES_AND_DEPTH=${WIDTH}x${HEIGHT}x24
+
+# Function to check if Xvfb is already running
+check_xvfb_running() {
+    if [ -e /tmp/.X${DISPLAY_NUM}-lock ]; then
+        return 0  # Xvfb is already running
+    else
+        return 1  # Xvfb is not running
+    fi
+}
+
+# Function to check if Xvfb is ready
+wait_for_xvfb() {
+    local timeout=10
+    local start_time=$(date +%s)
+    while ! xdpyinfo >/dev/null 2>&1; do
+        if [ $(($(date +%s) - start_time)) -gt $timeout ]; then
+            echo "Xvfb failed to start within $timeout seconds" >&2
+            return 1
+        fi
+        sleep 0.1
+    done
+    return 0
+}
+
+# Check if Xvfb is already running
+if check_xvfb_running; then
+    echo "Xvfb is already running on display ${DISPLAY}"
+    exit 0
+fi
+
+# Start Xvfb
+Xvfb $DISPLAY -ac -screen 0 $RES_AND_DEPTH -retro -dpi $DPI -nolisten tcp -nolisten unix &
+XVFB_PID=$!
+
+# Wait for Xvfb to start
+if wait_for_xvfb; then
+    echo "Xvfb started successfully on display ${DISPLAY}"
+    echo "Xvfb PID: $XVFB_PID"
+else
+    echo "Xvfb failed to start"
+    kill $XVFB_PID
+    exit 1
+fi

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=Firefox Web Browser
+Comment=Browse the World Wide Web
+Exec=firefox-esr %u
+Icon=firefox-esr
+Path=
+Terminal=false
+StartupNotify=true

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=Visual Studio Code
+Comment=Code Editing. Redefined.
+Exec=/usr/share/code/code %F
+Icon=vscode
+Path=
+Terminal=false
+StartupNotify=false

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/XPaint.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=XPaint
+Comment=Xpaint painting application
+Exec=xpaint
+Icon=xpaint
+Path=
+Terminal=false
+StartupNotify=false

inspect_ai/tool/beta/_computer/_resources/tool/_logger.py

@@ -0,0 +1,22 @@
+import logging
+
+
+def setup_logger(level=logging.INFO):
+    """
+    This logger emits all of its output to PID 1's stdout.
+
+    This makes it so that logging from invocations of the computer_tool cli show up in `docker logs` output.
+    """
+    new_logger = logging.getLogger("computer_tool")
+    new_logger.setLevel(level)
+
+    stdout_handler = logging.FileHandler("/proc/1/fd/1", mode="w")
+    stdout_handler.setLevel(level)
+    stdout_handler.setFormatter(
+        logging.Formatter("%(name)s(pid=%(process)d) - %(levelname)s - %(message)s")
+    )
+
+    if not new_logger.handlers:
+        new_logger.addHandler(stdout_handler)
+
+    return new_logger

inspect_ai/tool/beta/_computer/_resources/tool/_run.py

@@ -0,0 +1,42 @@
+"""Utility to run shell commands asynchronously with a timeout."""
+
+import asyncio
+
+TRUNCATED_MESSAGE: str = "<response clipped><NOTE>To save on context only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for.</NOTE>"
+MAX_RESPONSE_LEN: int = 16000
+
+
+def maybe_truncate(content: str, truncate_after: int | None = MAX_RESPONSE_LEN):
+    """Truncate content and append a notice if content exceeds the specified length."""
+    return (
+        content
+        if not truncate_after or len(content) <= truncate_after
+        else content[:truncate_after] + TRUNCATED_MESSAGE
+    )
+
+
+async def run(
+    cmd: str,
+    timeout: float | None = 120.0,  # seconds
+    truncate_after: int | None = MAX_RESPONSE_LEN,
+):
+    """Run a shell command asynchronously with a timeout."""
+    process = await asyncio.create_subprocess_shell(
+        cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+    )
+
+    try:
+        stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
+        return (
+            process.returncode or 0,
+            maybe_truncate(stdout.decode(), truncate_after=truncate_after),
+            maybe_truncate(stderr.decode(), truncate_after=truncate_after),
+        )
+    except asyncio.TimeoutError as exc:
+        try:
+            process.kill()
+        except ProcessLookupError:
+            pass
+        raise TimeoutError(
+            f"Command '{cmd}' timed out after {timeout} seconds"
+        ) from exc

inspect_ai/tool/beta/_computer/_resources/tool/_tool_result.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass, fields, replace
+
+
+@dataclass(kw_only=True, frozen=True)
+class ToolResult:
+    """Represents the result of a tool execution."""
+
+    output: str | None = None
+    error: str | None = None
+    base64_image: str | None = None
+
+    def __bool__(self):
+        return any(getattr(self, field.name) for field in fields(self))
+
+    def __add__(self, other: "ToolResult"):
+        def combine_fields(
+            field: str | None, other_field: str | None, concatenate: bool = True
+        ):
+            if field and other_field:
+                if concatenate:
+                    return field + other_field
+                raise ValueError("Cannot combine tool results")
+            return field or other_field
+
+        return ToolResult(
+            output=combine_fields(self.output, other.output),
+            error=combine_fields(self.error, other.error),
+            base64_image=combine_fields(self.base64_image, other.base64_image, False),
+        )
+
+    def replace(self, **kwargs):
+        """Returns a new ToolResult with the given fields replaced."""
+        return replace(self, **kwargs)

inspect_ai/tool/beta/_computer/_resources/tool/_x11_client.py

@@ -0,0 +1,262 @@
+"""Based on https://github.com/anthropics/anthropic-quickstarts/blob/main/computer-use-demo/computer_use_demo/tools/computer.py"""
+
+import asyncio
+import base64
+import logging
+import os
+import shlex
+from pathlib import Path
+from typing import Literal, TypedDict
+from uuid import uuid4
+
+from _run import run
+from _tool_result import ToolResult
+
+OUTPUT_DIR = "/tmp/outputs"
+
+TYPING_DELAY_MS = 12
+TYPING_GROUP_SIZE = 50
+
+ColorCount = Literal[4096, 2048, 1024, 512, 256, 128, 64, 32, 16, 8, 4]
+
+Action = Literal[
+    "key",
+    "type",
+    "mouse_move",
+    "left_click",
+    "left_click_drag",
+    "right_click",
+    "middle_click",
+    "double_click",
+    "screenshot",
+    "cursor_position",
+]
+
+
+class ToolError(Exception):
+    def __init__(self, message):
+        self.message = message
+
+
+class Resolution(TypedDict):
+    width: int
+    height: int
+
+
+# sizes above XGA/WXGA are not recommended (see README.md)
+# scale down to one of these targets if ComputerTool._scaling_enabled is set
+MAX_SCALING_TARGETS: dict[str, Resolution] = {
+    "XGA": Resolution(width=1024, height=768),  # 4:3
+    "WXGA": Resolution(width=1280, height=800),  # 16:10
+    "FWXGA": Resolution(width=1366, height=768),  # ~16:9
+}
+
+
+ScalingSource = Literal["computer", "api"]
+
+
+class ComputerToolOptions(TypedDict):
+    display_height_px: int
+    display_width_px: int
+    display_number: int | None
+
+
+def chunks(s: str, chunk_size: int) -> list[str]:
+    return [s[i : i + chunk_size] for i in range(0, len(s), chunk_size)]
+
+
+class X11Client:
+    """
+    A tool that allows the agent to interact with the screen, keyboard, and mouse of the current computer.
+
+    The tool parameters are defined by Anthropic and are not editable.
+    """
+
+    width: int
+    height: int
+    display_num: int | None
+    # TODO: Complete plumbing this or remove it
+    color_count: ColorCount | None = 256
+
+    _screenshot_delay = 2.0
+    _scaling_enabled = True
+
+    @property
+    def options(self) -> ComputerToolOptions:
+        width, height = self.scale_coordinates("computer", self.width, self.height)
+        return {
+            "display_width_px": width,
+            "display_height_px": height,
+            "display_number": self.display_num,
+        }
+
+    def __init__(self):
+        super().__init__()
+
+        self.width = int(os.getenv("WIDTH") or 0)
+        self.height = int(os.getenv("HEIGHT") or 0)
+        assert self.width and self.height, "WIDTH, HEIGHT must be set"
+        if (display_num := os.getenv("DISPLAY_NUM")) is not None:
+            self.display_num = int(display_num)
+            self._display_prefix = f"DISPLAY=:{self.display_num} "
+        else:
+            self.display_num = None
+            self._display_prefix = ""
+
+        self.xdotool = f"{self._display_prefix}xdotool"
+
+    async def __call__(
+        self,
+        *,
+        action: Action,
+        text: str | None = None,
+        coordinate: tuple[int, int] | None = None,
+        **kwargs,
+    ):
+        if action in ("mouse_move", "left_click_drag"):
+            if coordinate is None:
+                raise ToolError(f"coordinate is required for {action}")
+            if text is not None:
+                raise ToolError(f"text is not accepted for {action}")
+            if not isinstance(coordinate, list) or len(coordinate) != 2:
+                raise ToolError(f"{coordinate} must be a tuple of length 2")
+            if not all(isinstance(i, int) and i >= 0 for i in coordinate):
+                raise ToolError(f"{coordinate} must be a tuple of non-negative ints")
+
+            x, y = self.scale_coordinates("api", coordinate[0], coordinate[1])
+
+            if action == "mouse_move":
+                return await self.shell(f"{self.xdotool} mousemove --sync {x} {y}")
+            elif action == "left_click_drag":
+                return await self.shell(
+                    f"{self.xdotool} mousedown 1 mousemove --sync {x} {y} mouseup 1"
+                )
+
+        if action in ("key", "type"):
+            if text is None:
+                raise ToolError(f"text is required for {action}")
+            if coordinate is not None:
+                raise ToolError(f"coordinate is not accepted for {action}")
+            if not isinstance(text, str):
+                raise ToolError(output=f"{text} must be a string")
+
+            if action == "key":
+                return await self.shell(
+                    f"{self.xdotool} key -- {' '.join(shlex.quote(part) for part in text.split())}"
+                )
+            elif action == "type":
+                results: list[ToolResult] = []
+                for chunk in chunks(text, TYPING_GROUP_SIZE):
+                    cmd = f"{self.xdotool} type --delay {TYPING_DELAY_MS} -- {shlex.quote(chunk)}"
+                    results.append(await self.shell(cmd, take_screenshot=False))
+
+                screenshot_base64 = await self.take_screenshot_after_delay()
+                return ToolResult(
+                    output="".join(result.output or "" for result in results),
+                    error="".join(result.error or "" for result in results),
+                    base64_image=screenshot_base64,
+                )
+
+        if action in (
+            "left_click",
+            "right_click",
+            "double_click",
+            "middle_click",
+            "screenshot",
+            "cursor_position",
+        ):
+            if text is not None:
+                raise ToolError(f"text is not accepted for {action}")
+            if coordinate is not None:
+                raise ToolError(f"coordinate is not accepted for {action}")
+
+            if action == "screenshot":
+                return await self.screenshot()
+            elif action == "cursor_position":
+                result = await self.shell(
+                    f"{self.xdotool} getmouselocation --shell",
+                    take_screenshot=False,
+                )
+                output = result.output or ""
+                x, y = self.scale_coordinates(
+                    "computer",
+                    int(output.split("X=")[1].split("\n")[0]),
+                    int(output.split("Y=")[1].split("\n")[0]),
+                )
+                return result.replace(output=f"X={x},Y={y}")
+            else:
+                click_arg = {
+                    "left_click": "1",
+                    "right_click": "3",
+                    "middle_click": "2",
+                    "double_click": "--repeat 2 --delay 300 1",
+                }[action]
+                return await self.shell(f"{self.xdotool} click {click_arg}")
+
+        raise ToolError(f"Invalid action: {action}")
+
+    async def screenshot(self):
+        """Take a screenshot of the current screen and return the base64 encoded image."""
+        output_dir = Path(OUTPUT_DIR)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        path = output_dir / f"screenshot_{uuid4().hex}.png"
+
+        result = await self.shell(
+            f"{self._display_prefix}scrot --silent -p {path}", take_screenshot=False
+        )
+        if self._scaling_enabled:
+            x, y = self.scale_coordinates("computer", self.width, self.height)
+            convert_cmd = f"convert {path} -resize {x}x{y}!"
+            if self.color_count is not None:
+                convert_cmd += f" -colors {self.color_count}"
+            convert_cmd += f" {path}"
+            await self.shell(convert_cmd, take_screenshot=False)
+
+        if path.exists():
+            return result.replace(
+                base64_image=base64.b64encode(path.read_bytes()).decode()
+            )
+        raise ToolError(f"Failed to take screenshot: {result.error}")
+
+    async def shell(self, command: str, take_screenshot=True) -> ToolResult:
+        """Run a shell command and return the output, error, and optionally a screenshot."""
+        logging.debug(f"running shell command {command}")
+        _, stdout, stderr = await run(command)
+        logging.debug(f"shell command returned stdout: {stdout}, stderr: {stderr}")
+        return ToolResult(
+            output=stdout,
+            error=stderr,
+            base64_image=(await self.take_screenshot_after_delay())
+            if take_screenshot
+            else None,
+        )
+
+    async def take_screenshot_after_delay(self) -> str:
+        # delay to let things settle before taking a screenshot
+        await asyncio.sleep(self._screenshot_delay)
+        return (await self.screenshot()).base64_image
+
+    def scale_coordinates(self, source: ScalingSource, x: int, y: int):
+        """Scale coordinates to a target maximum resolution."""
+        if not self._scaling_enabled:
+            return x, y
+        ratio = self.width / self.height
+        target_dimension = None
+        for dimension in MAX_SCALING_TARGETS.values():
+            # allow some error in the aspect ratio - not ratios are exactly 16:9
+            if abs(dimension["width"] / dimension["height"] - ratio) < 0.02:
+                if dimension["width"] < self.width:
+                    target_dimension = dimension
+                break
+        if target_dimension is None:
+            return x, y
+        # should be less than 1
+        x_scaling_factor = target_dimension["width"] / self.width
+        y_scaling_factor = target_dimension["height"] / self.height
+        if source == "api":
+            if x > self.width or y > self.height:
+                raise ToolError(f"Coordinates {x}, {y} are out of bounds")
+            # scale up
+            return round(x / x_scaling_factor), round(y / y_scaling_factor)
+        # scale down
+        return round(x * x_scaling_factor), round(y * y_scaling_factor)

inspect_ai/tool/beta/_computer/_resources/tool/computer_tool.py

@@ -0,0 +1,85 @@
+import argparse
+import asyncio
+import json
+import logging
+import os
+import sys
+import time
+
+from _logger import setup_logger
+from _tool_result import ToolResult
+from _x11_client import X11Client
+
+# This is a bit sketchy. We really want to use relative imports here. Using absolute imports
+# works at runtime, but it prevents intellisense from working. However, when this folder is
+# copied to the container, by default relative imports won't work if this file is launched
+# normally. To overcome this, two things need to happen:
+# 1. PYTHONPATH must be set to the parent of the container folder. `PYTHONPATH=/opt`
+# 2. The program must be launched with the -m flag. `python3 -m computer_tool.computer_tool`
+#
+# TODO: There's got to be a cleaner way.
+
+my_logger = setup_logger(logging.INFO)
+
+
+def main():
+    try:
+        args = parse_arguments()
+        my_logger.info(f"({args})")
+        result = asyncio.run(execute_action(args))
+
+        print(
+            json.dumps(
+                {
+                    "output": result.output,
+                    "error": result.error,
+                    "base64_image": result.base64_image,
+                }
+            )
+        )
+        my_logger.debug("SUCCESS")
+    except Exception as e:
+        my_logger.warning(f"An error occurred: {e}")
+        print(f"An error occurred: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Execute computer tool action")
+    parser.add_argument("--action", type=str, required=True, help="Action to perform")
+    parser.add_argument("--text", type=str, help="Optional text parameter")
+    parser.add_argument(
+        "--coordinate",
+        type=int,
+        nargs=2,
+        help="Optional coordinate parameter as a list of two integers",
+    )
+    return parser.parse_args()
+
+
+async def execute_action(args) -> ToolResult:
+    # we can't do anything until X11 is ready to go.
+    await wait_for_file("/tmp/xfce_started")
+
+    computer = X11Client()
+    return await computer(
+        action=args.action,
+        text=args.text,
+        coordinate=args.coordinate if args.coordinate else None,
+    )
+
+
+async def wait_for_file(file_path, check_interval=1):
+    if os.path.exists(file_path):
+        return
+    my_logger.info(f"Waiting for {file_path}")
+    start_time = time.time()
+    while not os.path.exists(file_path):
+        await asyncio.sleep(check_interval)
+    my_logger.info(
+        f"Done waiting for {file_path} after {time.time() - start_time:.1f} seconds"
+    )
+
+
+if __name__ == "__main__":
+    main()

inspect_ai/util/__init__.py

@@ -3,6 +3,7 @@ from inspect_ai._util.trace import trace_action, trace_message
 from ._concurrency import concurrency
 from ._console import input_screen
 from ._display import DisplayType, display_type
+from ._limit import SampleLimitExceededError
 from ._panel import InputPanel, input_panel
 from ._resource import resource
 from ._sandbox import (
@@ -26,7 +27,6 @@ from ._subprocess import (
 )
 from ._subtask import Subtask, subtask
 from ._throttle import throttle
-from ._trace import trace_enabled, trace_panel
 
 __all__ = [
     "ExecResult",
@@ -37,6 +37,7 @@ __all__ = [
     "input_panel",
     "input_screen",
     "OutputLimitExceededError",
+    "SampleLimitExceededError",
     "resource",
     "subprocess",
     "SandboxEnvironment",
@@ -56,8 +57,6 @@ __all__ = [
     "Subtask",
     "subtask",
     "throttle",
-    "trace_enabled",
-    "trace_panel",
     "trace_action",
     "trace_message",
 ]