inspect-ai 0.3.74__py3-none-any.whl → 0.3.76__py3-none-any.whl

inspect_ai/tool/_tool_call.py

@@ -44,8 +44,11 @@ class ToolCall:
     arguments: dict[str, Any]
     """Arguments to function."""
 
-    type: Literal["function"]
-    """Type of tool call (currently only 'function')"""
+    type: str
+    """Type of tool call ('function' or a model specific internal tool type)"""
+
+    internal_name: str | None = field(default=None)
+    """Model's internal name for the tool - if any."""
 
     parse_error: str | None = field(default=None)
     """Error which occurred parsing tool call."""

inspect_ai/tool/_tool_support_helpers.py

@@ -0,0 +1,200 @@
+"""
+This module provides helper code for handling JSON-RPC communication between the inspect process and the `inspect-tool-support` package code running in the sandbox environment.
+
+It includes definitions for JSON-RPC request and response models, as well as functions to create and parse JSON-RPC requests and responses.
+"""
+
+import json
+from itertools import count
+from textwrap import dedent
+from typing import Literal, Type, TypeVar, cast
+
+from pydantic import BaseModel, RootModel
+
+from inspect_ai._util.error import PrerequisiteError
+from inspect_ai.tool._tool import ToolError, ToolParsingError
+from inspect_ai.util import sandbox_with
+from inspect_ai.util._sandbox.environment import SandboxEnvironment
+
+
+class JSONRPCResponseBase(BaseModel):
+    jsonrpc: Literal["2.0"]
+    id: int | float | str
+
+
+class JSONRPCSuccessResponse(JSONRPCResponseBase):
+    result: object
+
+
+class JSONRPCError(BaseModel):
+    """See: https://www.jsonrpc.org/specification#error_object"""
+
+    code: int
+    message: str
+    data: object | None = None
+
+
+class JSONRPCErrorResponse(JSONRPCResponseBase):
+    error: JSONRPCError
+
+
+class JSONRPCResponse(RootModel[JSONRPCSuccessResponse | JSONRPCErrorResponse]):
+    pass
+
+
+BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
+StrOrModelT = TypeVar("StrOrModelT", bound=str | BaseModel)
+
+id_generator = count(666)
+
+
+async def exec_sandbox_rpc(
+    sandbox: SandboxEnvironment,
+    method: str,
+    params: dict[str, object] | tuple[object, ...],
+    result_cls: Type[StrOrModelT],
+    timeout: int | None = None,
+    user: str | None = None,
+) -> StrOrModelT:
+    """
+    Execute a JSON-RPC command to a sandbox environment.
+
+    Note that the JSON RPC request is sent to the exec'ed program via stdin.
+
+    Args:
+      sandbox (SandboxEnvironment): The sandbox environment to execute the command in.
+      method (str): The JSON-RPC method to call.
+      params (dict[str, object] | tuple[object, ...]): The parameters for the JSON-RPC method.
+      result_cls (Type[BaseModelT]): The class to use for parsing the result.
+      timeout (int | None, optional): The timeout for the execution. Defaults to None.
+      user: Optional username or UID to run the command as.
+
+    Returns:
+      BaseModelT: The parsed result of the JSON-RPC call.
+
+    Raises:
+      RuntimeError: If the sandbox execution fails or if there is an error in the JSON-RPC response.
+      ToolParsingError: If the JSON-RPC response contains a specific error code indicating a parsing error.
+    """
+    exec_result = await sandbox.exec(
+        [SANDBOX_CLI, "exec"],
+        input=_create_json_rpc_request(method, params),
+        timeout=timeout,
+        user=user,
+    )
+
+    if not exec_result.success:
+        raise RuntimeError(
+            f"Sandbox.exec failure executing {_rpc_call_description(method, params)}: {exec_result.stderr}"
+        )
+
+    match _parse_json_rpc_response(exec_result.stdout, result_cls):
+        case JSONRPCError(code=-32601 | -32602, message=message):
+            raise ToolParsingError(message)
+        case JSONRPCError(code=-32000, message=message):
+            raise ToolError(message)
+        case JSONRPCError(code=code, message=message):
+            raise RuntimeError(
+                f"Error executing tool command {_rpc_call_description(method, params)}: {code=} {message}"
+            )
+        # case result_cls() as model: yields a mypy error since it has narrowed model down
+        # to BaseModel and not BaseModelT. ???
+        case model if isinstance(model, result_cls):
+            return model
+        case not_possible:
+            raise RuntimeError(
+                f"Error executing tool command {_rpc_call_description(method, params)}: {not_possible}"
+            )
+
+
+SANDBOX_CLI = "inspect-tool-support"
+INSPECT_TOOL_SUPPORT_IMAGE_DOCKERHUB = "aisiuk/inspect-tool-support"
+
+
+async def tool_container_sandbox(tool_name: str) -> SandboxEnvironment:
+    sb = await sandbox_with(SANDBOX_CLI, True)
+    if sb:
+        return sb
+    else:
+        msg = dedent(f"""
+                The {tool_name} service was not found in any of the sandboxes for this sample. Please add the {tool_name} to your configuration.
+
+                For example, the following Docker compose file uses the {INSPECT_TOOL_SUPPORT_IMAGE_DOCKERHUB} reference image as its default sandbox:
+
+                services:
+                  default:
+                    image: "{INSPECT_TOOL_SUPPORT_IMAGE_DOCKERHUB}"
+                    init: true
+
+                Alternatively, you can include the service into your own Dockerfile:
+
+                RUN python -m venv /opt/inspect_tool_support
+                ENV PATH="/opt/inspect_tool_support/bin:$PATH"
+                RUN pip install inspect-tool-support
+                RUN inspect-tool-support post-install
+                """).strip()
+        raise PrerequisiteError(msg)
+
+
+def _create_json_rpc_request(
+    method: str, params: dict[str, object] | tuple[object, ...]
+) -> str:
+    return json.dumps(
+        {
+            "jsonrpc": "2.0",
+            "method": method,
+            "id": next(id_generator),
+            "params": list(params) if isinstance(params, tuple) else params,
+        }
+    )
+
+
+def _rpc_call_description(
+    method: str, params: dict[str, object] | tuple[object, ...]
+) -> str:
+    """
+    Generate a string description of an RPC call.
+
+    Args:
+        method (str): The name of the RPC method.
+        params (dict[str, object] | tuple[object, ...]): The parameters for the RPC method.
+
+    Returns:
+        str: A string description of the RPC call.
+
+    Examples:
+        >>> _rpc_call_description("subtract", {"minuend": 42, "subtrahend": 23})
+        'subtract(minuend: 42, subtrahend: 23)'
+
+        >>> _rpc_call_description("subtract", (42, 23))
+        'subtract(42, 23)'
+    """
+    normalized_params = (
+        list(map(str, params))
+        if isinstance(params, tuple)
+        else [f"{k}: {v}" for k, v in params.items()]
+    )
+    return f"{method}({', '.join(normalized_params)})"
+
+
+def _parse_json_rpc_response(
+    response_str: str,
+    result_cls: Type[StrOrModelT],
+) -> StrOrModelT | JSONRPCError:
+    match JSONRPCResponse.model_validate_json(response_str).root:
+        case JSONRPCErrorResponse(error=error):
+            return error
+        case JSONRPCSuccessResponse(result=rpc_result):
+            # TODO: Wow. Is there really no way to convince Python to narrow these types
+            # and avoid the cast's
+            if result_cls is str:
+                if not isinstance(rpc_result, str):
+                    raise ValueError(f"Expected string result, got {type(rpc_result)}")
+                return cast(StrOrModelT, rpc_result)
+            else:
+                return cast(
+                    StrOrModelT,
+                    cast(BaseModel, result_cls).model_validate(rpc_result, strict=True),
+                )
+        case _:
+            raise ValueError(f"Unexpected JSON RPC response: {response_str}")

inspect_ai/tool/_tools/_bash_session.py

@@ -0,0 +1,119 @@
+from pydantic import BaseModel, Field, RootModel
+
+from inspect_ai.tool import ToolResult
+from inspect_ai.tool._tool_support_helpers import (
+    exec_sandbox_rpc,
+    tool_container_sandbox,
+)
+from inspect_ai.util import StoreModel, store_as
+
+from .._tool import Tool, ToolParsingError, tool
+from .._tool_call import ToolCall, ToolCallContent, ToolCallView, ToolCallViewer
+
+
+# These models are cloned from the container code. If/when we decide to create
+# a package that is shared between the inspect and tool-container codebases, we'll
+# just have to live with it.
+class NewSessionResult(BaseModel):
+    session_name: str
+
+
+class BashRestartResult(BaseModel):
+    pass
+
+
+class BashCommandResult(BaseModel):
+    status: int
+    stdout: str
+    stderr: str
+
+
+class BashResult(RootModel[BashRestartResult | BashCommandResult]):
+    pass
+
+
+class BashSessionStore(StoreModel):
+    session_id: str = Field(default_factory=str)
+
+
+# custom viewer for bash
+def code_viewer(language: str, code_param: str) -> ToolCallViewer:
+    def viewer(tool_call: ToolCall) -> ToolCallView:
+        code = tool_call.arguments.get(code_param, None)
+        code = (code or tool_call.function).strip()
+        call = ToolCallContent(
+            title=language,
+            format="markdown",
+            content=f"```{language}\n" + code + "\n```\n",
+        )
+        return ToolCallView(call=call)
+
+    return viewer
+
+
+@tool(viewer=code_viewer("bash", "command"))
+def bash_session(timeout: int | None = None) -> Tool:
+    """Bash shell session command execution tool.
+
+    Execute bash shell commands in a long running session using a sandbox environment (e.g. "docker").
+
+    Args:
+      timeout: Timeout (in seconds) for command.
+
+    Returns:
+      String with command output (stdout) or command error (stderr).
+    """
+
+    async def execute(
+        command: str | None = None,
+        restart: bool | None = None,
+    ) -> ToolResult:
+        """
+        Use this function to execute bash commands.
+
+        Args:
+          command: The bash command to run. Required unless the tool is being restarted.
+          restart: Specifying true will restart this tool. Otherwise, leave this unspecified.
+
+        Returns:
+          The output of the command.
+        """
+        if not ((command is None) ^ (restart is None)):
+            raise ToolParsingError(
+                "Either 'command' or 'restart' must be specified, but not both."
+            )
+        params: dict[str, object] = {"command": command, "restart": restart}
+
+        sandbox = await tool_container_sandbox("bash session")
+        store = store_as(BashSessionStore)
+
+        if not store.session_id:
+            store.session_id = (
+                await exec_sandbox_rpc(
+                    sandbox,
+                    "bash_session_new_session",
+                    {},
+                    NewSessionResult,
+                    timeout=timeout,
+                )
+            ).session_name
+
+        params["session_name"] = store.session_id
+
+        result = (
+            await exec_sandbox_rpc(
+                sandbox,
+                "bash_session",
+                params,
+                BashResult,
+                timeout=timeout,
+            )
+        ).root
+
+        if isinstance(result, BashRestartResult):
+            return "Bash session restarted."
+
+        # return output (including stderr if any)
+        return f"{result.stderr}\n{result.stdout}" if result.stderr else result.stdout
+
+    return execute

inspect_ai/tool/_tools/_computer/_computer.py

@@ -15,7 +15,7 @@ ActionFunction = Callable[[str], ToolResult | Awaitable[ToolResult]]
 def computer(max_screenshots: int | None = 1, timeout: int | None = 180) -> Tool:
     """Desktop computer tool.
 
-    See documentation at <https://inspect.ai-safety-institute.org.uk/tools.html#sec-computer>.
+    See documentation at <https://inspect.aisi.org.uk/tools-standard.html#sec-computer>.
 
     Args:
       max_screenshots: The maximum number of screenshots to play

inspect_ai/tool/_tools/_text_editor.py

@@ -0,0 +1,121 @@
+import inspect
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, Discriminator, RootModel
+
+from inspect_ai.tool import ToolResult
+from inspect_ai.tool._tool_support_helpers import (
+    exec_sandbox_rpc,
+    tool_container_sandbox,
+)
+
+from .._tool import Tool, tool
+
+# These models are cloned from the container code. If/when we decide to create
+# a package that is shared between the inspect and tool-container codebases, we'll
+# just have to live with it.
+
+
+class BaseParams(BaseModel):
+    path: str
+
+
+class ViewParams(BaseParams):
+    command: Literal["view"] = "view"
+    view_range: list[int] | None = None
+
+
+class CreateParams(BaseParams):
+    command: Literal["create"] = "create"
+    file_text: str
+
+
+class StrReplaceParams(BaseParams):
+    command: Literal["str_replace"] = "str_replace"
+    old_str: str
+    new_str: str | None = None
+
+
+class InsertParams(BaseParams):
+    command: Literal["insert"] = "insert"
+    insert_line: int
+    new_str: str
+
+
+class UndoEditParams(BaseParams):
+    command: Literal["undo_edit"] = "undo_edit"
+
+
+class TextEditorParams(
+    RootModel[
+        ViewParams | CreateParams | StrReplaceParams | InsertParams | UndoEditParams
+    ]
+):
+    root: Annotated[
+        ViewParams | CreateParams | StrReplaceParams | InsertParams | UndoEditParams,
+        Discriminator("command"),
+    ]
+
+
+TextEditorResult = str
+
+
+@tool()
+def text_editor(timeout: int | None = None, user: str | None = None) -> Tool:
+    """Custom editing tool for viewing, creating and editing files.
+
+    Perform text editor operations using a sandbox environment (e.g. "docker").
+
+    IMPORTANT: This tool does not currently support Subtask isolation. This means
+    that a change made to a file by on Subtask will be visible to another Subtask.
+
+    Args:
+      timeout: Timeout (in seconds) for command.
+      user: User to execute commands as.
+
+    Returns:
+      String with command output (stdout) or command error (stderr).
+    """
+
+    async def execute(
+        command: Literal["view", "create", "str_replace", "insert", "undo_edit"],
+        path: str,
+        file_text: str | None = None,
+        insert_line: int | None = None,
+        new_str: str | None = None,
+        old_str: str | None = None,
+        view_range: list[int] | None = None,
+    ) -> ToolResult:
+        """
+        Use this function to execute text editing commands.
+
+        Args:
+          command: The command to execute.
+          path: Absolute path to file or directory, e.g. `/repo/file.py` or `/repo`.
+          file_text: Required parameter of `create` command, with the content of the file to be created.
+          insert_line: Required parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`.
+          new_str: Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert.
+          old_str: Required parameter of `str_replace` command containing the string in `path` to replace.
+          view_range: Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [11, 12] will show lines 11 and 12. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file.
+
+        Returns:
+          The output of the command.
+        """
+        sandbox = await tool_container_sandbox("editor")
+
+        # Create a dictionary of the parameters
+        params = {
+            k: v
+            for k, v in locals().items()
+            if k in inspect.signature(execute).parameters
+        }
+
+        return await exec_sandbox_rpc(
+            sandbox,
+            "text_editor",
+            params,
+            TextEditorResult,
+            timeout=timeout,
+        )
+
+    return execute

inspect_ai/tool/_tools/_web_browser/_back_compat.py

@@ -0,0 +1,150 @@
+"""This module provides the "old" client code for running against the, now deprecated, `aisiuk/inspect-web-browser-tool` image."""
+
+import re
+from logging import getLogger
+from textwrap import dedent
+
+from pydantic import Field
+
+from inspect_ai._util.content import ContentText
+from inspect_ai._util.error import PrerequisiteError
+from inspect_ai._util.logger import warn_once
+from inspect_ai.tool import ToolError, ToolResult
+from inspect_ai.util import SandboxEnvironment, StoreModel, sandbox_with, store_as
+from inspect_ai.util._sandbox.docker.internal import (
+    INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB_DEPRECATED,
+)
+
+logger = getLogger("web_browser")
+
+WEB_CLIENT_REQUEST = "/app/web_browser/web_client.py"
+WEB_CLIENT_NEW_SESSION = "/app/web_browser/web_client_new_session.py"
+
+
+class WebBrowserStore(StoreModel):
+    main_content: str = Field(default_factory=str)
+    web_at: str = Field(default_factory=str)
+    session_id: str = Field(default_factory=str)
+
+
+async def old_web_browser_cmd(cmd: str, *args: str) -> ToolResult:
+    sandbox_env = await _web_browser_sandbox()
+    warn_once(
+        logger,
+        "WARNING: Use of the `aisiuk/inspect-web-browser-tool` image is deprecated. Please update your configuration to use the `aisiuk/inspect-tool-support` image or install the `inspect-tool-support` package into your own image.",
+    )
+
+    store = store_as(WebBrowserStore)
+    if not store.session_id:
+        result = await sandbox_env.exec(
+            ["python3", WEB_CLIENT_NEW_SESSION], timeout=180
+        )
+
+        if not result.success:
+            raise RuntimeError(
+                f"Error creating new web browser session: {result.stderr}"
+            )
+
+        store.session_id = result.stdout.strip("\n")
+
+    session_flag = f"--session_name={store.session_id}"
+
+    arg_list = None
+    if session_flag:
+        arg_list = ["python3", WEB_CLIENT_REQUEST, session_flag, cmd] + list(args)
+    else:
+        arg_list = ["python3", WEB_CLIENT_REQUEST, cmd] + list(args)
+
+    result = await sandbox_env.exec(arg_list, timeout=180)
+    if not result.success:
+        raise RuntimeError(
+            f"Error executing web browser command {cmd}({', '.join(args)}): {result.stderr}"
+        )
+    else:
+        response = _parse_web_browser_output(result.stdout)
+        if "error" in response and response.get("error", "").strip() != "":
+            raise ToolError(str(response.get("error")) or "(unknown error)")
+        elif "web_at" in response:
+            main_content = str(response.get("main_content")) or None
+            web_at = (
+                str(response.get("web_at")) or "(no web accessibility tree available)"
+            )
+            # Remove base64 data from images.
+            web_at_lines = web_at.split("\n")
+            web_at_lines = [
+                line.partition("data:image/png;base64")[0] for line in web_at_lines
+            ]
+
+            store_as(WebBrowserStore).main_content = (
+                main_content or "(no main text summary)"
+            )
+            store_as(WebBrowserStore).web_at = web_at
+
+            web_at = "\n".join(web_at_lines)
+            return (
+                [
+                    ContentText(text=f"main content:\n{main_content}\n\n"),
+                    ContentText(text=f"accessibility tree:\n{web_at}"),
+                ]
+                if main_content
+                else web_at
+            )
+        else:
+            raise RuntimeError(
+                f"web_browser output must contain either 'error' or 'web_at' field: {result.stdout}"
+            )
+
+
+async def _web_browser_sandbox() -> SandboxEnvironment:
+    sb = await sandbox_with(WEB_CLIENT_REQUEST)
+    if sb:
+        return sb
+    else:
+        msg = dedent(f"""
+                The web browser service was not found in any of the sandboxes for this sample. Please add the web browser service to your configuration. For example, the following Docker compose file uses the {INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB_DEPRECATED} image as its default sandbox:
+
+                services:
+                  default:
+                    image: "{INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB_DEPRECATED}"
+                    init: true
+
+                Alternatively, this Docker compose file creates a dedicated image for the web browser service:
+
+                services:
+                  default:
+                    image: "python:3.12-bookworm"
+                    init: true
+                    command: "tail -f /dev/null"
+
+                  web_browser:
+                    image: "{INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB_DEPRECATED}"
+                    init: true
+                """).strip()
+        raise PrerequisiteError(msg)
+
+
+def _parse_web_browser_output(output: str) -> dict[str, str]:
+    response: dict[str, str] = dict(
+        web_url="", main_content="", web_at="", info="", error=""
+    )
+    active_field: str | None = None
+    active_field_lines: list[str] = []
+
+    def collect_active_field() -> None:
+        if active_field is not None:
+            response[active_field] = "\n".join(active_field_lines)
+        active_field_lines.clear()
+
+    for line in output.splitlines():
+        field_match = re.match(
+            r"^(error|main_content|web_at|web_url|info)\s*:\s*(.+)$", line
+        )
+        if field_match:
+            collect_active_field()
+            active_field = field_match.group(1)
+            active_field_lines.append(field_match.group(2))
+        else:
+            active_field_lines.append(line)
+    collect_active_field()
+
+    return response