inspect-ai 0.3.75__py3-none-any.whl → 0.3.77__py3-none-any.whl

inspect_ai/model/_providers/openai_responses.py

@@ -0,0 +1,177 @@
+from logging import getLogger
+from typing import Any
+
+from openai import (
+    AsyncAzureOpenAI,
+    AsyncOpenAI,
+    BadRequestError,
+)
+from openai._types import NOT_GIVEN
+from openai.types.responses import Response, ResponseFormatTextJSONSchemaConfigParam
+
+from inspect_ai._util.logger import warn_once
+from inspect_ai.tool import ToolChoice, ToolInfo
+
+from .._chat_message import ChatMessage
+from .._generate_config import GenerateConfig
+from .._model_call import ModelCall
+from .._model_output import (
+    ModelOutput,
+    ModelUsage,
+)
+from .._openai import (
+    OpenAIResponseError,
+    is_gpt,
+    is_o1_mini,
+    is_o1_preview,
+    is_o_series,
+    openai_handle_bad_request,
+    openai_media_filter,
+)
+from .._openai_responses import (
+    openai_responses_chat_choices,
+    openai_responses_inputs,
+    openai_responses_tool_choice,
+    openai_responses_tools,
+)
+from .util.hooks import HttpxHooks
+
+logger = getLogger(__name__)
+
+
+async def generate_responses(
+    client: AsyncAzureOpenAI | AsyncOpenAI,
+    http_hooks: HttpxHooks,
+    model_name: str,
+    input: list[ChatMessage],
+    tools: list[ToolInfo],
+    tool_choice: ToolChoice,
+    config: GenerateConfig,
+) -> ModelOutput | tuple[ModelOutput | Exception, ModelCall]:
+    # allocate request_id (so we can see it from ModelCall)
+    request_id = http_hooks.start_request()
+
+    # setup request and response for ModelCall
+    request: dict[str, Any] = {}
+    response: dict[str, Any] = {}
+
+    def model_call() -> ModelCall:
+        return ModelCall.create(
+            request=request,
+            response=response,
+            # TODO: is this the right filter?
+            filter=openai_media_filter,
+            time=http_hooks.end_request(request_id),
+        )
+
+    # prepare request (we do this so we can log the ModelCall)
+    request = dict(
+        input=await openai_responses_inputs(input, model_name),
+        tools=openai_responses_tools(tools) if len(tools) > 0 else NOT_GIVEN,
+        tool_choice=openai_responses_tool_choice(tool_choice)
+        if len(tools) > 0
+        else NOT_GIVEN,
+        extra_headers={HttpxHooks.REQUEST_ID_HEADER: request_id},
+        **completion_params_responses(model_name, config, len(tools) > 0),
+    )
+
+    try:
+        # generate response
+        model_response: Response = await client.responses.create(**request)
+
+        # check for error
+        if model_response.error is not None:
+            raise OpenAIResponseError(
+                code=model_response.error.code, message=model_response.error.message
+            )
+
+        # save response for model_call
+        response = model_response.model_dump()
+
+        # parse out choices
+        choices = openai_responses_chat_choices(model_response, tools)
+
+        # return output and call
+        return ModelOutput(
+            model=model_response.model,
+            choices=choices,
+            usage=(
+                ModelUsage(
+                    input_tokens=model_response.usage.input_tokens,
+                    output_tokens=model_response.usage.output_tokens,
+                    input_tokens_cache_read=(
+                        model_response.usage.input_tokens_details.cached_tokens
+                    ),
+                    reasoning_tokens=model_response.usage.output_tokens_details.reasoning_tokens,
+                    total_tokens=model_response.usage.total_tokens,
+                )
+                if model_response.usage
+                else None
+            ),
+        ), model_call()
+    except BadRequestError as e:
+        return openai_handle_bad_request(model_name, e), model_call()
+
+
+def completion_params_responses(
+    model_name: str, config: GenerateConfig, tools: bool
+) -> dict[str, Any]:
+    # TODO: we'll need a computer_use_preview bool for the 'include'
+    # and 'reasoning' parameters
+    def unsupported_warning(param: str) -> None:
+        warn_once(
+            logger,
+            f"OpenAI Responses API does not support the '{param}' parameter.",
+        )
+
+    params: dict[str, Any] = dict(model=model_name, store=False)
+    if config.max_tokens is not None:
+        params["max_output_tokens"] = config.max_tokens
+    if config.frequency_penalty is not None:
+        unsupported_warning("frequency_penalty")
+    if config.stop_seqs is not None:
+        unsupported_warning("stop_seqs")
+    if config.presence_penalty is not None:
+        unsupported_warning("presence_penalty")
+    if config.logit_bias is not None:
+        unsupported_warning("logit_bias")
+    if config.seed is not None:
+        unsupported_warning("seed")
+    if config.temperature is not None:
+        if is_o_series(model_name):
+            warn_once(
+                logger,
+                "o series models do not support the 'temperature' parameter (temperature is always 1).",
+            )
+        else:
+            params["temperature"] = config.temperature
+    if config.top_p is not None:
+        params["top_p"] = config.top_p
+    if config.num_choices is not None:
+        unsupported_warning("num_choices")
+    if config.logprobs is not None:
+        unsupported_warning("logprobs")
+    if config.top_logprobs is not None:
+        unsupported_warning("top_logprobs")
+    if tools and config.parallel_tool_calls is not None and not is_o_series(model_name):
+        params["parallel_tool_calls"] = config.parallel_tool_calls
+    if (
+        config.reasoning_effort is not None
+        and not is_gpt(model_name)
+        and not is_o1_mini(model_name)
+        and not is_o1_preview(model_name)
+    ):
+        params["reasoning"] = dict(effort=config.reasoning_effort)
+    if config.response_schema is not None:
+        params["text"] = dict(
+            format=ResponseFormatTextJSONSchemaConfigParam(
+                type="json_schema",
+                name=config.response_schema.name,
+                schema=config.response_schema.json_schema.model_dump(exclude_none=True),
+                description=config.response_schema.description
+                or config.response_schema.name,
+                strict=config.response_schema.strict,
+            )
+        )
+
+    return params

inspect_ai/model/_providers/openrouter.py

@@ -1,9 +1,11 @@
+import json
 import os
-from typing import Any
+from typing import Any, TypedDict
 
-from typing_extensions import override
+from typing_extensions import NotRequired, override
 
 from inspect_ai._util.error import PrerequisiteError
+from inspect_ai.model._openai import OpenAIResponseError
 from inspect_ai.model._providers.util import model_base_url
 from inspect_ai.model._providers.util.util import environment_prerequisite_error
 
@@ -13,6 +15,28 @@ from .openai import OpenAIAPI
 OPENROUTER_API_KEY = "OPENROUTER_API_KEY"
 
 
+class ErrorResponse(TypedDict):
+    code: int
+    message: str
+    metadata: NotRequired[dict[str, Any]]
+
+
+class OpenRouterError(Exception):
+    def __init__(self, response: ErrorResponse) -> None:
+        self.response = response
+
+    @property
+    def message(self) -> str:
+        return f"Error {self.response['code']} - {self.response['message']}"
+
+    def __str__(self) -> str:
+        return (
+            self.message + ("\n" + json.dumps(self.response["metadata"], indent=2))
+            if "metadata" in self.response
+            else ""
+        )
+
+
 class OpenRouterAPI(OpenAIAPI):
     def __init__(
         self,
@@ -67,6 +91,32 @@ class OpenRouterAPI(OpenAIAPI):
             **model_args,
         )
 
+    @override
+    def on_response(self, response: dict[str, Any]) -> None:
+        """Handle documented OpenRouter error conditions.
+
+        https://openrouter.ai/docs/api-reference/errors
+        """
+        # check if open-router yielded an error (raise explicit
+        # OpenAIResponseError for cases where we should retry)
+        error: ErrorResponse | None = response.get("error", None)
+        if error is not None:
+            if error["code"] == 429:
+                raise OpenAIResponseError("rate_limit_exceeded", error["message"])
+            elif error["code"] in [408, 502]:
+                raise OpenAIResponseError("server_error", error["message"])
+            else:
+                raise OpenRouterError(error)
+
+        # check for an empty response (which they document can occur on
+        # startup). for this we'll return a "server_error" which will
+        # trigger a retry w/ exponential backoff
+        elif response.get("choices", None) is None:
+            raise OpenAIResponseError(
+                "server_error",
+                "Model is warming up, please retry again after waiting for warmup.",
+            )
+
     @override
     def completion_params(self, config: GenerateConfig, tools: bool) -> dict[str, Any]:
         # default params

inspect_ai/model/_providers/providers.py

@@ -282,7 +282,7 @@ def goodfire() -> type[ModelAPI]:
 def validate_openai_client(feature: str) -> None:
     FEATURE = feature
     PACKAGE = "openai"
-    MIN_VERSION = "1.58.1"
+    MIN_VERSION = "1.68.0"
 
     # verify we have the package
     try:

inspect_ai/model/_providers/vertex.py

@@ -34,8 +34,8 @@ from inspect_ai._util.content import (
     Content,
     ContentAudio,
     ContentImage,
+    ContentReasoning,
     ContentText,
-    ContentVideo,
 )
 from inspect_ai._util.http import is_retryable_http_status
 from inspect_ai._util.images import file_as_data
@@ -336,10 +336,13 @@ async def content_part(content: Content | str) -> Part:
     elif isinstance(content, ContentImage):
         image_bytes, mime_type = await file_as_data(content.image)
         return Part.from_image(image=Image.from_bytes(data=image_bytes))
+    elif isinstance(content, ContentReasoning):
+        return Part.from_text(content.reasoning or NO_CONTENT)
     else:
         if isinstance(content, ContentAudio):
             file = content.audio
-        elif isinstance(content, ContentVideo):
+        else:
+            # it's ContentVideo
             file = content.video
         file_bytes, mime_type = await file_as_data(file)
         return Part.from_data(file_bytes, mime_type)

inspect_ai/tool/__init__.py

@@ -22,17 +22,23 @@ from ._tool_def import ToolDef
 from ._tool_info import ToolInfo
 from ._tool_params import ToolParam, ToolParams
 from ._tool_with import tool_with
+from ._tools._bash_session import bash_session
 from ._tools._computer import computer
 from ._tools._execute import bash, python
+from ._tools._text_editor import text_editor
+from ._tools._think import think
 from ._tools._web_browser import web_browser
 from ._tools._web_search import web_search
 
 __all__ = [
     "bash",
+    "bash_session",
     "computer",
     "python",
     "web_browser",
     "web_search",
+    "think",
+    "text_editor",
     "tool",
     "tool_with",
     "Tool",

inspect_ai/tool/_tool.py

@@ -20,6 +20,7 @@ from inspect_ai._util.content import (
 )
 from inspect_ai._util.registry import (
     RegistryInfo,
+    is_registry_object,
     registry_add,
     registry_name,
     registry_tag,
@@ -200,7 +201,25 @@ def tool(
         # wrap instantiations of scorer so they carry registry info and metrics
         @wraps(tool_type)
         def tool_wrapper(*args: P.args, **kwargs: P.kwargs) -> Tool:
+            # create the tool
             tool = tool_type(*args, **kwargs)
+
+            # this might already have registry info, in that case
+            # capture it and use it as defaults
+            from inspect_ai.tool._tool_def import tool_registry_info
+
+            tool_parallel = parallel
+            tool_viewer = viewer
+            tool_model_input = model_input
+            if is_registry_object(tool):
+                _, _, reg_parallel, reg_viewer, reg_model_input = tool_registry_info(
+                    tool
+                )
+                tool_parallel = parallel and reg_parallel
+                tool_viewer = viewer or reg_viewer
+                tool_model_input = model_input or reg_model_input
+
+            # tag the object
             registry_tag(
                 tool_type,
                 tool,
@@ -209,10 +228,11 @@ def tool(
                     name=tool_name,
                     metadata={
                         TOOL_PROMPT: prompt,
-                        TOOL_PARALLEL: parallel,
-                        TOOL_VIEWER: viewer,
+                        TOOL_PARALLEL: tool_parallel,
+                        TOOL_VIEWER: tool_viewer,
                         TOOL_MODEL_INPUT: (
-                            model_input or getattr(tool, TOOL_INIT_MODEL_INPUT, None)
+                            tool_model_input
+                            or getattr(tool, TOOL_INIT_MODEL_INPUT, None)
                         ),
                     },
                 ),

inspect_ai/tool/_tool_call.py

@@ -44,8 +44,11 @@ class ToolCall:
     arguments: dict[str, Any]
     """Arguments to function."""
 
-    type: Literal["function"]
-    """Type of tool call (currently only 'function')"""
+    type: str
+    """Type of tool call ('function' or a model specific internal tool type)"""
+
+    internal_name: str | None = field(default=None)
+    """Model's internal name for the tool - if any."""
 
     parse_error: str | None = field(default=None)
     """Error which occurred parsing tool call."""

inspect_ai/tool/_tool_support_helpers.py

@@ -0,0 +1,200 @@
+"""
+This module provides helper code for handling JSON-RPC communication between the inspect process and the `inspect-tool-support` package code running in the sandbox environment.
+
+It includes definitions for JSON-RPC request and response models, as well as functions to create and parse JSON-RPC requests and responses.
+"""
+
+import json
+from itertools import count
+from textwrap import dedent
+from typing import Literal, Type, TypeVar, cast
+
+from pydantic import BaseModel, RootModel
+
+from inspect_ai._util.error import PrerequisiteError
+from inspect_ai.tool._tool import ToolError, ToolParsingError
+from inspect_ai.util import sandbox_with
+from inspect_ai.util._sandbox.environment import SandboxEnvironment
+
+
+class JSONRPCResponseBase(BaseModel):
+    jsonrpc: Literal["2.0"]
+    id: int | float | str
+
+
+class JSONRPCSuccessResponse(JSONRPCResponseBase):
+    result: object
+
+
+class JSONRPCError(BaseModel):
+    """See: https://www.jsonrpc.org/specification#error_object"""
+
+    code: int
+    message: str
+    data: object | None = None
+
+
+class JSONRPCErrorResponse(JSONRPCResponseBase):
+    error: JSONRPCError
+
+
+class JSONRPCResponse(RootModel[JSONRPCSuccessResponse | JSONRPCErrorResponse]):
+    pass
+
+
+BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
+StrOrModelT = TypeVar("StrOrModelT", bound=str | BaseModel)
+
+id_generator = count(666)
+
+
+async def exec_sandbox_rpc(
+    sandbox: SandboxEnvironment,
+    method: str,
+    params: dict[str, object] | tuple[object, ...],
+    result_cls: Type[StrOrModelT],
+    timeout: int | None = None,
+    user: str | None = None,
+) -> StrOrModelT:
+    """
+    Execute a JSON-RPC command to a sandbox environment.
+
+    Note that the JSON RPC request is sent to the exec'ed program via stdin.
+
+    Args:
+      sandbox (SandboxEnvironment): The sandbox environment to execute the command in.
+      method (str): The JSON-RPC method to call.
+      params (dict[str, object] | tuple[object, ...]): The parameters for the JSON-RPC method.
+      result_cls (Type[BaseModelT]): The class to use for parsing the result.
+      timeout (int | None, optional): The timeout for the execution. Defaults to None.
+      user: Optional username or UID to run the command as.
+
+    Returns:
+      BaseModelT: The parsed result of the JSON-RPC call.
+
+    Raises:
+      RuntimeError: If the sandbox execution fails or if there is an error in the JSON-RPC response.
+      ToolParsingError: If the JSON-RPC response contains a specific error code indicating a parsing error.
+    """
+    exec_result = await sandbox.exec(
+        [SANDBOX_CLI, "exec"],
+        input=_create_json_rpc_request(method, params),
+        timeout=timeout,
+        user=user,
+    )
+
+    if not exec_result.success:
+        raise RuntimeError(
+            f"Sandbox.exec failure executing {_rpc_call_description(method, params)}: {exec_result.stderr}"
+        )
+
+    match _parse_json_rpc_response(exec_result.stdout, result_cls):
+        case JSONRPCError(code=-32601 | -32602, message=message):
+            raise ToolParsingError(message)
+        case JSONRPCError(code=-32000, message=message):
+            raise ToolError(message)
+        case JSONRPCError(code=code, message=message):
+            raise RuntimeError(
+                f"Error executing tool command {_rpc_call_description(method, params)}: {code=} {message}"
+            )
+        # case result_cls() as model: yields a mypy error since it has narrowed model down
+        # to BaseModel and not BaseModelT. ???
+        case model if isinstance(model, result_cls):
+            return model
+        case not_possible:
+            raise RuntimeError(
+                f"Error executing tool command {_rpc_call_description(method, params)}: {not_possible}"
+            )
+
+
+SANDBOX_CLI = "inspect-tool-support"
+INSPECT_TOOL_SUPPORT_IMAGE_DOCKERHUB = "aisiuk/inspect-tool-support"
+
+
+async def tool_container_sandbox(tool_name: str) -> SandboxEnvironment:
+    sb = await sandbox_with(SANDBOX_CLI, True)
+    if sb:
+        return sb
+    else:
+        msg = dedent(f"""
+                The {tool_name} service was not found in any of the sandboxes for this sample. Please add the {tool_name} to your configuration.
+
+                For example, the following Docker compose file uses the {INSPECT_TOOL_SUPPORT_IMAGE_DOCKERHUB} reference image as its default sandbox:
+
+                services:
+                  default:
+                    image: "{INSPECT_TOOL_SUPPORT_IMAGE_DOCKERHUB}"
+                    init: true
+
+                Alternatively, you can include the service into your own Dockerfile:
+
+                RUN python -m venv /opt/inspect_tool_support
+                ENV PATH="/opt/inspect_tool_support/bin:$PATH"
+                RUN pip install inspect-tool-support
+                RUN inspect-tool-support post-install
+                """).strip()
+        raise PrerequisiteError(msg)
+
+
+def _create_json_rpc_request(
+    method: str, params: dict[str, object] | tuple[object, ...]
+) -> str:
+    return json.dumps(
+        {
+            "jsonrpc": "2.0",
+            "method": method,
+            "id": next(id_generator),
+            "params": list(params) if isinstance(params, tuple) else params,
+        }
+    )
+
+
+def _rpc_call_description(
+    method: str, params: dict[str, object] | tuple[object, ...]
+) -> str:
+    """
+    Generate a string description of an RPC call.
+
+    Args:
+        method (str): The name of the RPC method.
+        params (dict[str, object] | tuple[object, ...]): The parameters for the RPC method.
+
+    Returns:
+        str: A string description of the RPC call.
+
+    Examples:
+        >>> _rpc_call_description("subtract", {"minuend": 42, "subtrahend": 23})
+        'subtract(minuend: 42, subtrahend: 23)'
+
+        >>> _rpc_call_description("subtract", (42, 23))
+        'subtract(42, 23)'
+    """
+    normalized_params = (
+        list(map(str, params))
+        if isinstance(params, tuple)
+        else [f"{k}: {v}" for k, v in params.items()]
+    )
+    return f"{method}({', '.join(normalized_params)})"
+
+
+def _parse_json_rpc_response(
+    response_str: str,
+    result_cls: Type[StrOrModelT],
+) -> StrOrModelT | JSONRPCError:
+    match JSONRPCResponse.model_validate_json(response_str).root:
+        case JSONRPCErrorResponse(error=error):
+            return error
+        case JSONRPCSuccessResponse(result=rpc_result):
+            # TODO: Wow. Is there really no way to convince Python to narrow these types
+            # and avoid the cast's
+            if result_cls is str:
+                if not isinstance(rpc_result, str):
+                    raise ValueError(f"Expected string result, got {type(rpc_result)}")
+                return cast(StrOrModelT, rpc_result)
+            else:
+                return cast(
+                    StrOrModelT,
+                    cast(BaseModel, result_cls).model_validate(rpc_result, strict=True),
+                )
+        case _:
+            raise ValueError(f"Unexpected JSON RPC response: {response_str}")

inspect_ai/tool/_tools/_bash_session.py

@@ -0,0 +1,119 @@
+from pydantic import BaseModel, Field, RootModel
+
+from inspect_ai.tool import ToolResult
+from inspect_ai.tool._tool_support_helpers import (
+    exec_sandbox_rpc,
+    tool_container_sandbox,
+)
+from inspect_ai.util import StoreModel, store_as
+
+from .._tool import Tool, ToolParsingError, tool
+from .._tool_call import ToolCall, ToolCallContent, ToolCallView, ToolCallViewer
+
+
+# These models are cloned from the container code. If/when we decide to create
+# a package that is shared between the inspect and tool-container codebases, we'll
+# just have to live with it.
+class NewSessionResult(BaseModel):
+    session_name: str
+
+
+class BashRestartResult(BaseModel):
+    pass
+
+
+class BashCommandResult(BaseModel):
+    status: int
+    stdout: str
+    stderr: str
+
+
+class BashResult(RootModel[BashRestartResult | BashCommandResult]):
+    pass
+
+
+class BashSessionStore(StoreModel):
+    session_id: str = Field(default_factory=str)
+
+
+# custom viewer for bash
+def code_viewer(language: str, code_param: str) -> ToolCallViewer:
+    def viewer(tool_call: ToolCall) -> ToolCallView:
+        code = tool_call.arguments.get(code_param, None)
+        code = (code or tool_call.function).strip()
+        call = ToolCallContent(
+            title=language,
+            format="markdown",
+            content=f"```{language}\n" + code + "\n```\n",
+        )
+        return ToolCallView(call=call)
+
+    return viewer
+
+
+@tool(viewer=code_viewer("bash", "command"))
+def bash_session(timeout: int | None = None) -> Tool:
+    """Bash shell session command execution tool.
+
+    Execute bash shell commands in a long running session using a sandbox environment (e.g. "docker").
+
+    Args:
+      timeout: Timeout (in seconds) for command.
+
+    Returns:
+      String with command output (stdout) or command error (stderr).
+    """
+
+    async def execute(
+        command: str | None = None,
+        restart: bool | None = None,
+    ) -> ToolResult:
+        """
+        Use this function to execute bash commands.
+
+        Args:
+          command: The bash command to run. Required unless the tool is being restarted.
+          restart: Specifying true will restart this tool. Otherwise, leave this unspecified.
+
+        Returns:
+          The output of the command.
+        """
+        if not ((command is None) ^ (restart is None)):
+            raise ToolParsingError(
+                "Either 'command' or 'restart' must be specified, but not both."
+            )
+        params: dict[str, object] = {"command": command, "restart": restart}
+
+        sandbox = await tool_container_sandbox("bash session")
+        store = store_as(BashSessionStore)
+
+        if not store.session_id:
+            store.session_id = (
+                await exec_sandbox_rpc(
+                    sandbox,
+                    "bash_session_new_session",
+                    {},
+                    NewSessionResult,
+                    timeout=timeout,
+                )
+            ).session_name
+
+        params["session_name"] = store.session_id
+
+        result = (
+            await exec_sandbox_rpc(
+                sandbox,
+                "bash_session",
+                params,
+                BashResult,
+                timeout=timeout,
+            )
+        ).root
+
+        if isinstance(result, BashRestartResult):
+            return "Bash session restarted."
+
+        # return output (including stderr if any)
+        return f"{result.stderr}\n{result.stdout}" if result.stderr else result.stdout
+
+    return execute