inspect-ai 0.3.75__py3-none-any.whl → 0.3.77__py3-none-any.whl

inspect_ai/model/_providers/anthropic.py

@@ -1,23 +1,12 @@
 import functools
 import os
 import re
-import sys
 from copy import copy
 from logging import getLogger
-from typing import Any, Literal, Optional, Tuple, TypedDict, cast
+from typing import Any, Literal, NamedTuple, Optional, Tuple, cast
 
 import httpcore
 import httpx
-
-from inspect_ai._util.http import is_retryable_http_status
-
-from .util.hooks import HttpxHooks
-
-if sys.version_info >= (3, 11):
-    from typing import NotRequired
-else:
-    from typing_extensions import NotRequired
-
 from anthropic import (
     APIConnectionError,
     APIStatusError,
@@ -39,19 +28,19 @@ from anthropic.types import (
     TextBlockParam,
     ThinkingBlock,
     ThinkingBlockParam,
+    ToolBash20250124Param,
     ToolParam,
     ToolResultBlockParam,
+    ToolTextEditor20250124Param,
     ToolUseBlock,
     ToolUseBlockParam,
     message_create_params,
 )
+from anthropic.types.beta import BetaToolComputerUse20250124Param
 from pydantic import JsonValue
 from typing_extensions import override
 
-from inspect_ai._util.constants import (
-    BASE_64_DATA_REMOVED,
-    NO_CONTENT,
-)
+from inspect_ai._util.constants import BASE_64_DATA_REMOVED, NO_CONTENT
 from inspect_ai._util.content import (
     Content,
     ContentImage,
@@ -59,6 +48,7 @@ from inspect_ai._util.content import (
     ContentText,
 )
 from inspect_ai._util.error import exception_message
+from inspect_ai._util.http import is_retryable_http_status
 from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.logger import warn_once
 from inspect_ai._util.url import data_uri_mime_type, data_uri_to_base64
@@ -70,11 +60,14 @@ from .._model import ModelAPI
 from .._model_call import ModelCall
 from .._model_output import ChatCompletionChoice, ModelOutput, ModelUsage, StopReason
 from .util import environment_prerequisite_error, model_base_url
+from .util.hooks import HttpxHooks
 
 logger = getLogger(__name__)
 
 ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY"
 
+INTERNAL_COMPUTER_TOOL_NAME = "computer"
+
 
 class AnthropicAPI(ModelAPI):
     def __init__(
@@ -93,7 +86,7 @@ class AnthropicAPI(ModelAPI):
         else:
             self.service = None
 
-        # collect gemerate model_args (then delete them so we can pass the rest on)
+        # collect generate model_args (then delete them so we can pass the rest on)
         def collect_model_arg(name: str) -> Any | None:
             nonlocal model_args
             value = model_args.get(name, None)
@@ -193,14 +186,11 @@ class AnthropicAPI(ModelAPI):
 
         # generate
         try:
-            (
-                system_param,
-                tools_param,
-                messages,
-                computer_use,
-            ) = await self.resolve_chat_input(input, tools, config)
+            system_param, tools_param, messages = await self.resolve_chat_input(
+                input, tools, config
+            )
 
-            # prepare request params (assembed this way so we can log the raw model call)
+            # prepare request params (assembled this way so we can log the raw model call)
             request = dict(messages=messages)
 
             # system messages and tools
@@ -218,7 +208,13 @@ class AnthropicAPI(ModelAPI):
 
             # extra headers (for time tracker and computer use)
             extra_headers = headers | {HttpxHooks.REQUEST_ID_HEADER: request_id}
-            if computer_use:
+            if any(
+                tool.get("type", None) == "computer_20250124" for tool in tools_param
+            ):
+                # From: https://docs.anthropic.com/en/docs/agents-and-tools/computer-use#claude-3-7-sonnet-beta-flag
+                # Note: The Bash (bash_20250124) and Text Editor (text_editor_20250124)
+                # tools are generally available for Claude 3.5 Sonnet (new) as well and
+                # can be used without the computer use beta header.
                 betas.append("computer-use-2025-01-24")
             if len(betas) > 0:
                 extra_headers["anthropic-beta"] = ",".join(betas)
@@ -405,9 +401,7 @@ class AnthropicAPI(ModelAPI):
         input: list[ChatMessage],
         tools: list[ToolInfo],
         config: GenerateConfig,
-    ) -> Tuple[
-        list[TextBlockParam] | None, list["ToolParamDef"], list[MessageParam], bool
-    ]:
+    ) -> Tuple[list[TextBlockParam] | None, list["ToolParamDef"], list[MessageParam]]:
         # extract system message
         system_messages, messages = split_system_messages(input, config)
 
@@ -420,7 +414,7 @@ class AnthropicAPI(ModelAPI):
         )
 
         # tools
-        tools_params, computer_use = self.tool_params_for_tools(tools, config)
+        tools_params = [self.tool_param_for_tool_info(tool, config) for tool in tools]
 
         # system messages
         if len(system_messages) > 0:
@@ -470,40 +464,35 @@ class AnthropicAPI(ModelAPI):
                     add_cache_control(cast(dict[str, Any], content[-1]))
 
         # return chat input
-        return system_param, tools_params, message_params, computer_use
-
-    def tool_params_for_tools(
-        self, tools: list[ToolInfo], config: GenerateConfig
-    ) -> tuple[list["ToolParamDef"], bool]:
-        # tool params and computer_use bit to return
-        tool_params: list["ToolParamDef"] = []
-        computer_use = False
-
-        # for each tool, check if it has a native computer use implementation and use that
-        # when available (noting that we need to set the computer use request header)
-        for tool in tools:
-            computer_use_tool = (
+        return system_param, tools_params, message_params
+
+    def tool_param_for_tool_info(
+        self, tool: ToolInfo, config: GenerateConfig
+    ) -> "ToolParamDef":
+        # Use a native tool implementation when available. Otherwise, use the
+        # standard tool implementation
+        return self.maybe_native_tool_param(tool, config) or ToolParam(
+            name=tool.name,
+            description=tool.description,
+            input_schema=tool.parameters.model_dump(exclude_none=True),
+        )
+
+    def maybe_native_tool_param(
+        self, tool: ToolInfo, config: GenerateConfig
+    ) -> Optional["ToolParamDef"]:
+        return (
+            (
                 self.computer_use_tool_param(tool)
-                if config.internal_tools is not False
-                else None
+                or self.text_editor_tool_param(tool)
+                or self.bash_tool_param(tool)
             )
-            if computer_use_tool:
-                tool_params.append(computer_use_tool)
-                computer_use = True
-            else:
-                tool_params.append(
-                    ToolParam(
-                        name=tool.name,
-                        description=tool.description,
-                        input_schema=tool.parameters.model_dump(exclude_none=True),
-                    )
-                )
-
-        return tool_params, computer_use
+            if config.internal_tools is not False
+            else None
+        )
 
     def computer_use_tool_param(
         self, tool: ToolInfo
-    ) -> Optional["ComputerUseToolParam"]:
+    ) -> Optional[BetaToolComputerUse20250124Param]:
         # check for compatible 'computer' tool
         if tool.name == "computer" and (
             sorted(tool.parameters.properties.keys())
@@ -525,7 +514,7 @@ class AnthropicAPI(ModelAPI):
                     "Use of Anthropic's native computer use support is not enabled in Claude 3.5. Please use 3.7 or later to leverage the native support.",
                 )
                 return None
-            return ComputerUseToolParam(
+            return BetaToolComputerUse20250124Param(
                 type="computer_20250124",
                 name="computer",
                 # Note: The dimensions passed here for display_width_px and display_height_px should
@@ -542,23 +531,58 @@ class AnthropicAPI(ModelAPI):
         else:
             return None
 
+    def text_editor_tool_param(
+        self, tool: ToolInfo
+    ) -> Optional[ToolTextEditor20250124Param]:
+        # check for compatible 'text editor' tool
+        if tool.name == "text_editor" and (
+            sorted(tool.parameters.properties.keys())
+            == sorted(
+                [
+                    "command",
+                    "file_text",
+                    "insert_line",
+                    "new_str",
+                    "old_str",
+                    "path",
+                    "view_range",
+                ]
+            )
+        ):
+            return ToolTextEditor20250124Param(
+                type="text_editor_20250124", name="str_replace_editor"
+            )
+        # not a text_editor tool
+        else:
+            return None
 
-# native anthropic tool definitions for computer use beta
-# https://docs.anthropic.com/en/docs/build-with-claude/computer-use
-class ComputerUseToolParam(TypedDict):
-    type: str
-    name: str
-    display_width_px: NotRequired[int]
-    display_height_px: NotRequired[int]
-    display_number: NotRequired[int]
+    def bash_tool_param(self, tool: ToolInfo) -> Optional[ToolBash20250124Param]:
+        # check for compatible 'bash' tool
+        if tool.name == "bash_session" and (
+            sorted(tool.parameters.properties.keys()) == sorted(["command", "restart"])
+        ):
+            return ToolBash20250124Param(type="bash_20250124", name="bash")
+        # not a bash tool
+        else:
+            return None
 
 
-# tools can be either a stock tool param or a special computer use tool param
-ToolParamDef = ToolParam | ComputerUseToolParam
+# tools can be either a stock tool param or a special Anthropic native use tool param
+ToolParamDef = (
+    ToolParam
+    | BetaToolComputerUse20250124Param
+    | ToolTextEditor20250124Param
+    | ToolBash20250124Param
+)
 
 
 def add_cache_control(
-    param: TextBlockParam | ToolParam | ComputerUseToolParam | dict[str, Any],
+    param: TextBlockParam
+    | ToolParam
+    | BetaToolComputerUse20250124Param
+    | ToolTextEditor20250124Param
+    | ToolBash20250124Param
+    | dict[str, Any],
 ) -> None:
     cast(dict[str, Any], param)["cache_control"] = {"type": "ephemeral"}
 
@@ -567,10 +591,10 @@ def consecutive_user_message_reducer(
     messages: list[MessageParam],
     message: MessageParam,
 ) -> list[MessageParam]:
-    return consective_message_reducer(messages, message, "user")
+    return consecutive_message_reducer(messages, message, "user")
 
 
-def consective_message_reducer(
+def consecutive_message_reducer(
     messages: list[MessageParam],
     message: MessageParam,
     role: Literal["user", "assistant"],
@@ -583,6 +607,7 @@ def consective_message_reducer(
 
 
 def combine_messages(a: MessageParam, b: MessageParam) -> MessageParam:
+    # TODO: Fix this code as it currently drops interesting properties when combining
     role = a["role"]
     a_content = a["content"]
     b_content = b["content"]
@@ -702,7 +727,7 @@ async def message_param(message: ChatMessage) -> MessageParam:
                 ToolUseBlockParam(
                     type="tool_use",
                     id=tool_call.id,
-                    name=tool_call.function,
+                    name=tool_call.internal_name or tool_call.function,
                     input=tool_call.arguments,
                 )
             )
@@ -749,11 +774,13 @@ async def model_output_from_message(
             content.append(ContentText(type="text", text=content_text))
         elif isinstance(content_block, ToolUseBlock):
             tool_calls = tool_calls or []
+            info = maybe_mapped_call_info(content_block.name, tools)
             tool_calls.append(
                 ToolCall(
-                    type="function",
+                    type=info.internal_type,
                     id=content_block.id,
-                    function=content_block.name,
+                    function=info.inspect_name,
+                    internal_name=info.internal_name,
                     arguments=content_block.model_dump().get("input", {}),
                 )
             )
@@ -788,6 +815,7 @@ async def model_output_from_message(
         + (input_tokens_cache_write or 0)
         + (input_tokens_cache_read or 0)
         + message.usage.output_tokens
+        + reasoning_tokens
     )
     return ModelOutput(
         model=message.model,
@@ -803,6 +831,37 @@ async def model_output_from_message(
     )
 
 
+class CallInfo(NamedTuple):
+    internal_name: str | None
+    internal_type: str
+    inspect_name: str
+
+
+def maybe_mapped_call_info(tool_called: str, tools: list[ToolInfo]) -> CallInfo:
+    """
+    Return call info - potentially transformed by native tool mappings.
+
+    Anthropic prescribes names for their native tools - `computer`, `bash`, and
+    `str_replace_editor`. For a variety of reasons, Inspect's tool names to not
+    necessarily conform to internal names. Anthropic also provides specific tool
+    types for these built-in tools.
+    """
+    mappings = (
+        (INTERNAL_COMPUTER_TOOL_NAME, "computer_20250124", "computer"),
+        ("str_replace_editor", "text_editor_20250124", "text_editor"),
+        ("bash", "bash_20250124", "bash_session"),
+    )
+
+    return next(
+        (
+            CallInfo(entry[0], entry[1], entry[2])
+            for entry in mappings
+            if entry[0] == tool_called and any(tool.name == entry[2] for tool in tools)
+        ),
+        CallInfo(None, "function", tool_called),
+    )
+
+
 def message_stop_reason(message: Message) -> StopReason:
     match message.stop_reason:
         case "end_turn" | "stop_sequence":

inspect_ai/model/_providers/azureai.py

@@ -51,7 +51,6 @@ from .._chat_message import (
     ChatMessageUser,
 )
 from .._generate_config import GenerateConfig
-from .._image import image_url_filter
 from .._model import ModelAPI
 from .._model_call import ModelCall
 from .._model_output import (
@@ -60,6 +59,7 @@ from .._model_output import (
     ModelUsage,
     StopReason,
 )
+from .._openai import openai_media_filter
 from .util import (
     environment_prerequisite_error,
     model_base_url,
@@ -182,7 +182,7 @@ class AzureAIAPI(ModelAPI):
                     else None,
                 ),
                 response=response.as_dict() if response else {},
-                filter=image_url_filter,
+                filter=openai_media_filter,
             )
 
         # make call

inspect_ai/model/_providers/mistral.py

@@ -82,6 +82,14 @@ class MistralAPI(ModelAPI):
         config: GenerateConfig = GenerateConfig(),
         **model_args: Any,
     ):
+        # extract any service prefix from model name
+        parts = model_name.split("/")
+        if len(parts) > 1:
+            self.service: str | None = parts[0]
+            model_name = "/".join(parts[1:])
+        else:
+            self.service = None
+
         super().__init__(
             model_name=model_name,
             base_url=base_url,
@@ -94,31 +102,39 @@ class MistralAPI(ModelAPI):
             config=config,
         )
 
-        # resolve api_key -- look for mistral then azure
+        # resolve api_key
         if not self.api_key:
-            self.api_key = os.environ.get(MISTRAL_API_KEY, None)
-            if self.api_key:
-                base_url = model_base_url(base_url, "MISTRAL_BASE_URL")
-            else:
+            if self.is_azure():
                 self.api_key = os.environ.get(
                     AZUREAI_MISTRAL_API_KEY, os.environ.get(AZURE_MISTRAL_API_KEY, None)
                 )
-                if not self.api_key:
-                    raise environment_prerequisite_error(
-                        "Mistral", [MISTRAL_API_KEY, AZUREAI_MISTRAL_API_KEY]
-                    )
-                base_url = model_base_url(base_url, "AZUREAI_MISTRAL_BASE_URL")
-                if not base_url:
+            else:
+                self.api_key = os.environ.get(MISTRAL_API_KEY, None)
+
+            if not self.api_key:
+                raise environment_prerequisite_error(
+                    "Mistral", [MISTRAL_API_KEY, AZUREAI_MISTRAL_API_KEY]
+                )
+
+        if not self.base_url:
+            if self.is_azure():
+                self.base_url = model_base_url(base_url, "AZUREAI_MISTRAL_BASE_URL")
+                if not self.base_url:
                     raise ValueError(
                         "You must provide a base URL when using Mistral on Azure. Use the AZUREAI_MISTRAL_BASE_URL "
                         + " environment variable or the --model-base-url CLI flag to set the base URL."
                     )
+            else:
+                self.base_url = model_base_url(base_url, "MISTRAL_BASE_URL")
 
-        if base_url:
-            model_args["server_url"] = base_url
+        if self.base_url:
+            model_args["server_url"] = self.base_url
 
         self.model_args = model_args
 
+    def is_azure(self) -> bool:
+        return self.service == "azure"
+
     @override
     async def close(self) -> None:
         # client is created and destroyed in generate

inspect_ai/model/_providers/openai.py

@@ -22,28 +22,27 @@ from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.http import is_retryable_http_status
 from inspect_ai._util.logger import warn_once
 from inspect_ai.model._openai import chat_choices_from_openai
+from inspect_ai.model._providers.openai_responses import generate_responses
 from inspect_ai.model._providers.util.hooks import HttpxHooks
 from inspect_ai.tool import ToolChoice, ToolInfo
 
 from .._chat_message import ChatMessage
 from .._generate_config import GenerateConfig
-from .._image import image_url_filter
 from .._model import ModelAPI
 from .._model_call import ModelCall
-from .._model_output import (
-    ChatCompletionChoice,
-    ModelOutput,
-    ModelUsage,
-    StopReason,
-)
+from .._model_output import ChatCompletionChoice, ModelOutput, ModelUsage
 from .._openai import (
+    OpenAIResponseError,
     is_gpt,
     is_o1_mini,
     is_o1_preview,
+    is_o1_pro,
     is_o_series,
     openai_chat_messages,
     openai_chat_tool_choice,
     openai_chat_tools,
+    openai_handle_bad_request,
+    openai_media_filter,
 )
 from .openai_o1 import generate_o1
 from .util import (
@@ -65,8 +64,22 @@ class OpenAIAPI(ModelAPI):
         base_url: str | None = None,
         api_key: str | None = None,
         config: GenerateConfig = GenerateConfig(),
+        responses_api: bool | None = None,
         **model_args: Any,
     ) -> None:
+        # extract azure service prefix from model name (other providers
+        # that subclass from us like together expect to have the qualifier
+        # in the model name e.g. google/gemma-2b-it)
+        parts = model_name.split("/")
+        if parts[0] == "azure" and len(parts) > 1:
+            self.service: str | None = parts[0]
+            model_name = "/".join(parts[1:])
+        else:
+            self.service = None
+
+        # note whether we are forcing the responses_api
+        self.responses_api = True if responses_api else False
+
         # call super
         super().__init__(
             model_name=model_name,
@@ -76,32 +89,23 @@ class OpenAIAPI(ModelAPI):
             config=config,
         )
 
-        # extract any service prefix from model name
-        parts = model_name.split("/")
-        if len(parts) > 1:
-            self.service: str | None = parts[0]
-            model_name = "/".join(parts[1:])
-        else:
-            self.service = None
-
         # resolve api_key
         if not self.api_key:
-            self.api_key = os.environ.get(
-                AZUREAI_OPENAI_API_KEY, os.environ.get(AZURE_OPENAI_API_KEY, None)
-            )
-            # backward compatibility for when env vars determined service
-            if self.api_key and (os.environ.get(OPENAI_API_KEY, None) is None):
-                self.service = "azure"
+            if self.service == "azure":
+                self.api_key = os.environ.get(
+                    AZUREAI_OPENAI_API_KEY, os.environ.get(AZURE_OPENAI_API_KEY, None)
+                )
             else:
                 self.api_key = os.environ.get(OPENAI_API_KEY, None)
-                if not self.api_key:
-                    raise environment_prerequisite_error(
-                        "OpenAI",
-                        [
-                            OPENAI_API_KEY,
-                            AZUREAI_OPENAI_API_KEY,
-                        ],
-                    )
+
+            if not self.api_key:
+                raise environment_prerequisite_error(
+                    "OpenAI",
+                    [
+                        OPENAI_API_KEY,
+                        AZUREAI_OPENAI_API_KEY,
+                    ],
+                )
 
         # create async http client
         http_client = OpenAIAsyncHttpxClient()
@@ -123,10 +127,16 @@ class OpenAIAPI(ModelAPI):
                     + "environment variable or the --model-base-url CLI flag to set the base URL."
                 )
 
+            # resolve version
+            api_version = os.environ.get(
+                "AZUREAI_OPENAI_API_VERSION",
+                os.environ.get("OPENAI_API_VERSION", "2025-02-01-preview"),
+            )
+
             self.client: AsyncAzureOpenAI | AsyncOpenAI = AsyncAzureOpenAI(
                 api_key=self.api_key,
+                api_version=api_version,
                 azure_endpoint=base_url,
-                azure_deployment=model_name,
                 http_client=http_client,
                 **model_args,
             )
@@ -147,6 +157,9 @@ class OpenAIAPI(ModelAPI):
     def is_o_series(self) -> bool:
         return is_o_series(self.model_name)
 
+    def is_o1_pro(self) -> bool:
+        return is_o1_pro(self.model_name)
+
     def is_o1_mini(self) -> bool:
         return is_o1_mini(self.model_name)
 
@@ -175,6 +188,16 @@ class OpenAIAPI(ModelAPI):
                 tools=tools,
                 **self.completion_params(config, False),
             )
+        elif self.is_o1_pro() or self.responses_api:
+            return await generate_responses(
+                client=self.client,
+                http_hooks=self._http_hooks,
+                model_name=self.model_name,
+                input=input,
+                tools=tools,
+                tool_choice=tool_choice,
+                config=config,
+            )
 
         # allocate request_id (so we can see it from ModelCall)
         request_id = self._http_hooks.start_request()
@@ -187,7 +210,7 @@ class OpenAIAPI(ModelAPI):
             return ModelCall.create(
                 request=request,
                 response=response,
-                filter=image_url_filter,
+                filter=openai_media_filter,
                 time=self._http_hooks.end_request(request_id),
             )
 
@@ -219,6 +242,7 @@ class OpenAIAPI(ModelAPI):
 
             # save response for model_call
             response = completion.model_dump()
+            self.on_response(response)
 
             # parse out choices
             choices = self._chat_choices_from_response(completion, tools)
@@ -250,6 +274,12 @@ class OpenAIAPI(ModelAPI):
         except BadRequestError as e:
             return self.handle_bad_request(e), model_call()
 
+    def on_response(self, response: dict[str, Any]) -> None:
+        pass
+
+    def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | Exception:
+        return openai_handle_bad_request(self.model_name, ex)
+
     def _chat_choices_from_response(
         self, response: ChatCompletion, tools: list[ToolInfo]
     ) -> list[ChatCompletionChoice]:
@@ -268,6 +298,8 @@ class OpenAIAPI(ModelAPI):
                 return True
         elif isinstance(ex, APIStatusError):
             return is_retryable_http_status(ex.status_code)
+        elif isinstance(ex, OpenAIResponseError):
+            return ex.code in ["rate_limit_exceeded", "server_error"]
         elif isinstance(ex, APITimeoutError):
             return True
         else:
@@ -322,6 +354,7 @@ class OpenAIAPI(ModelAPI):
             config.reasoning_effort is not None
             and not self.is_gpt()
             and not self.is_o1_mini()
+            and not self.is_o1_preview()
         ):
             params["reasoning_effort"] = config.reasoning_effort
         if config.response_schema is not None:
@@ -339,32 +372,6 @@ class OpenAIAPI(ModelAPI):
 
         return params
 
-    # convert some well known bad request errors into ModelOutput
-    def handle_bad_request(self, e: BadRequestError) -> ModelOutput | Exception:
-        # extract message
-        if isinstance(e.body, dict) and "message" in e.body.keys():
-            content = str(e.body.get("message"))
-        else:
-            content = e.message
-
-        # narrow stop_reason
-        stop_reason: StopReason | None = None
-        if e.code == "context_length_exceeded":
-            stop_reason = "model_length"
-        elif (
-            e.code == "invalid_prompt"  # seems to happen for o1/o3
-            or e.code == "content_policy_violation"  # seems to happen for vision
-            or e.code == "content_filter"  # seems to happen on azure
-        ):
-            stop_reason = "content_filter"
-
-        if stop_reason:
-            return ModelOutput.from_content(
-                model=self.model_name, content=content, stop_reason=stop_reason
-            )
-        else:
-            return e
-
 
 class OpenAIAsyncHttpxClient(httpx.AsyncClient):
     """Custom async client that deals better with long running Async requests.