inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/model/_providers/anthropic.py

@@ -1,8 +1,14 @@
 import functools
 import os
+import sys
 from copy import copy
 from logging import getLogger
-from typing import Any, Literal, Tuple, cast
+from typing import Any, Literal, Tuple, TypedDict, cast
+
+if sys.version_info >= (3, 11):
+    from typing import NotRequired
+else:
+    from typing_extensions import NotRequired
 
 from anthropic import (
     APIConnectionError,
@@ -27,28 +33,23 @@ from anthropic.types import (
 from pydantic import JsonValue
 from typing_extensions import override
 
-from inspect_ai._util.constants import BASE_64_DATA_REMOVED, DEFAULT_MAX_RETRIES
-from inspect_ai._util.content import Content, ContentText
+from inspect_ai._util.constants import (
+    BASE_64_DATA_REMOVED,
+    DEFAULT_MAX_RETRIES,
+    NO_CONTENT,
+)
+from inspect_ai._util.content import Content, ContentImage, ContentText
 from inspect_ai._util.error import exception_message
-from inspect_ai._util.images import image_as_data_uri
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.logger import warn_once
-from inspect_ai._util.url import data_uri_mime_type, data_uri_to_base64, is_data_uri
+from inspect_ai._util.url import data_uri_mime_type, data_uri_to_base64
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
 
-from .._chat_message import (
-    ChatMessage,
-    ChatMessageAssistant,
-    ChatMessageSystem,
-)
+from .._chat_message import ChatMessage, ChatMessageAssistant, ChatMessageSystem
 from .._generate_config import GenerateConfig
 from .._model import ModelAPI
 from .._model_call import ModelCall
-from .._model_output import (
-    ChatCompletionChoice,
-    ModelOutput,
-    ModelUsage,
-    StopReason,
-)
+from .._model_output import ChatCompletionChoice, ModelOutput, ModelUsage, StopReason
 from .util import environment_prerequisite_error, model_base_url
 
 logger = getLogger(__name__)
@@ -124,7 +125,7 @@ class AnthropicAPI(ModelAPI):
         tools: list[ToolInfo],
         tool_choice: ToolChoice,
         config: GenerateConfig,
-    ) -> ModelOutput | tuple[ModelOutput, ModelCall]:
+    ) -> ModelOutput | tuple[ModelOutput | Exception, ModelCall]:
         # setup request and response for ModelCall
         request: dict[str, Any] = {}
         response: dict[str, Any] = {}
@@ -142,7 +143,7 @@ class AnthropicAPI(ModelAPI):
                 system_param,
                 tools_param,
                 messages,
-                cache_prompt,
+                computer_use,
             ) = await resolve_chat_input(self.model_name, input, tools, config)
 
             # prepare request params (assembed this way so we can log the raw model call)
@@ -158,13 +159,11 @@ class AnthropicAPI(ModelAPI):
             # additional options
             request = request | self.completion_params(config)
 
-            # caching header
-            if cache_prompt:
-                request["extra_headers"] = {
-                    "anthropic-beta": "prompt-caching-2024-07-31"
-                }
+            # computer use beta
+            if computer_use:
+                request["extra_headers"] = {"anthropic-beta": "computer-use-2024-10-22"}
 
-            # call model
+            # make request
             message = await self.client.messages.create(**request, stream=False)
 
             # set response for ModelCall
@@ -177,11 +176,7 @@ class AnthropicAPI(ModelAPI):
             return output, model_call()
 
         except BadRequestError as ex:
-            error_output = self.handle_bad_request(ex)
-            if error_output is not None:
-                return error_output, model_call()
-            else:
-                raise ex
+            return self.handle_bad_request(ex), model_call()
 
     def completion_params(self, config: GenerateConfig) -> dict[str, Any]:
         params = dict(model=self.model_name, max_tokens=cast(int, config.max_tokens))
@@ -234,7 +229,7 @@ class AnthropicAPI(ModelAPI):
         return True
 
     # convert some common BadRequestError states into 'refusal' model output
-    def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | None:
+    def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | Exception:
         error = exception_message(ex).lower()
         content: str | None = None
         stop_reason: StopReason | None = None
@@ -256,6 +251,9 @@ class AnthropicAPI(ModelAPI):
         elif "content filtering" in error:
             content = "Sorry, but I am unable to help with that request."
             stop_reason = "content_filter"
+        else:
+            content = error
+            stop_reason = "unknown"
 
         if content and stop_reason:
             return ModelOutput.from_content(
@@ -265,7 +263,21 @@ class AnthropicAPI(ModelAPI):
                 error=error,
             )
         else:
-            return None
+            return ex
+
+
+# native anthropic tool definitions for computer use beta
+# https://docs.anthropic.com/en/docs/build-with-claude/computer-use
+class ComputerUseToolParam(TypedDict):
+    type: str
+    name: str
+    display_width_px: NotRequired[int]
+    display_height_px: NotRequired[int]
+    display_number: NotRequired[int]
+
+
+# tools can be either a stock tool param or a special computer use tool param
+ToolParamDef = ToolParam | ComputerUseToolParam
 
 
 async def resolve_chat_input(
@@ -273,7 +285,7 @@ async def resolve_chat_input(
     input: list[ChatMessage],
     tools: list[ToolInfo],
     config: GenerateConfig,
-) -> Tuple[list[TextBlockParam] | None, list[ToolParam], list[MessageParam], bool]:
+) -> Tuple[list[TextBlockParam] | None, list[ToolParamDef], list[MessageParam], bool]:
     # extract system message
     system_messages, messages = split_system_messages(input, config)
 
@@ -286,14 +298,7 @@ async def resolve_chat_input(
     )
 
     # tools
-    tools_params = [
-        ToolParam(
-            name=tool.name,
-            description=tool.description,
-            input_schema=tool.parameters.model_dump(exclude_none=True),
-        )
-        for tool in tools
-    ]
+    tools_params, computer_use = tool_params_for_tools(tools, config)
 
     # system messages
     if len(system_messages) > 0:
@@ -343,10 +348,66 @@ async def resolve_chat_input(
                 add_cache_control(cast(dict[str, Any], content[-1]))
 
     # return chat input
-    return system_param, tools_params, message_params, cache_prompt
+    return system_param, tools_params, message_params, computer_use
+
+
+def tool_params_for_tools(
+    tools: list[ToolInfo], config: GenerateConfig
+) -> tuple[list[ToolParamDef], bool]:
+    # tool params and computer_use bit to return
+    tool_params: list[ToolParamDef] = []
+    computer_use = False
+
+    # for each tool, check if it has a native computer use implementation and use that
+    # when available (noting that we need to set the computer use request header)
+    for tool in tools:
+        computer_use_tool = (
+            computer_use_tool_param(tool)
+            if config.internal_tools is not False
+            else None
+        )
+        if computer_use_tool:
+            tool_params.append(computer_use_tool)
+            computer_use = True
+        else:
+            tool_params.append(
+                ToolParam(
+                    name=tool.name,
+                    description=tool.description,
+                    input_schema=tool.parameters.model_dump(exclude_none=True),
+                )
+            )
+
+    return tool_params, computer_use
+
 
+def computer_use_tool_param(tool: ToolInfo) -> ComputerUseToolParam | None:
+    # check for compatible 'computer' tool
+    if tool.name == "computer" and (
+        sorted(tool.parameters.properties.keys())
+        == sorted(["action", "coordinate", "text"])
+    ):
+        return ComputerUseToolParam(
+            type="computer_20241022",
+            name="computer",
+            # Note: The dimensions passed here for display_width_px and display_height_px should
+            # match the dimensions of screenshots returned by the tool.
+            # Those dimensions will always be one of the values in MAX_SCALING_TARGETS
+            # in _x11_client.py.
+            # TODO: enhance this code to calculate the dimensions based on the scaled screen
+            # size used by the container.
+            display_width_px=1366,
+            display_height_px=768,
+            display_number=1,
+        )
+    # not a computer_use tool
+    else:
+        return None
 
-def add_cache_control(param: TextBlockParam | ToolParam | dict[str, Any]) -> None:
+
+def add_cache_control(
+    param: TextBlockParam | ToolParam | ComputerUseToolParam | dict[str, Any],
+) -> None:
     cast(dict[str, Any], param)["cache_control"] = {"type": "ephemeral"}
 
 
@@ -404,11 +465,6 @@ def message_tool_choice(tool_choice: ToolChoice) -> message_create_params.ToolCh
         return {"type": "auto"}
 
 
-# text we insert when there is no content passed
-# (as this will result in an Anthropic API error)
-NO_CONTENT = "(no content)"
-
-
 async def message_param(message: ChatMessage) -> MessageParam:
     # no system role for anthropic (this is more like an assertion,
     # as these should have already been filtered out)
@@ -584,11 +640,9 @@ async def message_param_content(
 ) -> TextBlockParam | ImageBlockParam:
     if isinstance(content, ContentText):
         return TextBlockParam(type="text", text=content.text or NO_CONTENT)
-    else:
+    elif isinstance(content, ContentImage):
         # resolve to url
-        image = content.image
-        if not is_data_uri(image):
-            image = await image_as_data_uri(image)
+        image = await file_as_data_uri(content.image)
 
         # resolve mime type and base64 content
         media_type = data_uri_mime_type(image) or "image/png"
@@ -601,6 +655,10 @@ async def message_param_content(
             type="image",
             source=dict(type="base64", media_type=cast(Any, media_type), data=image),
         )
+    else:
+        raise RuntimeError(
+            "Anthropic models do not currently support audio or video inputs."
+        )
 
 
 def model_call_filter(key: JsonValue | None, value: JsonValue) -> JsonValue:

inspect_ai/model/_providers/azureai.py

@@ -31,8 +31,8 @@ from azure.core.exceptions import AzureError, HttpResponseError
 from typing_extensions import override
 
 from inspect_ai._util.constants import DEFAULT_MAX_TOKENS
-from inspect_ai._util.content import Content, ContentText
-from inspect_ai._util.images import image_as_data_uri
+from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai.tool import ToolChoice, ToolInfo
 from inspect_ai.tool._tool_call import ToolCall
 from inspect_ai.tool._tool_choice import ToolFunction
@@ -130,7 +130,7 @@ class AzureAIAPI(ModelAPI):
         tools: list[ToolInfo],
         tool_choice: ToolChoice,
         config: GenerateConfig,
-    ) -> ModelOutput | tuple[ModelOutput, ModelCall]:
+    ) -> ModelOutput | tuple[ModelOutput | Exception, ModelCall]:
         # emulate tools (auto for llama, opt-in for others)
         if self.emulate_tools is None and self.is_llama():
             handler: ChatAPIHandler | None = Llama31Handler()
@@ -162,6 +162,19 @@ class AzureAIAPI(ModelAPI):
             model_extras=self.model_args,
         )
 
+        def model_call(response: ChatCompletions | None = None) -> ModelCall:
+            return ModelCall.create(
+                request=request
+                | dict(
+                    messages=[message.as_dict() for message in request["messages"]],
+                    tools=[tool.as_dict() for tool in request["tools"]]
+                    if request.get("tools", None) is not None
+                    else None,
+                ),
+                response=response.as_dict() if response else {},
+                filter=image_url_filter,
+            )
+
         # make call
         try:
             response: ChatCompletions = await client.complete(**request)
@@ -173,19 +186,10 @@ class AzureAIAPI(ModelAPI):
                     output_tokens=response.usage.completion_tokens,
                     total_tokens=response.usage.total_tokens,
                 ),
-            ), ModelCall.create(
-                request=request
-                | dict(
-                    messages=[message.as_dict() for message in request["messages"]],
-                    tools=[tool.as_dict() for tool in request["tools"]]
-                    if request.get("tools", None) is not None
-                    else None,
-                ),
-                response=response.as_dict(),
-                filter=image_url_filter,
-            )
+            ), model_call(response)
+
         except AzureError as ex:
-            return self.handle_azure_error(ex)
+            return self.handle_azure_error(ex), model_call()
         finally:
             await client.close()
 
@@ -251,7 +255,7 @@ class AzureAIAPI(ModelAPI):
     def is_mistral(self) -> bool:
         return "mistral" in self.model_name.lower()
 
-    def handle_azure_error(self, ex: AzureError) -> ModelOutput:
+    def handle_azure_error(self, ex: AzureError) -> ModelOutput | Exception:
         if isinstance(ex, HttpResponseError):
             response = str(ex.message)
             if "maximum context length" in response.lower():
@@ -260,12 +264,8 @@ class AzureAIAPI(ModelAPI):
                     content=response,
                     stop_reason="model_length",
                 )
-            elif ex.status_code == 400 and ex.error:
-                return ModelOutput.from_content(
-                    model=self.model_name,
-                    content=f"Your request triggered an error: {ex.error}",
-                    stop_reason="content_filter",
-                )
+            elif ex.status_code == 400:
+                return ex
 
         raise ex
 
@@ -312,12 +312,14 @@ async def chat_request_message(
 async def chat_content_item(content: Content) -> ContentItem:
     if isinstance(content, ContentText):
         return TextContentItem(text=content.text)
-    else:
+    elif isinstance(content, ContentImage):
         return ImageContentItem(
             image_url=ImageUrl(
-                url=await image_as_data_uri(content.image), detail=content.detail
+                url=await file_as_data_uri(content.image), detail=content.detail
             )
         )
+    else:
+        raise RuntimeError("Azure AI models do not support audio or video inputs.")
 
 
 def chat_tool_call(tool_call: ToolCall) -> ChatCompletionsToolCall:

inspect_ai/model/_providers/bedrock.py

@@ -11,7 +11,7 @@ from inspect_ai._util.constants import (
 )
 from inspect_ai._util.content import Content, ContentImage, ContentText
 from inspect_ai._util.error import pip_dependency_error
-from inspect_ai._util.images import image_as_data
+from inspect_ai._util.images import file_as_data
 from inspect_ai._util.version import verify_required_version
 from inspect_ai.tool import ToolChoice, ToolInfo
 from inspect_ai.tool._tool_call import ToolCall
@@ -27,11 +27,7 @@ from .._chat_message import (
 from .._generate_config import GenerateConfig
 from .._model import ModelAPI
 from .._model_call import ModelCall
-from .._model_output import (
-    ChatCompletionChoice,
-    ModelOutput,
-    ModelUsage,
-)
+from .._model_output import ChatCompletionChoice, ModelOutput, ModelUsage
 from .util import (
     model_base_url,
 )
@@ -307,7 +303,7 @@ class BedrockAPI(ModelAPI):
         tools: list[ToolInfo],
         tool_choice: ToolChoice,
         config: GenerateConfig,
-    ) -> ModelOutput | tuple[ModelOutput, ModelCall]:
+    ) -> ModelOutput | tuple[ModelOutput | Exception, ModelCall]:
         from botocore.config import Config
         from botocore.exceptions import ClientError
 
@@ -339,25 +335,33 @@ class BedrockAPI(ModelAPI):
             # Resolve the input messages into converse messages
             system, messages = await converse_messages(input)
 
-            try:
-                # Make the request
-                request = ConverseClientConverseRequest(
-                    modelId=self.model_name,
-                    messages=messages,
-                    system=system,
-                    inferenceConfig=ConverseInferenceConfig(
-                        maxTokens=config.max_tokens,
-                        temperature=config.temperature,
-                        topP=config.top_p,
-                        stopSequences=config.stop_seqs,
+            # Make the request
+            request = ConverseClientConverseRequest(
+                modelId=self.model_name,
+                messages=messages,
+                system=system,
+                inferenceConfig=ConverseInferenceConfig(
+                    maxTokens=config.max_tokens,
+                    temperature=config.temperature,
+                    topP=config.top_p,
+                    stopSequences=config.stop_seqs,
+                ),
+                additionalModelRequestFields={
+                    "top_k": config.top_k,
+                    **config.model_config,
+                },
+                toolConfig=tool_config,
+            )
+
+            def model_call(response: dict[str, Any] | None = None) -> ModelCall:
+                return ModelCall.create(
+                    request=replace_bytes_with_placeholder(
+                        request.model_dump(exclude_none=True)
                     ),
-                    additionalModelRequestFields={
-                        "top_k": config.top_k,
-                        **config.model_config,
-                    },
-                    toolConfig=tool_config,
+                    response=response,
                 )
 
+            try:
                 # Process the reponse
                 response = await client.converse(
                     **request.model_dump(exclude_none=True)
@@ -366,32 +370,24 @@ class BedrockAPI(ModelAPI):
 
             except ClientError as ex:
                 # Look for an explicit validation exception
-                if (
-                    ex.response["Error"]["Code"] == "ValidationException"
-                    and "Too many input tokens" in ex.response["Error"]["Message"]
-                ):
+                if ex.response["Error"]["Code"] == "ValidationException":
                     response = ex.response["Error"]["Message"]
-                    return ModelOutput.from_content(
-                        model=self.model_name,
-                        content=response,
-                        stop_reason="model_length",
-                    )
+                    if "Too many input tokens" in response:
+                        return ModelOutput.from_content(
+                            model=self.model_name,
+                            content=response,
+                            stop_reason="model_length",
+                        )
+                    else:
+                        return ex, model_call(None)
                 else:
                     raise ex
 
         # create a model output from the response
         output = model_output_from_response(self.model_name, converse_response, tools)
 
-        # record call
-        call = ModelCall.create(
-            request=replace_bytes_with_placeholder(
-                request.model_dump(exclude_none=True)
-            ),
-            response=response,
-        )
-
         # return
-        return output, call
+        return output, model_call(response)
 
 
 async def converse_messages(
@@ -430,7 +426,9 @@ def model_output_from_response(
             content.append(ContentText(type="text", text=c.text))
         elif c.image is not None:
             base64_image = base64.b64encode(c.image.source.bytes).decode("utf-8")
-            content.append(ContentImage(image=base64_image))
+            content.append(
+                ContentImage(image=f"data:image/{c.image.format};base64,{base64_image}")
+            )
         elif c.toolUse is not None:
             tool_calls.append(
                 ToolCall(
@@ -548,6 +546,7 @@ async def converse_chat_message(
                 "Tool call is missing a tool call id, which is required for Converse API"
             )
         if message.function is None:
+            print(message)
             raise ValueError(
                 "Tool call is missing a function, which is required for Converse API"
             )
@@ -565,7 +564,7 @@ async def converse_chat_message(
                 if c.type == "text":
                     tool_result_content.append(ConverseToolResultContent(text=c.text))
                 elif c.type == "image":
-                    image_data, image_type = await image_as_data(c.image)
+                    image_data, image_type = await file_as_data(c.image)
                     tool_result_content.append(
                         ConverseToolResultContent(
                             image=ConverseImage(
@@ -604,7 +603,7 @@ async def converse_contents(
         result: list[ConverseMessageContent] = []
         for c in content:
             if c.type == "image":
-                image_data, image_type = await image_as_data(c.image)
+                image_data, image_type = await file_as_data(c.image)
                 result.append(
                     ConverseMessageContent(
                         image=ConverseImage(