inspect-ai 0.3.70__py3-none-any.whl → 0.3.72__py3-none-any.whl

inspect_ai/model/_providers/anthropic.py

@@ -1,9 +1,10 @@
 import functools
 import os
+import re
 import sys
 from copy import copy
 from logging import getLogger
-from typing import Any, Literal, Tuple, TypedDict, cast
+from typing import Any, Literal, Optional, Tuple, TypedDict, cast
 
 from .util.tracker import HttpxTimeTracker
 
@@ -28,8 +29,12 @@ from anthropic.types import (
     ImageBlockParam,
     Message,
     MessageParam,
+    RedactedThinkingBlock,
+    RedactedThinkingBlockParam,
     TextBlock,
     TextBlockParam,
+    ThinkingBlock,
+    ThinkingBlockParam,
     ToolParam,
     ToolResultBlockParam,
     ToolUseBlock,
@@ -44,7 +49,12 @@ from inspect_ai._util.constants import (
     DEFAULT_MAX_RETRIES,
     NO_CONTENT,
 )
-from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.content import (
+    Content,
+    ContentImage,
+    ContentReasoning,
+    ContentText,
+)
 from inspect_ai._util.error import exception_message
 from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.logger import warn_once
@@ -194,7 +204,7 @@ class AnthropicAPI(ModelAPI):
                 tools_param,
                 messages,
                 computer_use,
-            ) = await resolve_chat_input(self.model_name, input, tools, config)
+            ) = await self.resolve_chat_input(input, tools, config)
 
             # prepare request params (assembed this way so we can log the raw model call)
             request = dict(messages=messages)
@@ -204,23 +214,33 @@ class AnthropicAPI(ModelAPI):
                 request["system"] = system_param
             request["tools"] = tools_param
             if len(tools) > 0:
-                request["tool_choice"] = message_tool_choice(tool_choice)
+                request["tool_choice"] = message_tool_choice(
+                    tool_choice, self.is_using_thinking(config)
+                )
 
             # additional options
-            request = request | self.completion_params(config)
+            req, headers, betas = self.completion_config(config)
+            request = request | req
 
             # extra headers (for time tracker and computer use)
-            extra_headers = {HttpxTimeTracker.REQUEST_ID_HEADER: request_id}
+            extra_headers = headers | {HttpxTimeTracker.REQUEST_ID_HEADER: request_id}
             if computer_use:
-                extra_headers["anthropic-beta"] = "computer-use-2024-10-22"
+                betas.append("computer-use-2025-01-24")
+            if len(betas) > 0:
+                extra_headers["anthropic-beta"] = ",".join(betas)
+
             request["extra_headers"] = extra_headers
 
             # extra_body
             if self.extra_body is not None:
                 request["extra_body"] = self.extra_body
 
-            # make request
-            message = await self.client.messages.create(**request, stream=False)
+            # make request (stream if we are using reasoning)
+            if self.is_using_thinking(config):
+                async with self.client.messages.stream(**request) as stream:
+                    message = await stream.get_final_message()
+            else:
+                message = await self.client.messages.create(**request, stream=False)
 
             # set response for ModelCall
             response = message.model_dump()
@@ -245,27 +265,70 @@ class AnthropicAPI(ModelAPI):
             else:
                 raise ex
 
-    def completion_params(self, config: GenerateConfig) -> dict[str, Any]:
-        params = dict(model=self.model_name, max_tokens=cast(int, config.max_tokens))
-        if config.temperature is not None:
-            params["temperature"] = config.temperature
-        if config.top_p is not None:
-            params["top_p"] = config.top_p
-        if config.top_k is not None:
-            params["top_k"] = config.top_k
+    def completion_config(
+        self, config: GenerateConfig
+    ) -> tuple[dict[str, Any], dict[str, str], list[str]]:
+        max_tokens = cast(int, config.max_tokens)
+        params = dict(model=self.model_name, max_tokens=max_tokens)
+        headers: dict[str, str] = {}
+        betas: list[str] = []
+        # some params not compatible with thinking models
+        if not self.is_using_thinking(config):
+            if config.temperature is not None:
+                params["temperature"] = config.temperature
+            if config.top_p is not None:
+                params["top_p"] = config.top_p
+            if config.top_k is not None:
+                params["top_k"] = config.top_k
+
+        # some thinking-only stuff
+        if self.is_using_thinking(config):
+            params["thinking"] = dict(
+                type="enabled", budget_tokens=config.reasoning_tokens
+            )
+            headers["anthropic-version"] = "2023-06-01"
+            if max_tokens > 8192:
+                betas.append("output-128k-2025-02-19")
+
+        # config that applies to all models
         if config.timeout is not None:
             params["timeout"] = float(config.timeout)
         if config.stop_seqs is not None:
             params["stop_sequences"] = config.stop_seqs
-        return params
+
+        # return config
+        return params, headers, betas
 
     @override
     def max_tokens(self) -> int | None:
         # anthropic requires you to explicitly specify max_tokens (most others
         # set it to the maximum allowable output tokens for the model).
-        # set to 4096 which is the lowest documented max_tokens for claude models
+        # set to 4096 which is the highest possible for claude 3 (claude 3.5
+        # allows up to 8192)
         return 4096
 
+    @override
+    def max_tokens_for_config(self, config: GenerateConfig) -> int | None:
+        max_tokens = cast(int, self.max_tokens())
+        if self.is_thinking_model() and config.reasoning_tokens is not None:
+            max_tokens = max_tokens + config.reasoning_tokens
+        return max_tokens
+
+    def is_using_thinking(self, config: GenerateConfig) -> bool:
+        return self.is_thinking_model() and config.reasoning_tokens is not None
+
+    def is_thinking_model(self) -> bool:
+        return not self.is_claude_3() and not self.is_claude_3_5()
+
+    def is_claude_3(self) -> bool:
+        return re.search(r"claude-3-[a-zA-Z]", self.model_name) is not None
+
+    def is_claude_3_5(self) -> bool:
+        return "claude-3-5-" in self.model_name
+
+    def is_claude_3_7(self) -> bool:
+        return "claude-3-7-" in self.model_name
+
     @override
     def connection_key(self) -> str:
         return str(self.api_key)
@@ -295,6 +358,14 @@ class AnthropicAPI(ModelAPI):
     def tool_result_images(self) -> bool:
         return True
 
+    @override
+    def emulate_reasoning_history(self) -> bool:
+        return False
+
+    @override
+    def force_reasoning_history(self) -> Literal["none", "all", "last"] | None:
+        return "all"
+
     # convert some common BadRequestError states into 'refusal' model output
     def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | Exception:
         error = exception_message(ex).lower()
@@ -329,6 +400,148 @@ class AnthropicAPI(ModelAPI):
         else:
             return ex
 
+    async def resolve_chat_input(
+        self,
+        input: list[ChatMessage],
+        tools: list[ToolInfo],
+        config: GenerateConfig,
+    ) -> Tuple[
+        list[TextBlockParam] | None, list["ToolParamDef"], list[MessageParam], bool
+    ]:
+        # extract system message
+        system_messages, messages = split_system_messages(input, config)
+
+        # messages
+        message_params = [(await message_param(message)) for message in messages]
+
+        # collapse user messages (as Inspect 'tool' messages become Claude 'user' messages)
+        message_params = functools.reduce(
+            consecutive_user_message_reducer, message_params, []
+        )
+
+        # tools
+        tools_params, computer_use = self.tool_params_for_tools(tools, config)
+
+        # system messages
+        if len(system_messages) > 0:
+            system_param: list[TextBlockParam] | None = [
+                TextBlockParam(type="text", text=message.text)
+                for message in system_messages
+            ]
+        else:
+            system_param = None
+
+        # add caching directives if necessary
+        cache_prompt = (
+            config.cache_prompt
+            if isinstance(config.cache_prompt, bool)
+            else True
+            if len(tools_params)
+            else False
+        )
+
+        # only certain claude models qualify
+        if cache_prompt:
+            if (
+                "claude-3-sonnet" in self.model_name
+                or "claude-2" in self.model_name
+                or "claude-instant" in self.model_name
+            ):
+                cache_prompt = False
+
+        if cache_prompt:
+            # system
+            if system_param:
+                add_cache_control(system_param[-1])
+            # tools
+            if tools_params:
+                add_cache_control(tools_params[-1])
+            # last 2 user messages
+            user_message_params = list(
+                filter(lambda m: m["role"] == "user", reversed(message_params))
+            )
+            for message in user_message_params[:2]:
+                if isinstance(message["content"], str):
+                    text_param = TextBlockParam(type="text", text=message["content"])
+                    add_cache_control(text_param)
+                    message["content"] = [text_param]
+                else:
+                    content = list(message["content"])
+                    add_cache_control(cast(dict[str, Any], content[-1]))
+
+        # return chat input
+        return system_param, tools_params, message_params, computer_use
+
+    def tool_params_for_tools(
+        self, tools: list[ToolInfo], config: GenerateConfig
+    ) -> tuple[list["ToolParamDef"], bool]:
+        # tool params and computer_use bit to return
+        tool_params: list["ToolParamDef"] = []
+        computer_use = False
+
+        # for each tool, check if it has a native computer use implementation and use that
+        # when available (noting that we need to set the computer use request header)
+        for tool in tools:
+            computer_use_tool = (
+                self.computer_use_tool_param(tool)
+                if config.internal_tools is not False
+                else None
+            )
+            if computer_use_tool:
+                tool_params.append(computer_use_tool)
+                computer_use = True
+            else:
+                tool_params.append(
+                    ToolParam(
+                        name=tool.name,
+                        description=tool.description,
+                        input_schema=tool.parameters.model_dump(exclude_none=True),
+                    )
+                )
+
+        return tool_params, computer_use
+
+    def computer_use_tool_param(
+        self, tool: ToolInfo
+    ) -> Optional["ComputerUseToolParam"]:
+        # check for compatible 'computer' tool
+        if tool.name == "computer" and (
+            sorted(tool.parameters.properties.keys())
+            == sorted(
+                [
+                    "action",
+                    "coordinate",
+                    "duration",
+                    "scroll_amount",
+                    "scroll_direction",
+                    "start_coordinate",
+                    "text",
+                ]
+            )
+        ):
+            if self.is_claude_3_5():
+                warn_once(
+                    logger,
+                    "Use of Anthropic's native computer use support is not enabled in Claude 3.5. Please use 3.7 or later to leverage the native support.",
+                )
+                return None
+            return ComputerUseToolParam(
+                type="computer_20250124",
+                name="computer",
+                # Note: The dimensions passed here for display_width_px and display_height_px should
+                # match the dimensions of screenshots returned by the tool.
+                # Those dimensions will always be one of the values in MAX_SCALING_TARGETS
+                # in _x11_client.py.
+                # TODO: enhance this code to calculate the dimensions based on the scaled screen
+                # size used by the container.
+                display_width_px=1366,
+                display_height_px=768,
+                display_number=1,
+            )
+        # not a computer_use tool
+        else:
+            return None
+
 
 # native anthropic tool definitions for computer use beta
 # https://docs.anthropic.com/en/docs/build-with-claude/computer-use
@@ -344,131 +557,6 @@ class ComputerUseToolParam(TypedDict):
 ToolParamDef = ToolParam | ComputerUseToolParam
 
 
-async def resolve_chat_input(
-    model: str,
-    input: list[ChatMessage],
-    tools: list[ToolInfo],
-    config: GenerateConfig,
-) -> Tuple[list[TextBlockParam] | None, list[ToolParamDef], list[MessageParam], bool]:
-    # extract system message
-    system_messages, messages = split_system_messages(input, config)
-
-    # messages
-    message_params = [(await message_param(message)) for message in messages]
-
-    # collapse user messages (as Inspect 'tool' messages become Claude 'user' messages)
-    message_params = functools.reduce(
-        consecutive_user_message_reducer, message_params, []
-    )
-
-    # tools
-    tools_params, computer_use = tool_params_for_tools(tools, config)
-
-    # system messages
-    if len(system_messages) > 0:
-        system_param: list[TextBlockParam] | None = [
-            TextBlockParam(type="text", text=message.text)
-            for message in system_messages
-        ]
-    else:
-        system_param = None
-
-    # add caching directives if necessary
-    cache_prompt = (
-        config.cache_prompt
-        if isinstance(config.cache_prompt, bool)
-        else True
-        if len(tools_params)
-        else False
-    )
-
-    # only certain claude models qualify
-    if cache_prompt:
-        if (
-            "claude-3-sonnet" in model
-            or "claude-2" in model
-            or "claude-instant" in model
-        ):
-            cache_prompt = False
-
-    if cache_prompt:
-        # system
-        if system_param:
-            add_cache_control(system_param[-1])
-        # tools
-        if tools_params:
-            add_cache_control(tools_params[-1])
-        # last 2 user messages
-        user_message_params = list(
-            filter(lambda m: m["role"] == "user", reversed(message_params))
-        )
-        for message in user_message_params[:2]:
-            if isinstance(message["content"], str):
-                text_param = TextBlockParam(type="text", text=message["content"])
-                add_cache_control(text_param)
-                message["content"] = [text_param]
-            else:
-                content = list(message["content"])
-                add_cache_control(cast(dict[str, Any], content[-1]))
-
-    # return chat input
-    return system_param, tools_params, message_params, computer_use
-
-
-def tool_params_for_tools(
-    tools: list[ToolInfo], config: GenerateConfig
-) -> tuple[list[ToolParamDef], bool]:
-    # tool params and computer_use bit to return
-    tool_params: list[ToolParamDef] = []
-    computer_use = False
-
-    # for each tool, check if it has a native computer use implementation and use that
-    # when available (noting that we need to set the computer use request header)
-    for tool in tools:
-        computer_use_tool = (
-            computer_use_tool_param(tool)
-            if config.internal_tools is not False
-            else None
-        )
-        if computer_use_tool:
-            tool_params.append(computer_use_tool)
-            computer_use = True
-        else:
-            tool_params.append(
-                ToolParam(
-                    name=tool.name,
-                    description=tool.description,
-                    input_schema=tool.parameters.model_dump(exclude_none=True),
-                )
-            )
-
-    return tool_params, computer_use
-
-
-def computer_use_tool_param(tool: ToolInfo) -> ComputerUseToolParam | None:
-    # check for compatible 'computer' tool
-    if tool.name == "computer" and (
-        sorted(tool.parameters.properties.keys())
-        == sorted(["action", "coordinate", "text"])
-    ):
-        return ComputerUseToolParam(
-            type="computer_20241022",
-            name="computer",
-            # Note: The dimensions passed here for display_width_px and display_height_px should
-            # match the dimensions of screenshots returned by the tool.
-            # Those dimensions will always be one of the values in MAX_SCALING_TARGETS
-            # in _x11_client.py.
-            # TODO: enhance this code to calculate the dimensions based on the scaled screen
-            # size used by the container.
-            display_width_px=1366,
-            display_height_px=768,
-            display_number=1,
-        )
-    # not a computer_use tool
-    else:
-        return None
-
-
 def add_cache_control(
     param: TextBlockParam | ToolParam | ComputerUseToolParam | dict[str, Any],
 ) -> None:
@@ -498,7 +586,7 @@ def combine_messages(a: MessageParam, b: MessageParam) -> MessageParam:
     role = a["role"]
     a_content = a["content"]
     b_content = b["content"]
-    if isinstance(a_content, str) and isinstance(a_content, str):
+    if isinstance(a_content, str) and isinstance(b_content, str):
         return MessageParam(role=role, content=f"{a_content}\n{b_content}")
     elif isinstance(a_content, list) and isinstance(b_content, list):
         return MessageParam(role=role, content=a_content + b_content)
@@ -514,9 +602,15 @@ def combine_messages(a: MessageParam, b: MessageParam) -> MessageParam:
         raise ValueError(f"Unexpected content types for messages: {a}, {b}")
 
 
-def message_tool_choice(tool_choice: ToolChoice) -> message_create_params.ToolChoice:
+def message_tool_choice(
+    tool_choice: ToolChoice, thinking_model: bool
+) -> message_create_params.ToolChoice:
     if isinstance(tool_choice, ToolFunction):
-        return {"type": "tool", "name": tool_choice.name}
+        # forced tool use not compatible with thinking models
+        if thinking_model:
+            return {"type": "any"}
+        else:
+            return {"type": "tool", "name": tool_choice.name}
     elif tool_choice == "any":
         return {"type": "any"}
     elif tool_choice == "none":
@@ -544,9 +638,15 @@ async def message_param(message: ChatMessage) -> MessageParam:
     # "tool" means serving a tool call result back to claude
     elif message.role == "tool":
         if message.error is not None:
-            content: str | list[TextBlockParam | ImageBlockParam] = (
-                message.error.message
-            )
+            content: (
+                str
+                | list[
+                    TextBlockParam
+                    | ImageBlockParam
+                    | ThinkingBlockParam
+                    | RedactedThinkingBlockParam
+                ]
+            ) = message.error.message
             # anthropic requires that content be populated when
             # is_error is true (throws bad_request_error when not)
             # so make sure this precondition is met
@@ -567,7 +667,7 @@ async def message_param(message: ChatMessage) -> MessageParam:
                 ToolResultBlockParam(
                     tool_use_id=str(message.tool_call_id),
                     type="tool_result",
-                    content=content,
+                    content=cast(list[TextBlockParam | ImageBlockParam], content),
                     is_error=message.error is not None,
                 )
             ],
@@ -576,7 +676,13 @@ async def message_param(message: ChatMessage) -> MessageParam:
     # tool_calls means claude is attempting to call our tools
     elif message.role == "assistant" and message.tool_calls:
         # first include content (claude <thinking>)
-        tools_content: list[TextBlockParam | ImageBlockParam | ToolUseBlockParam] = (
+        tools_content: list[
+            TextBlockParam
+            | ThinkingBlockParam
+            | RedactedThinkingBlockParam
+            | ImageBlockParam
+            | ToolUseBlockParam
+        ] = (
             [TextBlockParam(type="text", text=message.content or NO_CONTENT)]
             if isinstance(message.content, str)
             else (
@@ -645,6 +751,16 @@ def model_output_from_message(message: Message, tools: list[ToolInfo]) -> ModelO
                     arguments=content_block.model_dump().get("input", {}),
                 )
             )
+        elif isinstance(content_block, RedactedThinkingBlock):
+            content.append(
+                ContentReasoning(reasoning=content_block.data, redacted=True)
+            )
+        elif isinstance(content_block, ThinkingBlock):
+            content.append(
+                ContentReasoning(
+                    reasoning=content_block.thinking, signature=content_block.signature
+                )
+            )
 
     # resolve choice
     choice = ChatCompletionChoice(
@@ -702,7 +818,7 @@ def split_system_messages(
 
 async def message_param_content(
     content: Content,
-) -> TextBlockParam | ImageBlockParam:
+) -> TextBlockParam | ImageBlockParam | ThinkingBlockParam | RedactedThinkingBlockParam:
     if isinstance(content, ContentText):
         return TextBlockParam(type="text", text=content.text or NO_CONTENT)
     elif isinstance(content, ContentImage):
@@ -720,6 +836,18 @@ async def message_param_content(
             type="image",
             source=dict(type="base64", media_type=cast(Any, media_type), data=image),
         )
+    elif isinstance(content, ContentReasoning):
+        if content.redacted:
+            return RedactedThinkingBlockParam(
+                type="redacted_thinking",
+                data=content.reasoning,
+            )
+        else:
+            if content.signature is None:
+                raise ValueError("Thinking content without signature.")
+            return ThinkingBlockParam(
+                type="thinking", thinking=content.reasoning, signature=content.signature
+            )
     else:
         raise RuntimeError(
             "Anthropic models do not currently support audio or video inputs."

inspect_ai/model/_providers/google.py

@@ -38,10 +38,13 @@ from pydantic import JsonValue
 from typing_extensions import override
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED, NO_CONTENT
-from inspect_ai._util.content import Content as InspectContent
+from inspect_ai._util.content import (
+    Content as InspectContent,
+)
 from inspect_ai._util.content import (
     ContentAudio,
     ContentImage,
+    ContentReasoning,
     ContentText,
     ContentVideo,
 )
@@ -250,7 +253,10 @@ class GoogleGenAIAPI(ModelAPI):
 
     @override
     def is_rate_limit(self, ex: BaseException) -> bool:
-        return isinstance(ex, APIError) and ex.code in (429, 500, 503, 504)
+        # see https://cloud.google.com/storage/docs/retry-strategy
+        return isinstance(ex, APIError) and (
+            ex.code in (408, 429, 429) or ex.code >= 500
+        )
 
     @override
     def connection_key(self) -> str:
@@ -405,6 +411,8 @@ async def content_part(client: Client, content: InspectContent | str) -> Part:
         return Part.from_text(text=content or NO_CONTENT)
     elif isinstance(content, ContentText):
         return Part.from_text(text=content.text or NO_CONTENT)
+    elif isinstance(content, ContentReasoning):
+        return Part.from_text(text=content.reasoning or NO_CONTENT)
     else:
         return await chat_content_to_part(client, content)
 
@@ -417,7 +425,8 @@ async def chat_content_to_part(
         content_bytes, mime_type = await file_as_data(content.image)
         return Part.from_bytes(mime_type=mime_type, data=content_bytes)
     else:
-        return await file_for_content(client, content)
+        file = await file_for_content(client, content)
+        return Part.from_uri(file_uri=file.uri, mime_type=file.mime_type)
 
 
 async def extract_system_message_as_parts(
@@ -552,11 +561,19 @@ def completion_choice_from_candidate(candidate: Candidate) -> ChatCompletionChoi
     # stop reason
     stop_reason = finish_reason_to_stop_reason(candidate.finish_reason)
 
+    # choice content may include reasoning
+    if reasoning:
+        choice_content: str | list[Content] = [
+            ContentReasoning(reasoning=reasoning),
+            ContentText(text=content),
+        ]
+    else:
+        choice_content = content
+
     # build choice
     choice = ChatCompletionChoice(
         message=ChatMessageAssistant(
-            content=content,
-            reasoning=reasoning,
+            content=choice_content,
             tool_calls=tool_calls if len(tool_calls) > 0 else None,
             source="generate",
         ),
@@ -742,7 +759,7 @@ async def file_for_content(
         uploaded_file = files_db.get(content_sha256)
         if uploaded_file:
             try:
-                upload: File = client.files.get(uploaded_file)
+                upload: File = client.files.get(name=uploaded_file)
                 if upload.state.name == "ACTIVE":
                     trace(f"Using uploaded file: {uploaded_file}")
                     return upload
@@ -754,10 +771,12 @@ async def file_for_content(
                 trace(f"Error attempting to access uploaded file: {ex}")
                 files_db.delete(content_sha256)
         # do the upload (and record it)
-        upload = client.files.upload(BytesIO(content_bytes), mime_type=mime_type)
+        upload = client.files.upload(
+            file=BytesIO(content_bytes), config=dict(mime_type=mime_type)
+        )
         while upload.state.name == "PROCESSING":
             await asyncio.sleep(3)
-            upload = client.files.get(upload.name)
+            upload = client.files.get(name=upload.name)
         if upload.state.name == "FAILED":
             trace(f"Failed to upload file '{upload.name}: {upload.error}")
             raise ValueError(f"Google file upload failed: {upload.error}")

inspect_ai/model/_providers/groq.py

@@ -28,7 +28,7 @@ from inspect_ai._util.constants import (
     DEFAULT_MAX_RETRIES,
     DEFAULT_MAX_TOKENS,
 )
-from inspect_ai._util.content import Content
+from inspect_ai._util.content import Content, ContentReasoning, ContentText
 from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.url import is_http_url
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
@@ -326,12 +326,17 @@ def chat_tool_calls(message: Any, tools: list[ToolInfo]) -> Optional[List[ToolCa
 def chat_message_assistant(message: Any, tools: list[ToolInfo]) -> ChatMessageAssistant:
     reasoning = getattr(message, "reasoning", None)
     if reasoning is not None:
-        reasoning = str(reasoning)
+        content: str | list[Content] = [
+            ContentReasoning(reasoning=str(reasoning)),
+            ContentText(text=message.content or ""),
+        ]
+    else:
+        content = message.content or ""
+
     return ChatMessageAssistant(
-        content=message.content or "",
+        content=content,
         source="generate",
         tool_calls=chat_tool_calls(message, tools),
-        reasoning=reasoning,
     )