inspect-ai 0.3.90__py3-none-any.whl → 0.3.92__py3-none-any.whl

inspect_ai/model/_openai_responses.py

@@ -1,3 +1,4 @@
+import json
 from itertools import chain
 from typing import TypedDict, cast
 
@@ -56,17 +57,17 @@ from inspect_ai.tool._tool_info import ToolInfo
 
 
 async def openai_responses_inputs(
-    messages: list[ChatMessage], model: str
+    messages: list[ChatMessage], model: str, store: bool
 ) -> list[ResponseInputItemParam]:
     return [
         item
         for message in messages
-        for item in await _openai_input_item_from_chat_message(message, model)
+        for item in await _openai_input_item_from_chat_message(message, model, store)
     ]
 
 
 async def _openai_input_item_from_chat_message(
-    message: ChatMessage, model: str
+    message: ChatMessage, model: str, store: bool
 ) -> list[ResponseInputItemParam]:
     if message.role == "system":
         content = await _openai_responses_content_list_param(message.content)
@@ -84,7 +85,7 @@ async def _openai_input_item_from_chat_message(
             )
         ]
     elif message.role == "assistant":
-        return _openai_input_items_from_chat_message_assistant(message)
+        return _openai_input_items_from_chat_message_assistant(message, store)
     elif message.role == "tool":
         if message.internal:
             internal = _model_tool_call_for_internal(message.internal)
@@ -208,7 +209,7 @@ def openai_responses_chat_choices(
 
 class _AssistantInternal(TypedDict):
     output_message_id: str | None
-    reasoning_id: str | None
+    tool_message_ids: dict[str, str]
 
 
 def _chat_message_assistant_from_openai_response(
@@ -237,7 +238,7 @@ def _chat_message_assistant_from_openai_response(
     # collect output and tool calls
     message_content: list[Content] = []
     tool_calls: list[ToolCall] = []
-    internal = _AssistantInternal(output_message_id=None, reasoning_id=None)
+    internal = _AssistantInternal(output_message_id=None, tool_message_ids={})
     for output in response.output:
         match output:
             case ResponseOutputMessage(content=content, id=id):
@@ -252,24 +253,28 @@ def _chat_message_assistant_from_openai_response(
                     ]
                 )
             case ResponseReasoningItem(summary=summary, id=id):
-                assert internal["reasoning_id"] is None, "Multiple reasoning items"
-                internal["reasoning_id"] = id
                 message_content.append(
-                    ContentReasoning(reasoning="\n".join([s.text for s in summary]))
+                    ContentReasoning(
+                        reasoning="\n".join([s.text for s in summary]), signature=id
+                    )
                 )
             case _:
                 stop_reason = "tool_calls"
                 match output:
                     case ResponseFunctionToolCall():
+                        if output.id is not None:
+                            internal["tool_message_ids"][output.call_id] = output.id
                         tool_calls.append(
                             parse_tool_call(
                                 output.call_id,
-                                output.name,
+                                _from_responses_tool_alias(output.name),
                                 output.arguments,
                                 tools,
                             )
                         )
                     case ResponseComputerToolCall():
+                        if output.id is not None:
+                            internal["tool_message_ids"][output.call_id] = output.id
                         tool_calls.append(
                             tool_call_from_openai_computer_tool_call(output)
                         )
@@ -290,7 +295,7 @@ def _chat_message_assistant_from_openai_response(
 
 
 def _openai_input_items_from_chat_message_assistant(
-    message: ChatMessageAssistant,
+    message: ChatMessageAssistant, store: bool
 ) -> list[ResponseInputItemParam]:
     """
     Transform a `ChatMessageAssistant` into OpenAI `ResponseInputItem`'s for playback to the model.
@@ -300,13 +305,26 @@ def _openai_input_items_from_chat_message_assistant(
     field of the `ChatMessageAssistant` to help it provide the proper id's the
     items in the returned list.
     """
-    # As currently coded, this code only supports a single OutputMessage and
-    # a single ReasoningItem for each Response/ChatMessageAssistant.
-    reasoning_item: ResponseReasoningItemParam | None = None
+    (output_message_id, tool_message_ids) = _ids_from_assistant_internal(message)
+
+    # we want to prevent yielding output messages in the case where we have an
+    # 'internal' field (so the message came from the model API as opposed to
+    # being user synthesized) AND there is no output_message_id (indicating that
+    # when reading the message from the server we didn't find output). this could
+    # happen e.g. when a react() agent sets the output.completion in response
+    # to a submit() tool call
+    suppress_output_message = message.internal is not None and output_message_id is None
+
+    # if we are not storing messages on the server then blank these out
+    if not store:
+        output_message_id = None
+        tool_message_ids = {}
+
+    # items to return -- ensure we use a single output message (and just chain
+    # additional content on to it)
+    items: list[ResponseInputItemParam] = []
     output_message: ResponseOutputMessageParam | None = None
 
-    (output_message_id, reasoning_id) = _ids_from_assistant_internal(message)
-
     for content in (
         list[ContentText | ContentReasoning]([ContentText(text=message.content)])
         if isinstance(message.content, str)
@@ -316,14 +334,25 @@ def _openai_input_items_from_chat_message_assistant(
     ):
         match content:
             case ContentReasoning(reasoning=reasoning):
-                assert reasoning_item is None, "Multiple reasoning items"
-                assert reasoning_id is not None, "Must find reasoning id"
-                reasoning_item = ResponseReasoningItemParam(
-                    type="reasoning",
-                    id=reasoning_id,
-                    summary=[Summary(type="summary_text", text=reasoning)],
+                assert content.signature is not None, (
+                    "reasoning_id must be saved in signature"
                 )
+                # if items are not stored on the server then there is no
+                # sense appending the reasoning item as its just a pointer
+                if store:
+                    items.append(
+                        ResponseReasoningItemParam(
+                            type="reasoning",
+                            id=content.signature,
+                            summary=[Summary(type="summary_text", text=reasoning)]
+                            if reasoning
+                            else [],
+                        )
+                    )
             case ContentText(text=text, refusal=refusal):
+                if suppress_output_message:
+                    continue
+
                 new_content = (
                     ResponseOutputRefusalParam(type="refusal", refusal=text)
                     if refusal
@@ -332,22 +361,23 @@ def _openai_input_items_from_chat_message_assistant(
                     )
                 )
                 if output_message is None:
-                    assert output_message_id is not None, "Missing output message id"
                     output_message = ResponseOutputMessageParam(
                         type="message",
                         role="assistant",
-                        id=output_message_id,
+                        # this actually can be `None`, and it will in fact be `None` when the
+                        # assistant message is synthesized by the scaffold as opposed to being
+                        # replayed from the model (or when store=False)
+                        id=output_message_id,  # type: ignore[typeddict-item]
                         content=[new_content],
                         status="completed",
                     )
+                    items.append(output_message)
                 else:
                     output_message["content"] = chain(
                         output_message["content"], [new_content]
                     )
 
-    return [
-        item for item in (reasoning_item, output_message) if item
-    ] + _tool_call_items_from_assistant_message(message)
+    return items + _tool_call_items_from_assistant_message(message, tool_message_ids)
 
 
 def _model_tool_call_for_internal(
@@ -380,7 +410,7 @@ def _maybe_native_tool_param(
 
 
 def _tool_call_items_from_assistant_message(
-    message: ChatMessageAssistant,
+    message: ChatMessageAssistant, tool_message_ids: dict[str, str]
 ) -> list[ResponseInputItemParam]:
     tool_calls: list[ResponseInputItemParam] = []
     for call in message.tool_calls or []:
@@ -392,26 +422,36 @@ def _tool_call_items_from_assistant_message(
                 )
             )
         else:
-            tool_calls.append(
-                ResponseFunctionToolCallParam(
-                    type="function_call",
-                    call_id=call.id,
-                    name=call.function,
-                    arguments=call.function,
-                )
+            # create param
+            tool_call_param: ResponseFunctionToolCallParam = dict(
+                type="function_call",
+                call_id=call.id,
+                name=_responses_tool_alias(call.function),
+                arguments=json.dumps(call.arguments),
             )
 
+            # add id if available
+            tool_message_id = tool_message_ids.get(call.id, None)
+            if tool_message_id is not None:
+                tool_call_param["id"] = tool_message_id
+
+            # append the param
+            tool_calls.append(tool_call_param)
+
     return tool_calls
 
 
 def _ids_from_assistant_internal(
     message: ChatMessageAssistant,
-) -> tuple[str | None, str | None]:
-    assert isinstance(message.internal, dict), (
-        "OpenAI ChatMessageAssistant internal must be an _AssistantInternal"
-    )
-    internal = cast(_AssistantInternal, message.internal)
-    return (internal["output_message_id"], internal["reasoning_id"])
+) -> tuple[str | None, dict[str, str]]:
+    if message.internal is not None:
+        assert isinstance(message.internal, dict), (
+            "OpenAI ChatMessageAssistant internal must be an _AssistantInternal"
+        )
+        internal = cast(_AssistantInternal, message.internal)
+        return (internal["output_message_id"], internal["tool_message_ids"])
+    else:
+        return None, {}
 
 
 _ResponseToolCallParam = (
@@ -430,8 +470,21 @@ def _tool_param_for_tool_info(
     # standard tool implementation
     return _maybe_native_tool_param(tool, config) or FunctionToolParam(
         type="function",
-        name=tool.name,
+        name=_responses_tool_alias(tool.name),
         description=tool.description,
         parameters=tool.parameters.model_dump(exclude_none=True),
         strict=False,  # default parameters don't work in strict mode
     )
+
+
+# these functions enables us to 'escape' built in tool names like 'python'
+
+_responses_tool_aliases = {"python": "python_exec"}
+
+
+def _responses_tool_alias(name: str) -> str:
+    return _responses_tool_aliases.get(name, name)
+
+
+def _from_responses_tool_alias(name: str) -> str:
+    return next((k for k, v in _responses_tool_aliases.items() if v == name), name)

inspect_ai/model/_providers/anthropic.py

@@ -5,8 +5,6 @@ from copy import copy
 from logging import getLogger
 from typing import Any, Literal, Optional, Tuple, cast
 
-import httpcore
-import httpx
 from anthropic import (
     APIConnectionError,
     APIStatusError,
@@ -51,9 +49,11 @@ from inspect_ai._util.error import exception_message
 from inspect_ai._util.http import is_retryable_http_status
 from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.logger import warn_once
+from inspect_ai._util.trace import trace_message
 from inspect_ai._util.url import data_uri_mime_type, data_uri_to_base64
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
 
+from ..._util.httpx import httpx_should_retry
 from .._chat_message import ChatMessage, ChatMessageAssistant, ChatMessageSystem
 from .._generate_config import GenerateConfig
 from .._model import ModelAPI
@@ -330,13 +330,9 @@ class AnthropicAPI(ModelAPI):
     def should_retry(self, ex: Exception) -> bool:
         if isinstance(ex, APIStatusError):
             return is_retryable_http_status(ex.status_code)
-        elif isinstance(
-            ex,
-            APIConnectionError
-            | APITimeoutError
-            | httpx.RemoteProtocolError
-            | httpcore.RemoteProtocolError,
-        ):
+        elif httpx_should_retry(ex):
+            return True
+        elif isinstance(ex, APIConnectionError | APITimeoutError):
             return True
         else:
             return False
@@ -944,9 +940,15 @@ async def count_tokens(
             messages=[{"role": "user", "content": text}],
         )
         return response.input_tokens
-    except Exception as e:
-        logger.warning(
-            f"Error counting tokens (falling back to estimated tokens): {str(e)}"
+    except Exception as ex:
+        warn_once(
+            logger,
+            f"Unable to call count_tokens API for model {model} (falling back to estimated tokens)",
+        )
+        trace_message(
+            logger,
+            "Anthropic",
+            f"Unable to call count_tokens API for model {model} ({ex})",
         )
         words = text.split()
         estimated_tokens = int(len(words) * 1.3)