inspect-ai 0.3.76__py3-none-any.whl → 0.3.78__py3-none-any.whl

inspect_ai/model/_openai_responses.py

@@ -0,0 +1,283 @@
+import json
+
+from openai.types.responses import (
+    FunctionToolParam,
+    Response,
+    ResponseFunctionToolCall,
+    ResponseFunctionToolCallParam,
+    ResponseInputContentParam,
+    ResponseInputImageParam,
+    ResponseInputItemParam,
+    ResponseInputMessageContentListParam,
+    ResponseInputTextParam,
+    ResponseOutputMessage,
+    ResponseOutputMessageParam,
+    ResponseOutputRefusalParam,
+    ResponseOutputText,
+    ResponseOutputTextParam,
+    ResponseReasoningItem,
+    ResponseReasoningItemParam,
+    ToolChoiceFunctionParam,
+    ToolParam,
+)
+from openai.types.responses.response_create_params import (
+    ToolChoice as ResponsesToolChoice,
+)
+from openai.types.responses.response_input_item_param import FunctionCallOutput, Message
+from openai.types.responses.response_reasoning_item_param import Summary
+
+from inspect_ai._util.content import (
+    Content,
+    ContentImage,
+    ContentReasoning,
+    ContentText,
+)
+from inspect_ai._util.images import file_as_data_uri
+from inspect_ai._util.url import is_http_url
+from inspect_ai.model._call_tools import parse_tool_call
+from inspect_ai.model._model_output import ChatCompletionChoice, StopReason
+from inspect_ai.model._openai import is_o_series
+from inspect_ai.tool._tool_call import ToolCall
+from inspect_ai.tool._tool_choice import ToolChoice
+from inspect_ai.tool._tool_info import ToolInfo
+
+from ._chat_message import ChatMessage, ChatMessageAssistant
+
+
+async def openai_responses_inputs(
+    messages: list[ChatMessage], model: str
+) -> list[ResponseInputItemParam]:
+    responses_inputs: list[ResponseInputItemParam] = []
+    for message in messages:
+        responses_inputs.extend(await openai_responses_input(message, model))
+    return responses_inputs
+
+
+async def openai_responses_input(
+    message: ChatMessage, model: str
+) -> list[ResponseInputItemParam]:
+    if message.role == "system":
+        content = await openai_responses_content_list_param(message.content)
+        if is_o_series(model):
+            return [Message(type="message", role="developer", content=content)]
+        else:
+            return [Message(type="message", role="system", content=content)]
+    elif message.role == "user":
+        return [
+            Message(
+                type="message",
+                role="user",
+                content=await openai_responses_content_list_param(message.content),
+            )
+        ]
+    elif message.role == "assistant":
+        reasoning_content = openai_responses_reasponing_content_params(message.content)
+        if message.content:
+            formatted_id = str(message.id).replace("resp_", "msg_", 1)
+            if not formatted_id.startswith("msg_"):
+                # These messages MUST start with `msg_`.
+                # As `store=False` for this provider, OpenAI doesn't validate the IDs.
+                # This will keep them consistent across calls though.
+                formatted_id = f"msg_{formatted_id}"
+            text_content = [
+                ResponseOutputMessageParam(
+                    type="message",
+                    role="assistant",
+                    id=formatted_id,
+                    content=openai_responses_text_content_params(message.content),
+                    status="completed",
+                )
+            ]
+        else:
+            text_content = []
+        tools_content = openai_responses_tools_content_params(message.tool_calls)
+        return reasoning_content + text_content + tools_content
+    elif message.role == "tool":
+        # TODO: Return ouptut types for internal tools e.g. computer, web_search
+        if message.error is not None:
+            output = message.error.message
+        else:
+            output = message.text
+        return [
+            FunctionCallOutput(
+                type="function_call_output",
+                call_id=message.tool_call_id or str(message.function),
+                output=output,
+            )
+        ]
+    else:
+        raise ValueError(f"Unexpected message role '{message.role}'")
+
+
+async def openai_responses_content_list_param(
+    content: str | list[Content],
+) -> ResponseInputMessageContentListParam:
+    if isinstance(content, str):
+        content = [ContentText(text=content)]
+    return [await openai_responses_content_param(c) for c in content]
+
+
+async def openai_responses_content_param(content: Content) -> ResponseInputContentParam:  # type: ignore[return]
+    if isinstance(content, ContentText):
+        return ResponseInputTextParam(type="input_text", text=content.text)
+    elif isinstance(content, ContentImage):
+        image_url = content.image
+        if not is_http_url(image_url):
+            image_url = await file_as_data_uri(image_url)
+
+        return ResponseInputImageParam(
+            type="input_image", detail=content.detail, image_url=image_url
+        )
+    else:
+        # TODO: support for files (PDFs) and audio and video whenever
+        # that is supported by the responses API (was not on initial release)
+
+        # TODO: note that when doing this we should ensure that the
+        # openai_media_filter is properly screening out base64 encoded
+        # audio and video (if it exists, looks like it may all be done
+        # w/ file uploads in the responses API)
+
+        raise ValueError("Unsupported content type.")
+
+
+def openai_responses_reasponing_content_params(
+    content: str | list[Content],
+) -> list[ResponseInputItemParam]:
+    if isinstance(content, list):
+        return [
+            ResponseReasoningItemParam(
+                type="reasoning",
+                id=str(c.signature),
+                summary=[Summary(type="summary_text", text=c.reasoning)],
+            )
+            for c in content
+            if isinstance(c, ContentReasoning)
+        ]
+    else:
+        return []
+
+
+def openai_responses_text_content_params(
+    content: str | list[Content],
+) -> list[ResponseOutputTextParam | ResponseOutputRefusalParam]:
+    if isinstance(content, str):
+        content = [ContentText(text=content)]
+
+    params: list[ResponseOutputTextParam | ResponseOutputRefusalParam] = []
+
+    for c in content:
+        if isinstance(c, ContentText):
+            if c.refusal:
+                params.append(
+                    ResponseOutputRefusalParam(type="refusal", refusal=c.text)
+                )
+            else:
+                params.append(
+                    ResponseOutputTextParam(
+                        type="output_text", text=c.text, annotations=[]
+                    )
+                )
+
+    return params
+
+
+def openai_responses_tools_content_params(
+    tool_calls: list[ToolCall] | None,
+) -> list[ResponseInputItemParam]:
+    if tool_calls is not None:
+        return [
+            ResponseFunctionToolCallParam(
+                type="function_call",
+                call_id=call.id,
+                name=call.function,
+                arguments=json.dumps(call.arguments),
+                status="completed",
+            )
+            for call in tool_calls
+        ]
+    else:
+        return []
+
+
+def openai_responses_tool_choice(tool_choice: ToolChoice) -> ResponsesToolChoice:
+    match tool_choice:
+        case "none" | "auto":
+            return tool_choice
+        case "any":
+            return "required"
+        # TODO: internal tools need to be converted to ToolChoiceTypesParam
+        case _:
+            return ToolChoiceFunctionParam(type="function", name=tool_choice.name)
+
+
+def openai_responses_tools(tools: list[ToolInfo]) -> list[ToolParam]:
+    # TODO: return special types for internal tools
+    return [
+        FunctionToolParam(
+            type="function",
+            name=tool.name,
+            description=tool.description,
+            parameters=tool.parameters.model_dump(exclude_none=True),
+            strict=False,  # default parameters don't work in strict mode
+        )
+        for tool in tools
+    ]
+
+
+def openai_responses_chat_choices(
+    response: Response, tools: list[ToolInfo]
+) -> list[ChatCompletionChoice]:
+    # determine the StopReason
+    stop_reason: StopReason = "stop"
+    if response.incomplete_details is not None:
+        if response.incomplete_details.reason == "max_output_tokens":
+            stop_reason = "max_tokens"
+        elif response.incomplete_details.reason == "content_filter":
+            stop_reason = "content_filter"
+
+    # collect output and tool calls
+    message_content: list[Content] = []
+    tool_calls: list[ToolCall] = []
+    for output in response.output:
+        if isinstance(output, ResponseOutputMessage):
+            for content in output.content:
+                if isinstance(content, ResponseOutputText):
+                    message_content.append(ContentText(text=content.text))
+                else:
+                    message_content.append(
+                        ContentText(text=content.refusal, refusal=True)
+                    )
+        elif isinstance(output, ResponseReasoningItem):
+            reasoning = "\n".join([summary.text for summary in output.summary])
+            if reasoning:
+                message_content.append(
+                    ContentReasoning(signature=output.id, reasoning=reasoning)
+                )
+        else:
+            stop_reason = "tool_calls"
+            if isinstance(output, ResponseFunctionToolCall):
+                tool_calls.append(
+                    parse_tool_call(
+                        output.call_id,
+                        output.name,
+                        output.arguments,
+                        tools,
+                    )
+                )
+                pass
+            else:
+                ## TODO: implement support for internal tools
+                raise ValueError(f"Unexpected output type: {output.__class__}")
+
+    # return choice
+    return [
+        ChatCompletionChoice(
+            message=ChatMessageAssistant(
+                id=response.id,
+                content=message_content,
+                tool_calls=tool_calls if len(tool_calls) > 0 else None,
+                source="generate",
+            ),
+            stop_reason=stop_reason,
+        )
+    ]

inspect_ai/model/_providers/anthropic.py

@@ -815,6 +815,7 @@ async def model_output_from_message(
         + (input_tokens_cache_write or 0)
         + (input_tokens_cache_read or 0)
         + message.usage.output_tokens
+        + reasoning_tokens
     )
     return ModelOutput(
         model=message.model,

inspect_ai/model/_providers/azureai.py

@@ -51,7 +51,6 @@ from .._chat_message import (
     ChatMessageUser,
 )
 from .._generate_config import GenerateConfig
-from .._image import image_url_filter
 from .._model import ModelAPI
 from .._model_call import ModelCall
 from .._model_output import (
@@ -60,6 +59,7 @@ from .._model_output import (
     ModelUsage,
     StopReason,
 )
+from .._openai import openai_media_filter
 from .util import (
     environment_prerequisite_error,
     model_base_url,
@@ -182,7 +182,7 @@ class AzureAIAPI(ModelAPI):
                     else None,
                 ),
                 response=response.as_dict() if response else {},
-                filter=image_url_filter,
+                filter=openai_media_filter,
             )
 
         # make call

inspect_ai/model/_providers/mistral.py

@@ -82,6 +82,14 @@ class MistralAPI(ModelAPI):
         config: GenerateConfig = GenerateConfig(),
         **model_args: Any,
     ):
+        # extract any service prefix from model name
+        parts = model_name.split("/")
+        if len(parts) > 1:
+            self.service: str | None = parts[0]
+            model_name = "/".join(parts[1:])
+        else:
+            self.service = None
+
         super().__init__(
             model_name=model_name,
             base_url=base_url,
@@ -94,31 +102,39 @@ class MistralAPI(ModelAPI):
             config=config,
         )
 
-        # resolve api_key -- look for mistral then azure
+        # resolve api_key
         if not self.api_key:
-            self.api_key = os.environ.get(MISTRAL_API_KEY, None)
-            if self.api_key:
-                base_url = model_base_url(base_url, "MISTRAL_BASE_URL")
-            else:
+            if self.is_azure():
                 self.api_key = os.environ.get(
                     AZUREAI_MISTRAL_API_KEY, os.environ.get(AZURE_MISTRAL_API_KEY, None)
                 )
-                if not self.api_key:
-                    raise environment_prerequisite_error(
-                        "Mistral", [MISTRAL_API_KEY, AZUREAI_MISTRAL_API_KEY]
-                    )
-                base_url = model_base_url(base_url, "AZUREAI_MISTRAL_BASE_URL")
-                if not base_url:
+            else:
+                self.api_key = os.environ.get(MISTRAL_API_KEY, None)
+
+            if not self.api_key:
+                raise environment_prerequisite_error(
+                    "Mistral", [MISTRAL_API_KEY, AZUREAI_MISTRAL_API_KEY]
+                )
+
+        if not self.base_url:
+            if self.is_azure():
+                self.base_url = model_base_url(base_url, "AZUREAI_MISTRAL_BASE_URL")
+                if not self.base_url:
                     raise ValueError(
                         "You must provide a base URL when using Mistral on Azure. Use the AZUREAI_MISTRAL_BASE_URL "
                         + " environment variable or the --model-base-url CLI flag to set the base URL."
                     )
+            else:
+                self.base_url = model_base_url(base_url, "MISTRAL_BASE_URL")
 
-        if base_url:
-            model_args["server_url"] = base_url
+        if self.base_url:
+            model_args["server_url"] = self.base_url
 
         self.model_args = model_args
 
+    def is_azure(self) -> bool:
+        return self.service == "azure"
+
     @override
     async def close(self) -> None:
         # client is created and destroyed in generate

inspect_ai/model/_providers/openai.py

@@ -22,28 +22,27 @@ from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.http import is_retryable_http_status
 from inspect_ai._util.logger import warn_once
 from inspect_ai.model._openai import chat_choices_from_openai
+from inspect_ai.model._providers.openai_responses import generate_responses
 from inspect_ai.model._providers.util.hooks import HttpxHooks
 from inspect_ai.tool import ToolChoice, ToolInfo
 
 from .._chat_message import ChatMessage
 from .._generate_config import GenerateConfig
-from .._image import image_url_filter
 from .._model import ModelAPI
 from .._model_call import ModelCall
-from .._model_output import (
-    ChatCompletionChoice,
-    ModelOutput,
-    ModelUsage,
-    StopReason,
-)
+from .._model_output import ChatCompletionChoice, ModelOutput, ModelUsage
 from .._openai import (
+    OpenAIResponseError,
     is_gpt,
     is_o1_mini,
     is_o1_preview,
+    is_o1_pro,
     is_o_series,
     openai_chat_messages,
     openai_chat_tool_choice,
     openai_chat_tools,
+    openai_handle_bad_request,
+    openai_media_filter,
 )
 from .openai_o1 import generate_o1
 from .util import (
@@ -65,6 +64,7 @@ class OpenAIAPI(ModelAPI):
         base_url: str | None = None,
         api_key: str | None = None,
         config: GenerateConfig = GenerateConfig(),
+        responses_api: bool | None = None,
         **model_args: Any,
     ) -> None:
         # extract azure service prefix from model name (other providers
@@ -77,6 +77,9 @@ class OpenAIAPI(ModelAPI):
         else:
             self.service = None
 
+        # note whether we are forcing the responses_api
+        self.responses_api = True if responses_api else False
+
         # call super
         super().__init__(
             model_name=model_name,
@@ -88,22 +91,21 @@ class OpenAIAPI(ModelAPI):
 
         # resolve api_key
         if not self.api_key:
-            self.api_key = os.environ.get(
-                AZUREAI_OPENAI_API_KEY, os.environ.get(AZURE_OPENAI_API_KEY, None)
-            )
-            # backward compatibility for when env vars determined service
-            if self.api_key and (os.environ.get(OPENAI_API_KEY, None) is None):
-                self.service = "azure"
+            if self.service == "azure":
+                self.api_key = os.environ.get(
+                    AZUREAI_OPENAI_API_KEY, os.environ.get(AZURE_OPENAI_API_KEY, None)
+                )
             else:
                 self.api_key = os.environ.get(OPENAI_API_KEY, None)
-                if not self.api_key:
-                    raise environment_prerequisite_error(
-                        "OpenAI",
-                        [
-                            OPENAI_API_KEY,
-                            AZUREAI_OPENAI_API_KEY,
-                        ],
-                    )
+
+            if not self.api_key:
+                raise environment_prerequisite_error(
+                    "OpenAI",
+                    [
+                        OPENAI_API_KEY,
+                        AZUREAI_OPENAI_API_KEY,
+                    ],
+                )
 
         # create async http client
         http_client = OpenAIAsyncHttpxClient()
@@ -125,10 +127,16 @@ class OpenAIAPI(ModelAPI):
                     + "environment variable or the --model-base-url CLI flag to set the base URL."
                 )
 
+            # resolve version
+            api_version = os.environ.get(
+                "AZUREAI_OPENAI_API_VERSION",
+                os.environ.get("OPENAI_API_VERSION", "2025-02-01-preview"),
+            )
+
             self.client: AsyncAzureOpenAI | AsyncOpenAI = AsyncAzureOpenAI(
                 api_key=self.api_key,
+                api_version=api_version,
                 azure_endpoint=base_url,
-                azure_deployment=model_name,
                 http_client=http_client,
                 **model_args,
             )
@@ -149,6 +157,9 @@ class OpenAIAPI(ModelAPI):
     def is_o_series(self) -> bool:
         return is_o_series(self.model_name)
 
+    def is_o1_pro(self) -> bool:
+        return is_o1_pro(self.model_name)
+
     def is_o1_mini(self) -> bool:
         return is_o1_mini(self.model_name)
 
@@ -177,6 +188,16 @@ class OpenAIAPI(ModelAPI):
                 tools=tools,
                 **self.completion_params(config, False),
             )
+        elif self.is_o1_pro() or self.responses_api:
+            return await generate_responses(
+                client=self.client,
+                http_hooks=self._http_hooks,
+                model_name=self.model_name,
+                input=input,
+                tools=tools,
+                tool_choice=tool_choice,
+                config=config,
+            )
 
         # allocate request_id (so we can see it from ModelCall)
         request_id = self._http_hooks.start_request()
@@ -189,7 +210,7 @@ class OpenAIAPI(ModelAPI):
             return ModelCall.create(
                 request=request,
                 response=response,
-                filter=image_url_filter,
+                filter=openai_media_filter,
                 time=self._http_hooks.end_request(request_id),
             )
 
@@ -221,6 +242,7 @@ class OpenAIAPI(ModelAPI):
 
             # save response for model_call
             response = completion.model_dump()
+            self.on_response(response)
 
             # parse out choices
             choices = self._chat_choices_from_response(completion, tools)
@@ -252,6 +274,12 @@ class OpenAIAPI(ModelAPI):
         except BadRequestError as e:
             return self.handle_bad_request(e), model_call()
 
+    def on_response(self, response: dict[str, Any]) -> None:
+        pass
+
+    def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | Exception:
+        return openai_handle_bad_request(self.model_name, ex)
+
     def _chat_choices_from_response(
         self, response: ChatCompletion, tools: list[ToolInfo]
     ) -> list[ChatCompletionChoice]:
@@ -270,6 +298,8 @@ class OpenAIAPI(ModelAPI):
                 return True
         elif isinstance(ex, APIStatusError):
             return is_retryable_http_status(ex.status_code)
+        elif isinstance(ex, OpenAIResponseError):
+            return ex.code in ["rate_limit_exceeded", "server_error"]
         elif isinstance(ex, APITimeoutError):
             return True
         else:
@@ -342,32 +372,6 @@ class OpenAIAPI(ModelAPI):
 
         return params
 
-    # convert some well known bad request errors into ModelOutput
-    def handle_bad_request(self, e: BadRequestError) -> ModelOutput | Exception:
-        # extract message
-        if isinstance(e.body, dict) and "message" in e.body.keys():
-            content = str(e.body.get("message"))
-        else:
-            content = e.message
-
-        # narrow stop_reason
-        stop_reason: StopReason | None = None
-        if e.code == "context_length_exceeded":
-            stop_reason = "model_length"
-        elif (
-            e.code == "invalid_prompt"  # seems to happen for o1/o3
-            or e.code == "content_policy_violation"  # seems to happen for vision
-            or e.code == "content_filter"  # seems to happen on azure
-        ):
-            stop_reason = "content_filter"
-
-        if stop_reason:
-            return ModelOutput.from_content(
-                model=self.model_name, content=content, stop_reason=stop_reason
-            )
-        else:
-            return e
-
 
 class OpenAIAsyncHttpxClient(httpx.AsyncClient):
     """Custom async client that deals better with long running Async requests.