inspect-ai 0.3.75__py3-none-any.whl → 0.3.77__py3-none-any.whl

inspect_ai/model/_openai.py

@@ -1,7 +1,9 @@
 import json
 import re
+from copy import copy
 from typing import Literal
 
+from openai import BadRequestError, OpenAIError
 from openai.types.chat import (
     ChatCompletion,
     ChatCompletionAssistantMessageParam,
@@ -26,7 +28,9 @@ from openai.types.chat.chat_completion import Choice, ChoiceLogprobs
 from openai.types.chat.chat_completion_message_tool_call import Function
 from openai.types.completion_usage import CompletionUsage
 from openai.types.shared_params.function_definition import FunctionDefinition
+from pydantic import JsonValue
 
+from inspect_ai._util.constants import BASE_64_DATA_REMOVED
 from inspect_ai._util.content import (
     Content,
     ContentAudio,
@@ -48,23 +52,39 @@ from ._chat_message import (
     ChatMessageTool,
     ChatMessageUser,
 )
-from ._model_output import ModelUsage, StopReason, as_stop_reason
+from ._model_output import ModelOutput, ModelUsage, StopReason, as_stop_reason
+
+
+class OpenAIResponseError(OpenAIError):
+    def __init__(self, code: str, message: str) -> None:
+        self.code = code
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"{self.code}: {self.message}"
 
 
 def is_o_series(name: str) -> bool:
-    return bool(re.match(r"(^|.*\/)o\d+", name))
+    if bool(re.match(r"^o\d+", name)):
+        return True
+    else:
+        return not is_gpt(name) and bool(re.search(r"o\d+", name))
+
+
+def is_o1_pro(name: str) -> bool:
+    return "o1-pro" in name
 
 
 def is_o1_mini(name: str) -> bool:
-    return name.startswith("o1-mini")
+    return "o1-mini" in name
 
 
 def is_o1_preview(name: str) -> bool:
-    return name.startswith("o1-preview")
+    return "o1-preview" in name
 
 
 def is_gpt(name: str) -> bool:
-    return name.startswith("gpt")
+    return "gpt" in name
 
 
 def openai_chat_tool_call(tool_call: ToolCall) -> ChatCompletionMessageToolCall:
@@ -80,12 +100,13 @@ def openai_chat_tool_call(tool_call: ToolCall) -> ChatCompletionMessageToolCall:
 def openai_chat_tool_call_param(
     tool_call: ToolCall,
 ) -> ChatCompletionMessageToolCallParam:
+    assert tool_call.type == "function", f"Unexpected tool call type {tool_call.type}"
     return ChatCompletionMessageToolCallParam(
         id=tool_call.id,
         function=dict(
             name=tool_call.function, arguments=json.dumps(tool_call.arguments)
         ),
-        type=tool_call.type,
+        type="function",  # Type narrowing couldn't figure it out
     )
 
 
@@ -108,7 +129,8 @@ async def openai_chat_completion_part(
             image_url=dict(url=image_url, detail=detail),
         )
     elif content.type == "audio":
-        audio_data = await file_as_data_uri(content.audio)
+        audio_data_uri = await file_as_data_uri(content.audio)
+        audio_data = audio_data_uri.split("base64,")[1]
 
         return ChatCompletionContentPartInputAudioParam(
             type="input_audio", input_audio=dict(data=audio_data, format=content.format)
@@ -315,6 +337,7 @@ def chat_messages_from_openai(
                 chat_messages.append(ChatMessageUser(content=content))
         elif message["role"] == "assistant":
             # resolve content
+            refusal: Literal[True] | None = None
             asst_content = message.get("content", None)
             if isinstance(asst_content, str):
                 result = parse_content_with_reasoning(asst_content)
@@ -331,6 +354,8 @@ def chat_messages_from_openai(
                     content = asst_content
             elif asst_content is None:
                 content = message.get("refusal", None) or ""
+                if content:
+                    refusal = True
             else:
                 content = []
                 for ac in asst_content:
@@ -343,7 +368,7 @@ def chat_messages_from_openai(
             )
             if reasoning is not None:
                 if isinstance(content, str):
-                    content = [ContentText(text=content)]
+                    content = [ContentText(text=content, refusal=refusal)]
                 else:
                     content.insert(0, ContentReasoning(reasoning=str(reasoning)))
 
@@ -432,7 +457,7 @@ def content_from_openai(
             )
         ]
     elif content["type"] == "refusal":
-        return [ContentText(text=content["refusal"])]
+        return [ContentText(text=content["refusal"], refusal=True)]
     else:
         content_type = content["type"]
         raise ValueError(f"Unexpected content type '{content_type}' in message.")
@@ -450,8 +475,10 @@ def chat_message_assistant_from_openai(
     if reasoning is not None:
         content: str | list[Content] = [
             ContentReasoning(reasoning=str(reasoning)),
-            ContentText(text=msg_content),
+            ContentText(text=msg_content, refusal=True if refusal else None),
         ]
+    elif refusal is not None:
+        content = [ContentText(text=msg_content, refusal=True)]
     else:
         content = msg_content
 
@@ -479,3 +506,44 @@ def chat_choices_from_openai(
         )
         for choice in choices
     ]
+
+
+def openai_handle_bad_request(
+    model_name: str, e: BadRequestError
+) -> ModelOutput | Exception:
+    # extract message
+    if isinstance(e.body, dict) and "message" in e.body.keys():
+        content = str(e.body.get("message"))
+    else:
+        content = e.message
+
+    # narrow stop_reason
+    stop_reason: StopReason | None = None
+    if e.code == "context_length_exceeded":
+        stop_reason = "model_length"
+    elif (
+        e.code == "invalid_prompt"  # seems to happen for o1/o3
+        or e.code == "content_policy_violation"  # seems to happen for vision
+        or e.code == "content_filter"  # seems to happen on azure
+    ):
+        stop_reason = "content_filter"
+
+    if stop_reason:
+        return ModelOutput.from_content(
+            model=model_name, content=content, stop_reason=stop_reason
+        )
+    else:
+        return e
+
+
+def openai_media_filter(key: JsonValue | None, value: JsonValue) -> JsonValue:
+    # remove images from raw api call
+    if key == "image_url" and isinstance(value, dict) and "url" in value:
+        url = str(value.get("url"))
+        if url.startswith("data:"):
+            value = copy(value)
+            value.update(url=BASE_64_DATA_REMOVED)
+    elif key == "input_audio" and isinstance(value, dict) and "data" in value:
+        value = copy(value)
+        value.update(data=BASE_64_DATA_REMOVED)
+    return value

inspect_ai/model/_openai_responses.py

@@ -0,0 +1,277 @@
+import json
+
+from openai.types.responses import (
+    FunctionToolParam,
+    Response,
+    ResponseFunctionToolCall,
+    ResponseFunctionToolCallParam,
+    ResponseInputContentParam,
+    ResponseInputImageParam,
+    ResponseInputItemParam,
+    ResponseInputMessageContentListParam,
+    ResponseInputTextParam,
+    ResponseOutputMessage,
+    ResponseOutputMessageParam,
+    ResponseOutputRefusalParam,
+    ResponseOutputText,
+    ResponseOutputTextParam,
+    ResponseReasoningItem,
+    ResponseReasoningItemParam,
+    ToolChoiceFunctionParam,
+    ToolParam,
+)
+from openai.types.responses.response_create_params import (
+    ToolChoice as ResponsesToolChoice,
+)
+from openai.types.responses.response_input_item_param import FunctionCallOutput, Message
+from openai.types.responses.response_reasoning_item_param import Summary
+
+from inspect_ai._util.content import (
+    Content,
+    ContentImage,
+    ContentReasoning,
+    ContentText,
+)
+from inspect_ai._util.images import file_as_data_uri
+from inspect_ai._util.url import is_http_url
+from inspect_ai.model._call_tools import parse_tool_call
+from inspect_ai.model._model_output import ChatCompletionChoice, StopReason
+from inspect_ai.model._openai import is_o_series
+from inspect_ai.tool._tool_call import ToolCall
+from inspect_ai.tool._tool_choice import ToolChoice
+from inspect_ai.tool._tool_info import ToolInfo
+
+from ._chat_message import ChatMessage, ChatMessageAssistant
+
+
+async def openai_responses_inputs(
+    messages: list[ChatMessage], model: str
+) -> list[ResponseInputItemParam]:
+    responses_inputs: list[ResponseInputItemParam] = []
+    for message in messages:
+        responses_inputs.extend(await openai_responses_input(message, model))
+    return responses_inputs
+
+
+async def openai_responses_input(
+    message: ChatMessage, model: str
+) -> list[ResponseInputItemParam]:
+    if message.role == "system":
+        content = await openai_responses_content_list_param(message.content)
+        if is_o_series(model):
+            return [Message(type="message", role="developer", content=content)]
+        else:
+            return [Message(type="message", role="system", content=content)]
+    elif message.role == "user":
+        return [
+            Message(
+                type="message",
+                role="user",
+                content=await openai_responses_content_list_param(message.content),
+            )
+        ]
+    elif message.role == "assistant":
+        reasoning_content = openai_responses_reasponing_content_params(message.content)
+        if message.content:
+            text_content = [
+                ResponseOutputMessageParam(
+                    type="message",
+                    role="assistant",
+                    id=str(message.id).replace("resp_", "msg_", 1),
+                    content=openai_responses_text_content_params(message.content),
+                    status="completed",
+                )
+            ]
+        else:
+            text_content = []
+        tools_content = openai_responses_tools_content_params(message.tool_calls)
+        return reasoning_content + text_content + tools_content
+    elif message.role == "tool":
+        # TODO: Return ouptut types for internal tools e.g. computer, web_search
+        if message.error is not None:
+            output = message.error.message
+        else:
+            output = message.text
+        return [
+            FunctionCallOutput(
+                type="function_call_output",
+                call_id=message.tool_call_id or str(message.function),
+                output=output,
+            )
+        ]
+    else:
+        raise ValueError(f"Unexpected message role '{message.role}'")
+
+
+async def openai_responses_content_list_param(
+    content: str | list[Content],
+) -> ResponseInputMessageContentListParam:
+    if isinstance(content, str):
+        content = [ContentText(text=content)]
+    return [await openai_responses_content_param(c) for c in content]
+
+
+async def openai_responses_content_param(content: Content) -> ResponseInputContentParam:  # type: ignore[return]
+    if isinstance(content, ContentText):
+        return ResponseInputTextParam(type="input_text", text=content.text)
+    elif isinstance(content, ContentImage):
+        image_url = content.image
+        if not is_http_url(image_url):
+            image_url = await file_as_data_uri(image_url)
+
+        return ResponseInputImageParam(
+            type="input_image", detail=content.detail, image_url=image_url
+        )
+    else:
+        # TODO: support for files (PDFs) and audio and video whenever
+        # that is supported by the responses API (was not on initial release)
+
+        # TODO: note that when doing this we should ensure that the
+        # openai_media_filter is properly screening out base64 encoded
+        # audio and video (if it exists, looks like it may all be done
+        # w/ file uploads in the responses API)
+
+        raise ValueError("Unsupported content type.")
+
+
+def openai_responses_reasponing_content_params(
+    content: str | list[Content],
+) -> list[ResponseInputItemParam]:
+    if isinstance(content, list):
+        return [
+            ResponseReasoningItemParam(
+                type="reasoning",
+                id=str(c.signature),
+                summary=[Summary(type="summary_text", text=c.reasoning)],
+            )
+            for c in content
+            if isinstance(c, ContentReasoning)
+        ]
+    else:
+        return []
+
+
+def openai_responses_text_content_params(
+    content: str | list[Content],
+) -> list[ResponseOutputTextParam | ResponseOutputRefusalParam]:
+    if isinstance(content, str):
+        content = [ContentText(text=content)]
+
+    params: list[ResponseOutputTextParam | ResponseOutputRefusalParam] = []
+
+    for c in content:
+        if isinstance(c, ContentText):
+            if c.refusal:
+                params.append(
+                    ResponseOutputRefusalParam(type="refusal", refusal=c.text)
+                )
+            else:
+                params.append(
+                    ResponseOutputTextParam(
+                        type="output_text", text=c.text, annotations=[]
+                    )
+                )
+
+    return params
+
+
+def openai_responses_tools_content_params(
+    tool_calls: list[ToolCall] | None,
+) -> list[ResponseInputItemParam]:
+    if tool_calls is not None:
+        return [
+            ResponseFunctionToolCallParam(
+                type="function_call",
+                call_id=call.id,
+                name=call.function,
+                arguments=json.dumps(call.arguments),
+                status="completed",
+            )
+            for call in tool_calls
+        ]
+    else:
+        return []
+
+
+def openai_responses_tool_choice(tool_choice: ToolChoice) -> ResponsesToolChoice:
+    match tool_choice:
+        case "none" | "auto":
+            return tool_choice
+        case "any":
+            return "required"
+        # TODO: internal tools need to be converted to ToolChoiceTypesParam
+        case _:
+            return ToolChoiceFunctionParam(type="function", name=tool_choice.name)
+
+
+def openai_responses_tools(tools: list[ToolInfo]) -> list[ToolParam]:
+    # TODO: return special types for internal tools
+    return [
+        FunctionToolParam(
+            type="function",
+            name=tool.name,
+            description=tool.description,
+            parameters=tool.parameters.model_dump(exclude_none=True),
+            strict=False,  # default parameters don't work in strict mode
+        )
+        for tool in tools
+    ]
+
+
+def openai_responses_chat_choices(
+    response: Response, tools: list[ToolInfo]
+) -> list[ChatCompletionChoice]:
+    # determine the StopReason
+    stop_reason: StopReason = "stop"
+    if response.incomplete_details is not None:
+        if response.incomplete_details.reason == "max_output_tokens":
+            stop_reason = "max_tokens"
+        elif response.incomplete_details.reason == "content_filter":
+            stop_reason = "content_filter"
+
+    # collect output and tool calls
+    message_content: list[Content] = []
+    tool_calls: list[ToolCall] = []
+    for output in response.output:
+        if isinstance(output, ResponseOutputMessage):
+            for content in output.content:
+                if isinstance(content, ResponseOutputText):
+                    message_content.append(ContentText(text=content.text))
+                else:
+                    message_content.append(
+                        ContentText(text=content.refusal, refusal=True)
+                    )
+        elif isinstance(output, ResponseReasoningItem):
+            reasoning = "\n".join([summary.text for summary in output.summary])
+            if reasoning:
+                message_content.append(
+                    ContentReasoning(signature=output.id, reasoning=reasoning)
+                )
+        else:
+            stop_reason = "tool_calls"
+            if isinstance(output, ResponseFunctionToolCall):
+                tool_calls.append(
+                    parse_tool_call(
+                        output.call_id,
+                        output.name,
+                        output.arguments,
+                        tools,
+                    )
+                )
+                pass
+            else:
+                ## TODO: implement support for internal tools
+                raise ValueError(f"Unexpected output type: {output.__class__}")
+
+    # return choice
+    return [
+        ChatCompletionChoice(
+            message=ChatMessageAssistant(
+                id=response.id,
+                content=message_content,
+                tool_calls=tool_calls if len(tool_calls) > 0 else None,
+                source="generate",
+            ),
+            stop_reason=stop_reason,
+        )
+    ]