openai-agents 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

agents/model_settings.py

@@ -17,9 +17,9 @@ from typing_extensions import TypeAlias
 class _OmitTypeAnnotation:
     @classmethod
     def __get_pydantic_core_schema__(
-            cls,
-            _source_type: Any,
-            _handler: GetCoreSchemaHandler,
+        cls,
+        _source_type: Any,
+        _handler: GetCoreSchemaHandler,
     ) -> core_schema.CoreSchema:
         def validate_from_none(value: None) -> _Omit:
             return _Omit()
@@ -39,12 +39,20 @@ class _OmitTypeAnnotation:
                     from_none_schema,
                 ]
             ),
-            serialization=core_schema.plain_serializer_function_ser_schema(
-                lambda instance: None
-            ),
+            serialization=core_schema.plain_serializer_function_ser_schema(lambda instance: None),
         )
+
+
+@dataclass
+class MCPToolChoice:
+    server_label: str
+    name: str
+
+
 Omit = Annotated[_Omit, _OmitTypeAnnotation]
 Headers: TypeAlias = Mapping[str, Union[str, Omit]]
+ToolChoice: TypeAlias = Union[Literal["auto", "required", "none"], str, MCPToolChoice, None]
+
 
 @dataclass
 class ModelSettings:
@@ -69,7 +77,7 @@ class ModelSettings:
     presence_penalty: float | None = None
     """The presence penalty to use when calling the model."""
 
-    tool_choice: Literal["auto", "required", "none"] | str | None = None
+    tool_choice: ToolChoice | None = None
     """The tool choice to use when calling the model."""
 
     parallel_tool_calls: bool | None = None

agents/models/chatcmpl_converter.py

@@ -19,6 +19,7 @@ from openai.types.chat import (
     ChatCompletionToolMessageParam,
     ChatCompletionUserMessageParam,
 )
+from openai.types.chat.chat_completion_content_part_param import File, FileFile
 from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
 from openai.types.chat.completion_create_params import ResponseFormat
 from openai.types.responses import (
@@ -27,6 +28,7 @@ from openai.types.responses import (
     ResponseFunctionToolCall,
     ResponseFunctionToolCallParam,
     ResponseInputContentParam,
+    ResponseInputFileParam,
     ResponseInputImageParam,
     ResponseInputTextParam,
     ResponseOutputMessage,
@@ -42,6 +44,7 @@ from ..agent_output import AgentOutputSchemaBase
 from ..exceptions import AgentsException, UserError
 from ..handoffs import Handoff
 from ..items import TResponseInputItem, TResponseOutputItem
+from ..model_settings import MCPToolChoice
 from ..tool import FunctionTool, Tool
 from .fake_id import FAKE_RESPONSES_ID
 
@@ -49,10 +52,12 @@ from .fake_id import FAKE_RESPONSES_ID
 class Converter:
     @classmethod
     def convert_tool_choice(
-        cls, tool_choice: Literal["auto", "required", "none"] | str | None
+        cls, tool_choice: Literal["auto", "required", "none"] | str | MCPToolChoice | None
     ) -> ChatCompletionToolChoiceOptionParam | NotGiven:
         if tool_choice is None:
             return NOT_GIVEN
+        elif isinstance(tool_choice, MCPToolChoice):
+            raise UserError("MCPToolChoice is not supported for Chat Completions models")
         elif tool_choice == "auto":
             return "auto"
         elif tool_choice == "required":
@@ -251,7 +256,19 @@ class Converter:
                     )
                 )
             elif isinstance(c, dict) and c.get("type") == "input_file":
-                raise UserError(f"File uploads are not supported for chat completions {c}")
+                casted_file_param = cast(ResponseInputFileParam, c)
+                if "file_data" not in casted_file_param or not casted_file_param["file_data"]:
+                    raise UserError(
+                        f"Only file_data is supported for input_file {casted_file_param}"
+                    )
+                out.append(
+                    File(
+                        type="file",
+                        file=FileFile(
+                            file_data=casted_file_param["file_data"],
+                        ),
+                    )
+                )
             else:
                 raise UserError(f"Unknown content: {c}")
         return out
@@ -467,7 +484,7 @@ class Converter:
         )
 
     @classmethod
-    def convert_handoff_tool(cls, handoff: Handoff[Any]) -> ChatCompletionToolParam:
+    def convert_handoff_tool(cls, handoff: Handoff[Any, Any]) -> ChatCompletionToolParam:
         return {
             "type": "function",
             "function": {

agents/models/chatcmpl_stream_handler.py

@@ -53,6 +53,9 @@ class StreamingState:
     refusal_content_index_and_output: tuple[int, ResponseOutputRefusal] | None = None
     reasoning_content_index_and_output: tuple[int, ResponseReasoningItem] | None = None
     function_calls: dict[int, ResponseFunctionToolCall] = field(default_factory=dict)
+    # Fields for real-time function call streaming
+    function_call_streaming: dict[int, bool] = field(default_factory=dict)
+    function_call_output_idx: dict[int, int] = field(default_factory=dict)
 
 
 class SequenceNumber:
@@ -255,9 +258,7 @@ class ChatCmplStreamHandler:
                 # Accumulate the refusal string in the output part
                 state.refusal_content_index_and_output[1].refusal += delta.refusal
 
-            # Handle tool calls
-            # Because we don't know the name of the function until the end of the stream, we'll
-            # save everything and yield events at the end
+            # Handle tool calls with real-time streaming support
             if delta.tool_calls:
                 for tc_delta in delta.tool_calls:
                     if tc_delta.index not in state.function_calls:
@@ -268,15 +269,76 @@ class ChatCmplStreamHandler:
                             type="function_call",
                             call_id="",
                         )
+                        state.function_call_streaming[tc_delta.index] = False
+
                     tc_function = tc_delta.function
 
+                    # Accumulate arguments as they come in
                     state.function_calls[tc_delta.index].arguments += (
                         tc_function.arguments if tc_function else ""
                     ) or ""
-                    state.function_calls[tc_delta.index].name += (
-                        tc_function.name if tc_function else ""
-                    ) or ""
-                    state.function_calls[tc_delta.index].call_id += tc_delta.id or ""
+
+                    # Set function name directly (it's correct from the first function call chunk)
+                    if tc_function and tc_function.name:
+                        state.function_calls[tc_delta.index].name = tc_function.name
+
+                    if tc_delta.id:
+                        state.function_calls[tc_delta.index].call_id = tc_delta.id
+
+                    function_call = state.function_calls[tc_delta.index]
+
+                    # Start streaming as soon as we have function name and call_id
+                    if (not state.function_call_streaming[tc_delta.index] and
+                        function_call.name and
+                        function_call.call_id):
+
+                        # Calculate the output index for this function call
+                        function_call_starting_index = 0
+                        if state.reasoning_content_index_and_output:
+                            function_call_starting_index += 1
+                        if state.text_content_index_and_output:
+                            function_call_starting_index += 1
+                        if state.refusal_content_index_and_output:
+                            function_call_starting_index += 1
+
+                        # Add offset for already started function calls
+                        function_call_starting_index += sum(
+                            1 for streaming in state.function_call_streaming.values() if streaming
+                        )
+
+                        # Mark this function call as streaming and store its output index
+                        state.function_call_streaming[tc_delta.index] = True
+                        state.function_call_output_idx[
+                            tc_delta.index
+                        ] = function_call_starting_index
+
+                        # Send initial function call added event
+                        yield ResponseOutputItemAddedEvent(
+                            item=ResponseFunctionToolCall(
+                                id=FAKE_RESPONSES_ID,
+                                call_id=function_call.call_id,
+                                arguments="",  # Start with empty arguments
+                                name=function_call.name,
+                                type="function_call",
+                            ),
+                            output_index=function_call_starting_index,
+                            type="response.output_item.added",
+                            sequence_number=sequence_number.get_and_increment(),
+                        )
+
+                    # Stream arguments if we've started streaming this function call
+                    if (state.function_call_streaming.get(tc_delta.index, False) and
+                        tc_function and
+                        tc_function.arguments):
+
+                        output_index = state.function_call_output_idx[tc_delta.index]
+                        yield ResponseFunctionCallArgumentsDeltaEvent(
+                            delta=tc_function.arguments,
+                            item_id=FAKE_RESPONSES_ID,
+                            output_index=output_index,
+                            type="response.function_call_arguments.delta",
+                            sequence_number=sequence_number.get_and_increment(),
+                        )
 
         if state.reasoning_content_index_and_output:
             yield ResponseReasoningSummaryPartDoneEvent(
@@ -327,42 +389,71 @@ class ChatCmplStreamHandler:
                 sequence_number=sequence_number.get_and_increment(),
             )
 
-        # Actually send events for the function calls
-        for function_call in state.function_calls.values():
-            # First, a ResponseOutputItemAdded for the function call
-            yield ResponseOutputItemAddedEvent(
-                item=ResponseFunctionToolCall(
-                    id=FAKE_RESPONSES_ID,
-                    call_id=function_call.call_id,
-                    arguments=function_call.arguments,
-                    name=function_call.name,
-                    type="function_call",
-                ),
-                output_index=function_call_starting_index,
-                type="response.output_item.added",
-                sequence_number=sequence_number.get_and_increment(),
-            )
-            # Then, yield the args
-            yield ResponseFunctionCallArgumentsDeltaEvent(
-                delta=function_call.arguments,
-                item_id=FAKE_RESPONSES_ID,
-                output_index=function_call_starting_index,
-                type="response.function_call_arguments.delta",
-                sequence_number=sequence_number.get_and_increment(),
-            )
-            # Finally, the ResponseOutputItemDone
-            yield ResponseOutputItemDoneEvent(
-                item=ResponseFunctionToolCall(
-                    id=FAKE_RESPONSES_ID,
-                    call_id=function_call.call_id,
-                    arguments=function_call.arguments,
-                    name=function_call.name,
-                    type="function_call",
-                ),
-                output_index=function_call_starting_index,
-                type="response.output_item.done",
-                sequence_number=sequence_number.get_and_increment(),
-            )
+        # Send completion events for function calls
+        for index, function_call in state.function_calls.items():
+            if state.function_call_streaming.get(index, False):
+                # Function call was streamed, just send the completion event
+                output_index = state.function_call_output_idx[index]
+                yield ResponseOutputItemDoneEvent(
+                    item=ResponseFunctionToolCall(
+                        id=FAKE_RESPONSES_ID,
+                        call_id=function_call.call_id,
+                        arguments=function_call.arguments,
+                        name=function_call.name,
+                        type="function_call",
+                    ),
+                    output_index=output_index,
+                    type="response.output_item.done",
+                    sequence_number=sequence_number.get_and_increment(),
+                )
+            else:
+                # Function call was not streamed (fallback to old behavior)
+                # This handles edge cases where function name never arrived
+                fallback_starting_index = 0
+                if state.reasoning_content_index_and_output:
+                    fallback_starting_index += 1
+                if state.text_content_index_and_output:
+                    fallback_starting_index += 1
+                if state.refusal_content_index_and_output:
+                    fallback_starting_index += 1
+
+                # Add offset for already started function calls
+                fallback_starting_index += sum(
+                    1 for streaming in state.function_call_streaming.values() if streaming
+                )
+
+                # Send all events at once (backward compatibility)
+                yield ResponseOutputItemAddedEvent(
+                    item=ResponseFunctionToolCall(
+                        id=FAKE_RESPONSES_ID,
+                        call_id=function_call.call_id,
+                        arguments=function_call.arguments,
+                        name=function_call.name,
+                        type="function_call",
+                    ),
+                    output_index=fallback_starting_index,
+                    type="response.output_item.added",
+                    sequence_number=sequence_number.get_and_increment(),
+                )
+                yield ResponseFunctionCallArgumentsDeltaEvent(
+                    delta=function_call.arguments,
+                    item_id=FAKE_RESPONSES_ID,
+                    output_index=fallback_starting_index,
+                    type="response.function_call_arguments.delta",
+                    sequence_number=sequence_number.get_and_increment(),
+                )
+                yield ResponseOutputItemDoneEvent(
+                    item=ResponseFunctionToolCall(
+                        id=FAKE_RESPONSES_ID,
+                        call_id=function_call.call_id,
+                        arguments=function_call.arguments,
+                        name=function_call.name,
+                        type="function_call",
+                    ),
+                    output_index=fallback_starting_index,
+                    type="response.output_item.done",
+                    sequence_number=sequence_number.get_and_increment(),
+                )
 
         # Finally, send the Response completed event
         outputs: list[ResponseOutputItem] = []

agents/models/openai_responses.py

@@ -25,6 +25,7 @@ from ..exceptions import UserError
 from ..handoffs import Handoff
 from ..items import ItemHelpers, ModelResponse, TResponseInputItem
 from ..logger import logger
+from ..model_settings import MCPToolChoice
 from ..tool import (
     CodeInterpreterTool,
     ComputerTool,
@@ -303,10 +304,16 @@ class ConvertedTools:
 class Converter:
     @classmethod
     def convert_tool_choice(
-        cls, tool_choice: Literal["auto", "required", "none"] | str | None
+        cls, tool_choice: Literal["auto", "required", "none"] | str | MCPToolChoice | None
     ) -> response_create_params.ToolChoice | NotGiven:
         if tool_choice is None:
             return NOT_GIVEN
+        elif isinstance(tool_choice, MCPToolChoice):
+            return {
+                "server_label": tool_choice.server_label,
+                "type": "mcp",
+                "name": tool_choice.name,
+            }
         elif tool_choice == "required":
             return "required"
         elif tool_choice == "auto":
@@ -334,9 +341,9 @@ class Converter:
                 "type": "code_interpreter",
             }
         elif tool_choice == "mcp":
-            return {
-                "type": "mcp",
-            }
+            # Note that this is still here for backwards compatibility,
+            # but migrating to MCPToolChoice is recommended.
+            return {"type": "mcp"}  # type: ignore [typeddict-item]
         else:
             return {
                 "type": "function",
@@ -363,7 +370,7 @@ class Converter:
     def convert_tools(
         cls,
         tools: list[Tool],
-        handoffs: list[Handoff[Any]],
+        handoffs: list[Handoff[Any, Any]],
     ) -> ConvertedTools:
         converted_tools: list[ToolParam] = []
         includes: list[ResponseIncludable] = []

agents/realtime/README.md

@@ -0,0 +1,3 @@
+# Realtime
+
+Realtime agents are in beta: expect some breaking changes over the next few weeks as we find issues and fix them.

agents/realtime/__init__.py

@@ -0,0 +1,177 @@
+from .agent import RealtimeAgent, RealtimeAgentHooks, RealtimeRunHooks
+from .config import (
+    RealtimeAudioFormat,
+    RealtimeClientMessage,
+    RealtimeGuardrailsSettings,
+    RealtimeInputAudioTranscriptionConfig,
+    RealtimeModelName,
+    RealtimeModelTracingConfig,
+    RealtimeRunConfig,
+    RealtimeSessionModelSettings,
+    RealtimeTurnDetectionConfig,
+    RealtimeUserInput,
+    RealtimeUserInputMessage,
+    RealtimeUserInputText,
+)
+from .events import (
+    RealtimeAgentEndEvent,
+    RealtimeAgentStartEvent,
+    RealtimeAudio,
+    RealtimeAudioEnd,
+    RealtimeAudioInterrupted,
+    RealtimeError,
+    RealtimeEventInfo,
+    RealtimeGuardrailTripped,
+    RealtimeHandoffEvent,
+    RealtimeHistoryAdded,
+    RealtimeHistoryUpdated,
+    RealtimeRawModelEvent,
+    RealtimeSessionEvent,
+    RealtimeToolEnd,
+    RealtimeToolStart,
+)
+from .handoffs import realtime_handoff
+from .items import (
+    AssistantMessageItem,
+    AssistantText,
+    InputAudio,
+    InputText,
+    RealtimeItem,
+    RealtimeMessageItem,
+    RealtimeResponse,
+    RealtimeToolCallItem,
+    SystemMessageItem,
+    UserMessageItem,
+)
+from .model import (
+    RealtimeModel,
+    RealtimeModelConfig,
+    RealtimeModelListener,
+)
+from .model_events import (
+    RealtimeConnectionStatus,
+    RealtimeModelAudioDoneEvent,
+    RealtimeModelAudioEvent,
+    RealtimeModelAudioInterruptedEvent,
+    RealtimeModelConnectionStatusEvent,
+    RealtimeModelErrorEvent,
+    RealtimeModelEvent,
+    RealtimeModelExceptionEvent,
+    RealtimeModelInputAudioTranscriptionCompletedEvent,
+    RealtimeModelItemDeletedEvent,
+    RealtimeModelItemUpdatedEvent,
+    RealtimeModelOtherEvent,
+    RealtimeModelToolCallEvent,
+    RealtimeModelTranscriptDeltaEvent,
+    RealtimeModelTurnEndedEvent,
+    RealtimeModelTurnStartedEvent,
+)
+from .model_inputs import (
+    RealtimeModelInputTextContent,
+    RealtimeModelRawClientMessage,
+    RealtimeModelSendAudio,
+    RealtimeModelSendEvent,
+    RealtimeModelSendInterrupt,
+    RealtimeModelSendRawMessage,
+    RealtimeModelSendSessionUpdate,
+    RealtimeModelSendToolOutput,
+    RealtimeModelSendUserInput,
+    RealtimeModelUserInput,
+    RealtimeModelUserInputMessage,
+)
+from .openai_realtime import (
+    DEFAULT_MODEL_SETTINGS,
+    OpenAIRealtimeWebSocketModel,
+    get_api_key,
+)
+from .runner import RealtimeRunner
+from .session import RealtimeSession
+
+__all__ = [
+    # Agent
+    "RealtimeAgent",
+    "RealtimeAgentHooks",
+    "RealtimeRunHooks",
+    "RealtimeRunner",
+    # Handoffs
+    "realtime_handoff",
+    # Config
+    "RealtimeAudioFormat",
+    "RealtimeClientMessage",
+    "RealtimeGuardrailsSettings",
+    "RealtimeInputAudioTranscriptionConfig",
+    "RealtimeModelName",
+    "RealtimeModelTracingConfig",
+    "RealtimeRunConfig",
+    "RealtimeSessionModelSettings",
+    "RealtimeTurnDetectionConfig",
+    "RealtimeUserInput",
+    "RealtimeUserInputMessage",
+    "RealtimeUserInputText",
+    # Events
+    "RealtimeAgentEndEvent",
+    "RealtimeAgentStartEvent",
+    "RealtimeAudio",
+    "RealtimeAudioEnd",
+    "RealtimeAudioInterrupted",
+    "RealtimeError",
+    "RealtimeEventInfo",
+    "RealtimeGuardrailTripped",
+    "RealtimeHandoffEvent",
+    "RealtimeHistoryAdded",
+    "RealtimeHistoryUpdated",
+    "RealtimeRawModelEvent",
+    "RealtimeSessionEvent",
+    "RealtimeToolEnd",
+    "RealtimeToolStart",
+    # Items
+    "AssistantMessageItem",
+    "AssistantText",
+    "InputAudio",
+    "InputText",
+    "RealtimeItem",
+    "RealtimeMessageItem",
+    "RealtimeResponse",
+    "RealtimeToolCallItem",
+    "SystemMessageItem",
+    "UserMessageItem",
+    # Model
+    "RealtimeModel",
+    "RealtimeModelConfig",
+    "RealtimeModelListener",
+    # Model Events
+    "RealtimeConnectionStatus",
+    "RealtimeModelAudioDoneEvent",
+    "RealtimeModelAudioEvent",
+    "RealtimeModelAudioInterruptedEvent",
+    "RealtimeModelConnectionStatusEvent",
+    "RealtimeModelErrorEvent",
+    "RealtimeModelEvent",
+    "RealtimeModelExceptionEvent",
+    "RealtimeModelInputAudioTranscriptionCompletedEvent",
+    "RealtimeModelItemDeletedEvent",
+    "RealtimeModelItemUpdatedEvent",
+    "RealtimeModelOtherEvent",
+    "RealtimeModelToolCallEvent",
+    "RealtimeModelTranscriptDeltaEvent",
+    "RealtimeModelTurnEndedEvent",
+    "RealtimeModelTurnStartedEvent",
+    # Model Inputs
+    "RealtimeModelInputTextContent",
+    "RealtimeModelRawClientMessage",
+    "RealtimeModelSendAudio",
+    "RealtimeModelSendEvent",
+    "RealtimeModelSendInterrupt",
+    "RealtimeModelSendRawMessage",
+    "RealtimeModelSendSessionUpdate",
+    "RealtimeModelSendToolOutput",
+    "RealtimeModelSendUserInput",
+    "RealtimeModelUserInput",
+    "RealtimeModelUserInputMessage",
+    # OpenAI Realtime
+    "DEFAULT_MODEL_SETTINGS",
+    "OpenAIRealtimeWebSocketModel",
+    "get_api_key",
+    # Session
+    "RealtimeSession",
+]

agents/realtime/agent.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import dataclasses
+import inspect
+from collections.abc import Awaitable
+from dataclasses import dataclass, field
+from typing import Any, Callable, Generic, cast
+
+from ..agent import AgentBase
+from ..handoffs import Handoff
+from ..lifecycle import AgentHooksBase, RunHooksBase
+from ..logger import logger
+from ..run_context import RunContextWrapper, TContext
+from ..util._types import MaybeAwaitable
+
+RealtimeAgentHooks = AgentHooksBase[TContext, "RealtimeAgent[TContext]"]
+"""Agent hooks for `RealtimeAgent`s."""
+
+RealtimeRunHooks = RunHooksBase[TContext, "RealtimeAgent[TContext]"]
+"""Run hooks for `RealtimeAgent`s."""
+
+
+@dataclass
+class RealtimeAgent(AgentBase, Generic[TContext]):
+    """A specialized agent instance that is meant to be used within a `RealtimeSession` to build
+    voice agents. Due to the nature of this agent, some configuration options are not supported
+    that are supported by regular `Agent` instances. For example:
+    - `model` choice is not supported, as all RealtimeAgents will be handled by the same model
+      within a `RealtimeSession`.
+    - `modelSettings` is not supported, as all RealtimeAgents will be handled by the same model
+      within a `RealtimeSession`.
+    - `outputType` is not supported, as RealtimeAgents do not support structured outputs.
+    - `toolUseBehavior` is not supported, as all RealtimeAgents will be handled by the same model
+      within a `RealtimeSession`.
+    - `voice` can be configured on an `Agent` level; however, it cannot be changed after the first
+      agent within a `RealtimeSession` has spoken.
+
+    See `AgentBase` for base parameters that are shared with `Agent`s.
+    """
+
+    instructions: (
+        str
+        | Callable[
+            [RunContextWrapper[TContext], RealtimeAgent[TContext]],
+            MaybeAwaitable[str],
+        ]
+        | None
+    ) = None
+    """The instructions for the agent. Will be used as the "system prompt" when this agent is
+    invoked. Describes what the agent should do, and how it responds.
+
+    Can either be a string, or a function that dynamically generates instructions for the agent. If
+    you provide a function, it will be called with the context and the agent instance. It must
+    return a string.
+    """
+
+    handoffs: list[RealtimeAgent[Any] | Handoff[TContext, RealtimeAgent[Any]]] = field(
+        default_factory=list
+    )
+    """Handoffs are sub-agents that the agent can delegate to. You can provide a list of handoffs,
+    and the agent can choose to delegate to them if relevant. Allows for separation of concerns and
+    modularity.
+    """
+
+    hooks: RealtimeAgentHooks | None = None
+    """A class that receives callbacks on various lifecycle events for this agent.
+    """
+
+    def clone(self, **kwargs: Any) -> RealtimeAgent[TContext]:
+        """Make a copy of the agent, with the given arguments changed. For example, you could do:
+        ```
+        new_agent = agent.clone(instructions="New instructions")
+        ```
+        """
+        return dataclasses.replace(self, **kwargs)
+
+    async def get_system_prompt(self, run_context: RunContextWrapper[TContext]) -> str | None:
+        """Get the system prompt for the agent."""
+        if isinstance(self.instructions, str):
+            return self.instructions
+        elif callable(self.instructions):
+            if inspect.iscoroutinefunction(self.instructions):
+                return await cast(Awaitable[str], self.instructions(run_context, self))
+            else:
+                return cast(str, self.instructions(run_context, self))
+        elif self.instructions is not None:
+            logger.error(f"Instructions must be a string or a function, got {self.instructions}")
+
+        return None