openai-agents 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

agents/__init__.py

@@ -5,7 +5,13 @@ from typing import Literal
 from openai import AsyncOpenAI
 
 from . import _config
-from .agent import Agent, AgentBase, ToolsToFinalOutputFunction, ToolsToFinalOutputResult
+from .agent import (
+    Agent,
+    AgentBase,
+    StopAtTools,
+    ToolsToFinalOutputFunction,
+    ToolsToFinalOutputResult,
+)
 from .agent_output import AgentOutputSchema, AgentOutputSchemaBase
 from .computer import AsyncComputer, Button, Computer, Environment
 from .exceptions import (
@@ -43,6 +49,7 @@ from .lifecycle import AgentHooks, RunHooks
 from .memory import Session, SQLiteSession
 from .model_settings import ModelSettings
 from .models.interface import Model, ModelProvider, ModelTracing
+from .models.multi_provider import MultiProvider
 from .models.openai_chatcompletions import OpenAIChatCompletionsModel
 from .models.openai_provider import OpenAIProvider
 from .models.openai_responses import OpenAIResponsesModel
@@ -162,6 +169,7 @@ def enable_verbose_stdout_logging():
 __all__ = [
     "Agent",
     "AgentBase",
+    "StopAtTools",
     "ToolsToFinalOutputFunction",
     "ToolsToFinalOutputResult",
     "Runner",
@@ -171,6 +179,7 @@ __all__ = [
     "ModelTracing",
     "ModelSettings",
     "OpenAIChatCompletionsModel",
+    "MultiProvider",
     "OpenAIProvider",
     "OpenAIResponsesModel",
     "AgentOutputSchema",

agents/_run_impl.py

@@ -774,6 +774,7 @@ class RunImpl:
                     else original_input,
                     pre_handoff_items=tuple(pre_step_items),
                     new_items=tuple(new_step_items),
+                    run_context=context_wrapper,
                 )
                 if not callable(input_filter):
                     _error_tracing.attach_error_to_span(
@@ -785,6 +786,8 @@ class RunImpl:
                     )
                     raise UserError(f"Invalid input filter: {input_filter}")
                 filtered = input_filter(handoff_input_data)
+                if inspect.isawaitable(filtered):
+                    filtered = await filtered
                 if not isinstance(filtered, HandoffInputData):
                     _error_tracing.attach_error_to_span(
                         span_handoff,
@@ -911,12 +914,12 @@ class RunImpl:
             return result
 
     @classmethod
-    def stream_step_result_to_queue(
+    def stream_step_items_to_queue(
         cls,
-        step_result: SingleStepResult,
+        new_step_items: list[RunItem],
         queue: asyncio.Queue[StreamEvent | QueueCompleteSentinel],
     ):
-        for item in step_result.new_step_items:
+        for item in new_step_items:
             if isinstance(item, MessageOutputItem):
                 event = RunItemStreamEvent(item=item, name="message_output_created")
             elif isinstance(item, HandoffCallItem):
@@ -941,6 +944,14 @@ class RunImpl:
             if event:
                 queue.put_nowait(event)
 
+    @classmethod
+    def stream_step_result_to_queue(
+        cls,
+        step_result: SingleStepResult,
+        queue: asyncio.Queue[StreamEvent | QueueCompleteSentinel],
+    ):
+        cls.stream_step_items_to_queue(step_result.new_step_items, queue)
+
     @classmethod
     async def _check_for_final_output_from_tools(
         cls,

agents/agent.py

@@ -101,7 +101,7 @@ class AgentBase(Generic[TContext]):
             self.mcp_servers, convert_schemas_to_strict, run_context, self
         )
 
-    async def get_all_tools(self, run_context: RunContextWrapper[Any]) -> list[Tool]:
+    async def get_all_tools(self, run_context: RunContextWrapper[TContext]) -> list[Tool]:
         """All agent tools, including MCP tools and function tools."""
         mcp_tools = await self.get_mcp_tools(run_context)
 
@@ -201,20 +201,22 @@ class Agent(AgentBase, Generic[TContext]):
     tool_use_behavior: (
         Literal["run_llm_again", "stop_on_first_tool"] | StopAtTools | ToolsToFinalOutputFunction
     ) = "run_llm_again"
-    """This lets you configure how tool use is handled.
+    """
+    This lets you configure how tool use is handled.
     - "run_llm_again": The default behavior. Tools are run, and then the LLM receives the results
         and gets to respond.
     - "stop_on_first_tool": The output of the first tool call is used as the final output. This
         means that the LLM does not process the result of the tool call.
-    - A list of tool names: The agent will stop running if any of the tools in the list are called.
-        The final output will be the output of the first matching tool call. The LLM does not
-        process the result of the tool call.
+    - A StopAtTools object: The agent will stop running if any of the tools listed in
+        `stop_at_tool_names` is called.
+        The final output will be the output of the first matching tool call.
+        The LLM does not process the result of the tool call.
     - A function: If you pass a function, it will be called with the run context and the list of
       tool results. It must return a `ToolsToFinalOutputResult`, which determines whether the tool
       calls result in a final output.
 
       NOTE: This configuration is specific to FunctionTools. Hosted tools, such as file search,
-      web search, etc are always processed by the LLM.
+      web search, etc. are always processed by the LLM.
     """
 
     reset_tool_choice: bool = True
@@ -222,10 +224,17 @@ class Agent(AgentBase, Generic[TContext]):
     to True. This ensures that the agent doesn't enter an infinite loop of tool usage."""
 
     def clone(self, **kwargs: Any) -> Agent[TContext]:
-        """Make a copy of the agent, with the given arguments changed. For example, you could do:
-        ```
-        new_agent = agent.clone(instructions="New instructions")
-        ```
+        """Make a copy of the agent, with the given arguments changed.
+        Notes:
+            - Uses `dataclasses.replace`, which performs a **shallow copy**.
+            - Mutable attributes like `tools` and `handoffs` are shallow-copied:
+              new list objects are created only if overridden, but their contents
+              (tool functions and handoff objects) are shared with the original.
+            - To modify these independently, pass new lists when calling `clone()`.
+        Example:
+            ```python
+            new_agent = agent.clone(instructions="New instructions")
+            ```
         """
         return dataclasses.replace(self, **kwargs)
 
@@ -289,30 +298,3 @@ class Agent(AgentBase, Generic[TContext]):
     ) -> ResponsePromptParam | None:
         """Get the prompt for the agent."""
         return await PromptUtil.to_model_input(self.prompt, run_context, self)
-
-    async def get_mcp_tools(self, run_context: RunContextWrapper[TContext]) -> list[Tool]:
-        """Fetches the available tools from the MCP servers."""
-        convert_schemas_to_strict = self.mcp_config.get("convert_schemas_to_strict", False)
-        return await MCPUtil.get_all_function_tools(
-            self.mcp_servers, convert_schemas_to_strict, run_context, self
-        )
-
-    async def get_all_tools(self, run_context: RunContextWrapper[Any]) -> list[Tool]:
-        """All agent tools, including MCP tools and function tools."""
-        mcp_tools = await self.get_mcp_tools(run_context)
-
-        async def _check_tool_enabled(tool: Tool) -> bool:
-            if not isinstance(tool, FunctionTool):
-                return True
-
-            attr = tool.is_enabled
-            if isinstance(attr, bool):
-                return attr
-            res = attr(run_context, self)
-            if inspect.isawaitable(res):
-                return bool(await res)
-            return bool(res)
-
-        results = await asyncio.gather(*(_check_tool_enabled(t) for t in self.tools))
-        enabled: list[Tool] = [t for t, ok in zip(self.tools, results) if ok]
-        return [*mcp_tools, *enabled]

agents/agent_output.py

@@ -116,7 +116,7 @@ class AgentOutputSchema(AgentOutputSchemaBase):
                 raise UserError(
                     "Strict JSON schema is enabled, but the output type is not valid. "
                     "Either make the output type strict, "
-                    "or wrap your type with AgentOutputSchema(your_type, strict_json_schema=False)"
+                    "or wrap your type with AgentOutputSchema(YourType, strict_json_schema=False)"
                 ) from e
 
     def is_plain_text(self) -> bool:

agents/extensions/handoff_filters.py

@@ -29,6 +29,7 @@ def remove_all_tools(handoff_input_data: HandoffInputData) -> HandoffInputData:
         input_history=filtered_history,
         pre_handoff_items=filtered_pre_handoff_items,
         new_items=filtered_new_items,
+        run_context=handoff_input_data.run_context,
     )
 
 

agents/extensions/models/litellm_model.py

@@ -45,6 +45,14 @@ from ...tracing.spans import Span
 from ...usage import Usage
 
 
+class InternalChatCompletionMessage(ChatCompletionMessage):
+    """
+    An internal subclass to carry reasoning_content without modifying the original model.
+    """
+
+    reasoning_content: str
+
+
 class LitellmModel(Model):
     """This class enables using any model via LiteLLM. LiteLLM allows you to acess OpenAPI,
     Anthropic, Gemini, Mistral, and many other models.
@@ -364,13 +372,18 @@ class LitellmConverter:
             provider_specific_fields.get("refusal", None) if provider_specific_fields else None
         )
 
-        return ChatCompletionMessage(
+        reasoning_content = ""
+        if hasattr(message, "reasoning_content") and message.reasoning_content:
+            reasoning_content = message.reasoning_content
+
+        return InternalChatCompletionMessage(
             content=message.content,
             refusal=refusal,
             role="assistant",
             annotations=cls.convert_annotations_to_openai(message),
             audio=message.get("audio", None),  # litellm deletes audio if not present
             tool_calls=tool_calls,
+            reasoning_content=reasoning_content,
         )
 
     @classmethod

agents/extensions/visualization.py

@@ -71,6 +71,12 @@ def get_all_nodes(
             f"fillcolor=lightgreen, width=0.5, height=0.3];"
         )
 
+    for mcp_server in agent.mcp_servers:
+        parts.append(
+            f'"{mcp_server.name}" [label="{mcp_server.name}", shape=box, style=filled, '
+            f"fillcolor=lightgrey, width=1, height=0.5];"
+        )
+
     for handoff in agent.handoffs:
         if isinstance(handoff, Handoff):
             parts.append(
@@ -119,6 +125,11 @@ def get_all_edges(
         "{agent.name}" -> "{tool.name}" [style=dotted, penwidth=1.5];
         "{tool.name}" -> "{agent.name}" [style=dotted, penwidth=1.5];""")
 
+    for mcp_server in agent.mcp_servers:
+        parts.append(f"""
+        "{agent.name}" -> "{mcp_server.name}" [style=dashed, penwidth=1.5];
+        "{mcp_server.name}" -> "{agent.name}" [style=dashed, penwidth=1.5];""")
+
     for handoff in agent.handoffs:
         if isinstance(handoff, Handoff):
             parts.append(f"""

agents/function_schema.py

@@ -76,7 +76,7 @@ class FuncSchema:
 
 @dataclass
 class FuncDocumentation:
-    """Contains metadata about a python function, extracted from its docstring."""
+    """Contains metadata about a Python function, extracted from its docstring."""
 
     name: str
     """The name of the function, via `__name__`."""
@@ -194,7 +194,7 @@ def function_schema(
     strict_json_schema: bool = True,
 ) -> FuncSchema:
     """
-    Given a python function, extracts a `FuncSchema` from it, capturing the name, description,
+    Given a Python function, extracts a `FuncSchema` from it, capturing the name, description,
     parameter descriptions, and other metadata.
 
     Args:
@@ -208,7 +208,7 @@ def function_schema(
             descriptions.
         strict_json_schema: Whether the JSON schema is in strict mode. If True, we'll ensure that
             the schema adheres to the "strict" standard the OpenAI API expects. We **strongly**
-            recommend setting this to True, as it increases the likelihood of the LLM providing
+            recommend setting this to True, as it increases the likelihood of the LLM producing
             correct JSON input.
 
     Returns:

agents/guardrail.py

@@ -78,8 +78,9 @@ class InputGuardrail(Generic[TContext]):
     You can use the `@input_guardrail()` decorator to turn a function into an `InputGuardrail`, or
     create an `InputGuardrail` manually.
 
-    Guardrails return a `GuardrailResult`. If `result.tripwire_triggered` is `True`, the agent
-    execution will immediately stop and a `InputGuardrailTripwireTriggered` exception will be raised
+    Guardrails return a `GuardrailResult`. If `result.tripwire_triggered` is `True`,
+    the agent's execution will immediately stop, and
+    an `InputGuardrailTripwireTriggered` exception will be raised
     """
 
     guardrail_function: Callable[
@@ -132,7 +133,7 @@ class OutputGuardrail(Generic[TContext]):
     You can use the `@output_guardrail()` decorator to turn a function into an `OutputGuardrail`,
     or create an `OutputGuardrail` manually.
 
-    Guardrails return a `GuardrailResult`. If `result.tripwire_triggered` is `True`, a
+    Guardrails return a `GuardrailResult`. If `result.tripwire_triggered` is `True`, an
     `OutputGuardrailTripwireTriggered` exception will be raised.
     """
 
@@ -316,7 +317,7 @@ def output_guardrail(
     ) -> OutputGuardrail[TContext_co]:
         return OutputGuardrail(
             guardrail_function=f,
-            # Guardrail name defaults to function name when not specified (None).
+            # Guardrail name defaults to function's name when not specified (None).
             name=name if name else f.__name__,
         )
 

agents/handoffs.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import inspect
 import json
 from collections.abc import Awaitable
-from dataclasses import dataclass
+from dataclasses import dataclass, replace as dataclasses_replace
 from typing import TYPE_CHECKING, Any, Callable, Generic, cast, overload
 
 from pydantic import TypeAdapter
@@ -49,8 +49,24 @@ class HandoffInputData:
     handoff and the tool output message representing the response from the handoff output.
     """
 
+    run_context: RunContextWrapper[Any] | None = None
+    """
+    The run context at the time the handoff was invoked.
+    Note that, since this property was added later on, it's optional for backwards compatibility.
+    """
+
+    def clone(self, **kwargs: Any) -> HandoffInputData:
+        """
+        Make a copy of the handoff input data, with the given arguments changed. For example, you
+        could do:
+        ```
+        new_handoff_input_data = handoff_input_data.clone(new_items=())
+        ```
+        """
+        return dataclasses_replace(self, **kwargs)
 
-HandoffInputFilter: TypeAlias = Callable[[HandoffInputData], HandoffInputData]
+
+HandoffInputFilter: TypeAlias = Callable[[HandoffInputData], MaybeAwaitable[HandoffInputData]]
 """A function that filters the input data passed to the next agent."""
 
 
@@ -103,9 +119,9 @@ class Handoff(Generic[TContext, TAgent]):
     True, as it increases the likelihood of correct JSON input.
     """
 
-    is_enabled: bool | Callable[[RunContextWrapper[Any], AgentBase[Any]], MaybeAwaitable[bool]] = (
-        True
-    )
+    is_enabled: bool | Callable[
+        [RunContextWrapper[Any], AgentBase[Any]], MaybeAwaitable[bool]
+    ] = True
     """Whether the handoff is enabled. Either a bool or a Callable that takes the run context and
     agent and returns whether the handoff is enabled. You can use this to dynamically enable/disable
     a handoff based on your context/state."""
@@ -248,7 +264,7 @@ def handoff(
     async def _is_enabled(ctx: RunContextWrapper[Any], agent_base: AgentBase[Any]) -> bool:
         from .agent import Agent
 
-        assert callable(is_enabled), "is_enabled must be non-null here"
+        assert callable(is_enabled), "is_enabled must be callable here"
         assert isinstance(agent_base, Agent), "Can't handoff to a non-Agent"
         result = is_enabled(ctx, agent_base)
 

agents/items.py

@@ -66,7 +66,7 @@ class RunItemBase(Generic[T], abc.ABC):
     """The agent whose run caused this item to be generated."""
 
     raw_item: T
-    """The raw Responses item from the run. This will always be a either an output item (i.e.
+    """The raw Responses item from the run. This will always be either an output item (i.e.
     `openai.types.responses.ResponseOutputItem` or an input item
     (i.e. `openai.types.responses.ResponseInputItemParam`).
     """
@@ -243,6 +243,8 @@ class ItemHelpers:
         if not isinstance(message, ResponseOutputMessage):
             return ""
 
+        if not message.content:
+            return ""
         last_content = message.content[-1]
         if isinstance(last_content, ResponseOutputText):
             return last_content.text
@@ -255,6 +257,8 @@ class ItemHelpers:
     def extract_last_text(cls, message: TResponseOutputItem) -> str | None:
         """Extracts the last text content from a message, if any. Ignores refusals."""
         if isinstance(message, ResponseOutputMessage):
+            if not message.content:
+                return None
             last_content = message.content[-1]
             if isinstance(last_content, ResponseOutputText):
                 return last_content.text

agents/mcp/util.py

@@ -194,23 +194,21 @@ class MCPUtil:
         else:
             logger.debug(f"MCP tool {tool.name} returned {result}")
 
-        # The MCP tool result is a list of content items, whereas OpenAI tool outputs are a single
-        # string. We'll try to convert.
-        if len(result.content) == 1:
-            tool_output = result.content[0].model_dump_json()
-            # Append structured content if it exists and we're using it.
-            if server.use_structured_content and result.structuredContent:
-                tool_output = f"{tool_output}\n{json.dumps(result.structuredContent)}"
-        elif len(result.content) > 1:
-            tool_results = [item.model_dump(mode="json") for item in result.content]
-            if server.use_structured_content and result.structuredContent:
-                tool_results.append(result.structuredContent)
-            tool_output = json.dumps(tool_results)
-        elif server.use_structured_content and result.structuredContent:
+        # If structured content is requested and available, use it exclusively
+        if server.use_structured_content and result.structuredContent:
             tool_output = json.dumps(result.structuredContent)
         else:
-            # Empty content is a valid result (e.g., "no results found")
-            tool_output = "[]"
+            # Fall back to regular text content processing
+            # The MCP tool result is a list of content items, whereas OpenAI tool
+            # outputs are a single string. We'll try to convert.
+            if len(result.content) == 1:
+                tool_output = result.content[0].model_dump_json()
+            elif len(result.content) > 1:
+                tool_results = [item.model_dump(mode="json") for item in result.content]
+                tool_output = json.dumps(tool_results)
+            else:
+                # Empty content is a valid result (e.g., "no results found")
+                tool_output = "[]"
 
         current_span = get_current_span()
         if current_span:

agents/models/chatcmpl_converter.py

@@ -36,6 +36,7 @@ from openai.types.responses import (
     ResponseOutputRefusal,
     ResponseOutputText,
     ResponseReasoningItem,
+    ResponseReasoningItemParam,
 )
 from openai.types.responses.response_input_param import FunctionCallOutput, ItemReference, Message
 from openai.types.responses.response_reasoning_item import Summary
@@ -210,6 +211,12 @@ class Converter:
             return cast(ResponseOutputMessageParam, item)
         return None
 
+    @classmethod
+    def maybe_reasoning_message(cls, item: Any) -> ResponseReasoningItemParam | None:
+        if isinstance(item, dict) and item.get("type") == "reasoning":
+            return cast(ResponseReasoningItemParam, item)
+        return None
+
     @classmethod
     def extract_text_content(
         cls, content: str | Iterable[ResponseInputContentParam]
@@ -459,7 +466,11 @@ class Converter:
                     f"Encountered an item_reference, which is not supported: {item_ref}"
                 )
 
-            # 7) If we haven't recognized it => fail or ignore
+            # 7) reasoning message => not handled
+            elif cls.maybe_reasoning_message(item):
+                pass
+
+            # 8) If we haven't recognized it => fail or ignore
             else:
                 raise UserError(f"Unhandled item type or structure: {item}")
 

agents/models/chatcmpl_stream_handler.py

@@ -198,6 +198,7 @@ class ChatCmplStreamHandler:
                     is not None,  # fixed 0 -> 0 or 1
                     type="response.output_text.delta",
                     sequence_number=sequence_number.get_and_increment(),
+                    logprobs=[],
                 )
                 # Accumulate the text into the response part
                 state.text_content_index_and_output[1].text += delta.content
@@ -288,10 +289,11 @@ class ChatCmplStreamHandler:
                     function_call = state.function_calls[tc_delta.index]
 
                     # Start streaming as soon as we have function name and call_id
-                    if (not state.function_call_streaming[tc_delta.index] and
-                        function_call.name and
-                        function_call.call_id):
-
+                    if (
+                        not state.function_call_streaming[tc_delta.index]
+                        and function_call.name
+                        and function_call.call_id
+                    ):
                         # Calculate the output index for this function call
                         function_call_starting_index = 0
                         if state.reasoning_content_index_and_output:
@@ -308,9 +310,9 @@ class ChatCmplStreamHandler:
 
                         # Mark this function call as streaming and store its output index
                         state.function_call_streaming[tc_delta.index] = True
-                        state.function_call_output_idx[
-                            tc_delta.index
-                        ] = function_call_starting_index
+                        state.function_call_output_idx[tc_delta.index] = (
+                            function_call_starting_index
+                        )
 
                         # Send initial function call added event
                         yield ResponseOutputItemAddedEvent(
@@ -327,10 +329,11 @@ class ChatCmplStreamHandler:
                         )
 
                     # Stream arguments if we've started streaming this function call
-                    if (state.function_call_streaming.get(tc_delta.index, False) and
-                        tc_function and
-                        tc_function.arguments):
-
+                    if (
+                        state.function_call_streaming.get(tc_delta.index, False)
+                        and tc_function
+                        and tc_function.arguments
+                    ):
                         output_index = state.function_call_output_idx[tc_delta.index]
                         yield ResponseFunctionCallArgumentsDeltaEvent(
                             delta=tc_function.arguments,

agents/models/openai_chatcompletions.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import json
 import time
 from collections.abc import AsyncIterator
-from typing import TYPE_CHECKING, Any, Literal, cast, overload
+from typing import TYPE_CHECKING, Any, Literal, overload
 
 from openai import NOT_GIVEN, AsyncOpenAI, AsyncStream
 from openai.types import ChatModel
@@ -28,6 +28,7 @@ from .chatcmpl_helpers import HEADERS, ChatCmplHelpers
 from .chatcmpl_stream_handler import ChatCmplStreamHandler
 from .fake_id import FAKE_RESPONSES_ID
 from .interface import Model, ModelTracing
+from .openai_responses import Converter as OpenAIResponsesConverter
 
 if TYPE_CHECKING:
     from ..model_settings import ModelSettings
@@ -296,15 +297,27 @@ class OpenAIChatCompletionsModel(Model):
         if isinstance(ret, ChatCompletion):
             return ret
 
+        responses_tool_choice = OpenAIResponsesConverter.convert_tool_choice(
+            model_settings.tool_choice
+        )
+        if responses_tool_choice is None or responses_tool_choice == NOT_GIVEN:
+            # For Responses API data compatibility with Chat Completions patterns,
+            # we need to set "none" if tool_choice is absent.
+            # Without this fix, you'll get the following error:
+            # pydantic_core._pydantic_core.ValidationError: 4 validation errors for Response
+            # tool_choice.literal['none','auto','required']
+            #   Input should be 'none', 'auto' or 'required'
+            #   [type=literal_error, input_value=NOT_GIVEN, input_type=NotGiven]
+            # see also: https://github.com/openai/openai-agents-python/issues/980
+            responses_tool_choice = "auto"
+
         response = Response(
             id=FAKE_RESPONSES_ID,
             created_at=time.time(),
             model=self.model,
             object="response",
             output=[],
-            tool_choice=cast(Literal["auto", "required", "none"], tool_choice)
-            if tool_choice != NOT_GIVEN
-            else "auto",
+            tool_choice=responses_tool_choice,  # type: ignore[arg-type]
             top_p=model_settings.top_p,
             temperature=model_settings.temperature,
             tools=[],

agents/realtime/__init__.py

@@ -47,6 +47,8 @@ from .model import (
     RealtimeModel,
     RealtimeModelConfig,
     RealtimeModelListener,
+    RealtimePlaybackState,
+    RealtimePlaybackTracker,
 )
 from .model_events import (
     RealtimeConnectionStatus,
@@ -139,6 +141,8 @@ __all__ = [
     "RealtimeModel",
     "RealtimeModelConfig",
     "RealtimeModelListener",
+    "RealtimePlaybackTracker",
+    "RealtimePlaybackState",
     # Model Events
     "RealtimeConnectionStatus",
     "RealtimeModelAudioDoneEvent",

agents/realtime/_default_tracker.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from ._util import calculate_audio_length_ms
+from .config import RealtimeAudioFormat
+
+
+@dataclass
+class ModelAudioState:
+    initial_received_time: datetime
+    audio_length_ms: float
+
+
+class ModelAudioTracker:
+    def __init__(self) -> None:
+        # (item_id, item_content_index) -> ModelAudioState
+        self._states: dict[tuple[str, int], ModelAudioState] = {}
+        self._last_audio_item: tuple[str, int] | None = None
+
+    def set_audio_format(self, format: RealtimeAudioFormat) -> None:
+        """Called when the model wants to set the audio format."""
+        self._format = format
+
+    def on_audio_delta(self, item_id: str, item_content_index: int, audio_bytes: bytes) -> None:
+        """Called when an audio delta is received from the model."""
+        ms = calculate_audio_length_ms(self._format, audio_bytes)
+        new_key = (item_id, item_content_index)
+
+        self._last_audio_item = new_key
+        if new_key not in self._states:
+            self._states[new_key] = ModelAudioState(datetime.now(), ms)
+        else:
+            self._states[new_key].audio_length_ms += ms
+
+    def on_interrupted(self) -> None:
+        """Called when the audio playback has been interrupted."""
+        self._last_audio_item = None
+
+    def get_state(self, item_id: str, item_content_index: int) -> ModelAudioState | None:
+        """Called when the model wants to get the current playback state."""
+        return self._states.get((item_id, item_content_index))
+
+    def get_last_audio_item(self) -> tuple[str, int] | None:
+        """Called when the model wants to get the last audio item ID and content index."""
+        return self._last_audio_item

agents/realtime/_util.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .config import RealtimeAudioFormat
+
+
+def calculate_audio_length_ms(format: RealtimeAudioFormat | None, audio_bytes: bytes) -> float:
+    if format and format.startswith("g711"):
+        return (len(audio_bytes) / 8000) * 1000
+    return (len(audio_bytes) / 24 / 2) * 1000

agents/realtime/config.py

@@ -94,6 +94,9 @@ class RealtimeSessionModelSettings(TypedDict):
     voice: NotRequired[str]
     """The voice to use for audio output."""
 
+    speed: NotRequired[float]
+    """The speed of the model's responses."""
+
     input_audio_format: NotRequired[RealtimeAudioFormat]
     """The format for input audio streams."""