openai-agents 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

agents/realtime/openai_realtime.py

@@ -57,11 +57,13 @@ from typing_extensions import assert_never
 from websockets.asyncio.client import ClientConnection
 
 from agents.handoffs import Handoff
+from agents.realtime._default_tracker import ModelAudioTracker
 from agents.tool import FunctionTool, Tool
 from agents.util._types import MaybeAwaitable
 
 from ..exceptions import UserError
 from ..logger import logger
+from ..version import __version__
 from .config import (
     RealtimeModelTracingConfig,
     RealtimeSessionModelSettings,
@@ -71,6 +73,8 @@ from .model import (
     RealtimeModel,
     RealtimeModelConfig,
     RealtimeModelListener,
+    RealtimePlaybackState,
+    RealtimePlaybackTracker,
 )
 from .model_events import (
     RealtimeModelAudioDoneEvent,
@@ -82,6 +86,7 @@ from .model_events import (
     RealtimeModelInputAudioTranscriptionCompletedEvent,
     RealtimeModelItemDeletedEvent,
     RealtimeModelItemUpdatedEvent,
+    RealtimeModelRawServerEvent,
     RealtimeModelToolCallEvent,
     RealtimeModelTranscriptDeltaEvent,
     RealtimeModelTurnEndedEvent,
@@ -97,6 +102,8 @@ from .model_inputs import (
     RealtimeModelSendUserInput,
 )
 
+_USER_AGENT = f"Agents/Python {__version__}"
+
 DEFAULT_MODEL_SETTINGS: RealtimeSessionModelSettings = {
     "voice": "ash",
     "modalities": ["text", "audio"],
@@ -130,11 +137,11 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
         self._websocket_task: asyncio.Task[None] | None = None
         self._listeners: list[RealtimeModelListener] = []
         self._current_item_id: str | None = None
-        self._audio_start_time: datetime | None = None
-        self._audio_length_ms: float = 0.0
+        self._audio_state_tracker: ModelAudioTracker = ModelAudioTracker()
         self._ongoing_response: bool = False
-        self._current_audio_content_index: int | None = None
         self._tracing_config: RealtimeModelTracingConfig | Literal["auto"] | None = None
+        self._playback_tracker: RealtimePlaybackTracker | None = None
+        self._created_session: OpenAISessionObject | None = None
 
     async def connect(self, options: RealtimeModelConfig) -> None:
         """Establish a connection to the model and keep it alive."""
@@ -143,6 +150,8 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
 
         model_settings: RealtimeSessionModelSettings = options.get("initial_model_settings", {})
 
+        self._playback_tracker = options.get("playback_tracker", RealtimePlaybackTracker())
+
         self.model = model_settings.get("model_name", self.model)
         api_key = await get_api_key(options.get("api_key"))
 
@@ -160,7 +169,9 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
             "Authorization": f"Bearer {api_key}",
             "OpenAI-Beta": "realtime=v1",
         }
-        self._websocket = await websockets.connect(url, additional_headers=headers)
+        self._websocket = await websockets.connect(
+            url, user_agent_header=_USER_AGENT, additional_headers=headers
+        )
         self._websocket_task = asyncio.create_task(self._listen_for_messages())
         await self._update_session_config(model_settings)
 
@@ -289,26 +300,69 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
         if event.start_response:
             await self._send_raw_message(OpenAIResponseCreateEvent(type="response.create"))
 
+    def _get_playback_state(self) -> RealtimePlaybackState:
+        if self._playback_tracker:
+            return self._playback_tracker.get_state()
+
+        if last_audio_item_id := self._audio_state_tracker.get_last_audio_item():
+            item_id, item_content_index = last_audio_item_id
+            audio_state = self._audio_state_tracker.get_state(item_id, item_content_index)
+            if audio_state:
+                elapsed_ms = (
+                    datetime.now() - audio_state.initial_received_time
+                ).total_seconds() * 1000
+                return {
+                    "current_item_id": item_id,
+                    "current_item_content_index": item_content_index,
+                    "elapsed_ms": elapsed_ms,
+                }
+
+        return {
+            "current_item_id": None,
+            "current_item_content_index": None,
+            "elapsed_ms": None,
+        }
+
     async def _send_interrupt(self, event: RealtimeModelSendInterrupt) -> None:
-        if not self._current_item_id or not self._audio_start_time:
+        playback_state = self._get_playback_state()
+        current_item_id = playback_state.get("current_item_id")
+        current_item_content_index = playback_state.get("current_item_content_index")
+        elapsed_ms = playback_state.get("elapsed_ms")
+        if current_item_id is None or elapsed_ms is None:
+            logger.info(
+                "Skipping interrupt. "
+                f"Item id: {current_item_id}, "
+                f"elapsed ms: {elapsed_ms}, "
+                f"content index: {current_item_content_index}"
+            )
             return
 
-        await self._cancel_response()
-
-        elapsed_time_ms = (datetime.now() - self._audio_start_time).total_seconds() * 1000
-        if elapsed_time_ms > 0 and elapsed_time_ms < self._audio_length_ms:
-            await self._emit_event(RealtimeModelAudioInterruptedEvent())
+        current_item_content_index = current_item_content_index or 0
+        if elapsed_ms > 0:
+            await self._emit_event(
+                RealtimeModelAudioInterruptedEvent(
+                    item_id=current_item_id,
+                    content_index=current_item_content_index,
+                )
+            )
             converted = _ConversionHelper.convert_interrupt(
-                self._current_item_id,
-                self._current_audio_content_index or 0,
-                int(elapsed_time_ms),
+                current_item_id,
+                current_item_content_index,
+                int(elapsed_ms),
             )
             await self._send_raw_message(converted)
 
-        self._current_item_id = None
-        self._audio_start_time = None
-        self._audio_length_ms = 0.0
-        self._current_audio_content_index = None
+        automatic_response_cancellation_enabled = (
+            self._created_session
+            and self._created_session.turn_detection
+            and self._created_session.turn_detection.interrupt_response
+        )
+        if not automatic_response_cancellation_enabled:
+            await self._cancel_response()
+
+        self._audio_state_tracker.on_interrupted()
+        if self._playback_tracker:
+            self._playback_tracker.on_interrupted()
 
     async def _send_session_update(self, event: RealtimeModelSendSessionUpdate) -> None:
         """Send a session update to the model."""
@@ -316,23 +370,21 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
 
     async def _handle_audio_delta(self, parsed: ResponseAudioDeltaEvent) -> None:
         """Handle audio delta events and update audio tracking state."""
-        self._current_audio_content_index = parsed.content_index
         self._current_item_id = parsed.item_id
-        if self._audio_start_time is None:
-            self._audio_start_time = datetime.now()
-            self._audio_length_ms = 0.0
 
         audio_bytes = base64.b64decode(parsed.delta)
-        # Calculate audio length in ms using 24KHz pcm16le
-        self._audio_length_ms += self._calculate_audio_length_ms(audio_bytes)
+
+        self._audio_state_tracker.on_audio_delta(parsed.item_id, parsed.content_index, audio_bytes)
+
         await self._emit_event(
-            RealtimeModelAudioEvent(data=audio_bytes, response_id=parsed.response_id)
+            RealtimeModelAudioEvent(
+                data=audio_bytes,
+                response_id=parsed.response_id,
+                item_id=parsed.item_id,
+                content_index=parsed.content_index,
+            )
         )
 
-    def _calculate_audio_length_ms(self, audio_bytes: bytes) -> float:
-        """Calculate audio length in milliseconds for 24KHz PCM16LE format."""
-        return len(audio_bytes) / 24 / 2
-
     async def _handle_output_item(self, item: ConversationItem) -> None:
         """Handle response output item events (function calls and messages)."""
         if item.type == "function_call" and item.status == "completed":
@@ -396,6 +448,7 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
             self._ongoing_response = False
 
     async def _handle_ws_event(self, event: dict[str, Any]):
+        await self._emit_event(RealtimeModelRawServerEvent(data=event))
         try:
             if "previous_item_id" in event and event["previous_item_id"] is None:
                 event["previous_item_id"] = ""  # TODO (rm) remove
@@ -424,7 +477,12 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
         if parsed.type == "response.audio.delta":
             await self._handle_audio_delta(parsed)
         elif parsed.type == "response.audio.done":
-            await self._emit_event(RealtimeModelAudioDoneEvent())
+            await self._emit_event(
+                RealtimeModelAudioDoneEvent(
+                    item_id=parsed.item_id,
+                    content_index=parsed.content_index,
+                )
+            )
         elif parsed.type == "input_audio_buffer.speech_started":
             await self._send_interrupt(RealtimeModelSendInterrupt())
         elif parsed.type == "response.created":
@@ -435,6 +493,9 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
             await self._emit_event(RealtimeModelTurnEndedEvent())
         elif parsed.type == "session.created":
             await self._send_tracing_config(self._tracing_config)
+            self._update_created_session(parsed.session)  # type: ignore
+        elif parsed.type == "session.updated":
+            self._update_created_session(parsed.session)  # type: ignore
         elif parsed.type == "error":
             await self._emit_event(RealtimeModelErrorEvent(error=parsed.error))
         elif parsed.type == "conversation.item.deleted":
@@ -484,6 +545,13 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
         ):
             await self._handle_output_item(parsed.item)
 
+    def _update_created_session(self, session: OpenAISessionObject) -> None:
+        self._created_session = session
+        if session.output_audio_format:
+            self._audio_state_tracker.set_audio_format(session.output_audio_format)
+            if self._playback_tracker:
+                self._playback_tracker.set_audio_format(session.output_audio_format)
+
     async def _update_session_config(self, model_settings: RealtimeSessionModelSettings) -> None:
         session_config = self._get_session_config(model_settings)
         await self._send_raw_message(

agents/realtime/session.py

@@ -107,6 +107,11 @@ class RealtimeSession(RealtimeModelListener):
 
         self._guardrail_tasks: set[asyncio.Task[Any]] = set()
 
+    @property
+    def model(self) -> RealtimeModel:
+        """Access the underlying model for adding listeners or other direct interaction."""
+        return self._model
+
     async def __aenter__(self) -> RealtimeSession:
         """Start the session by connecting to the model. After this, you will be able to stream
         events from the model and send messages and audio to the model.
@@ -116,7 +121,8 @@ class RealtimeSession(RealtimeModelListener):
 
         model_config = self._model_config.copy()
         model_config["initial_model_settings"] = await self._get_updated_model_settings_from_agent(
-            self._current_agent
+            starting_settings=self._model_config.get("initial_model_settings", None),
+            agent=self._current_agent,
         )
 
         # Connect to the model
@@ -182,11 +188,26 @@ class RealtimeSession(RealtimeModelListener):
         elif event.type == "function_call":
             await self._handle_tool_call(event)
         elif event.type == "audio":
-            await self._put_event(RealtimeAudio(info=self._event_info, audio=event))
+            await self._put_event(
+                RealtimeAudio(
+                    info=self._event_info,
+                    audio=event,
+                    item_id=event.item_id,
+                    content_index=event.content_index,
+                )
+            )
         elif event.type == "audio_interrupted":
-            await self._put_event(RealtimeAudioInterrupted(info=self._event_info))
+            await self._put_event(
+                RealtimeAudioInterrupted(
+                    info=self._event_info, item_id=event.item_id, content_index=event.content_index
+                )
+            )
         elif event.type == "audio_done":
-            await self._put_event(RealtimeAudioEnd(info=self._event_info))
+            await self._put_event(
+                RealtimeAudioEnd(
+                    info=self._event_info, item_id=event.item_id, content_index=event.content_index
+                )
+            )
         elif event.type == "input_audio_transcription_completed":
             self._history = RealtimeSession._get_new_history(self._history, event)
             await self._put_event(
@@ -253,6 +274,8 @@ class RealtimeSession(RealtimeModelListener):
             self._stored_exception = event.exception
         elif event.type == "other":
             pass
+        elif event.type == "raw_server_event":
+            pass
         else:
             assert_never(event)
 
@@ -325,7 +348,8 @@ class RealtimeSession(RealtimeModelListener):
 
             # Get updated model settings from new agent
             updated_settings = await self._get_updated_model_settings_from_agent(
-                self._current_agent
+                starting_settings=None,
+                agent=self._current_agent,
             )
 
             # Send handoff event
@@ -504,9 +528,16 @@ class RealtimeSession(RealtimeModelListener):
 
     async def _get_updated_model_settings_from_agent(
         self,
+        starting_settings: RealtimeSessionModelSettings | None,
         agent: RealtimeAgent,
     ) -> RealtimeSessionModelSettings:
-        updated_settings: RealtimeSessionModelSettings = {}
+        # Start with run config model settings as base
+        run_config_settings = self._run_config.get("model_settings", {})
+        updated_settings: RealtimeSessionModelSettings = run_config_settings.copy()
+        # Apply starting settings (from model config) next
+        if starting_settings:
+            updated_settings.update(starting_settings)
+
         instructions, tools, handoffs = await asyncio.gather(
             agent.get_system_prompt(self._context_wrapper),
             agent.get_all_tools(self._context_wrapper),
@@ -516,10 +547,6 @@ class RealtimeSession(RealtimeModelListener):
         updated_settings["tools"] = tools or []
         updated_settings["handoffs"] = handoffs or []
 
-        # Override with initial settings
-        initial_settings = self._model_config.get("initial_model_settings", {})
-        updated_settings.update(initial_settings)
-
         disable_tracing = self._run_config.get("tracing_disabled", False)
         if disable_tracing:
             updated_settings["tracing"] = None

agents/tool.py

@@ -24,6 +24,7 @@ from .function_schema import DocstringStyle, function_schema
 from .items import RunItem
 from .logger import logger
 from .run_context import RunContextWrapper
+from .strict_schema import ensure_strict_json_schema
 from .tool_context import ToolContext
 from .tracing import SpanError
 from .util import _error_tracing
@@ -92,6 +93,10 @@ class FunctionTool:
     and returns whether the tool is enabled. You can use this to dynamically enable/disable a tool
     based on your context/state."""
 
+    def __post_init__(self):
+        if self.strict_json_schema:
+            self.params_json_schema = ensure_strict_json_schema(self.params_json_schema)
+
 
 @dataclass
 class FileSearchTool:

agents/tracing/create.py

@@ -50,8 +50,7 @@ def trace(
         group_id: Optional grouping identifier to link multiple traces from the same conversation
             or process. For instance, you might use a chat thread ID.
         metadata: Optional dictionary of additional metadata to attach to the trace.
-        disabled: If True, we will return a Trace but the Trace will not be recorded. This will
-            not be checked if there's an existing trace and `even_if_trace_running` is True.
+        disabled: If True, we will return a Trace but the Trace will not be recorded.
 
     Returns:
         The newly created trace object.

agents/tracing/processors.py

@@ -22,7 +22,7 @@ class ConsoleSpanExporter(TracingExporter):
     def export(self, items: list[Trace | Span[Any]]) -> None:
         for item in items:
             if isinstance(item, Trace):
-                print(f"[Exporter] Export trace_id={item.trace_id}, name={item.name}, ")
+                print(f"[Exporter] Export trace_id={item.trace_id}, name={item.name}")
             else:
                 print(f"[Exporter] Export span: {item.export()}")
 
@@ -121,7 +121,7 @@ class BackendSpanExporter(TracingExporter):
                     logger.debug(f"Exported {len(items)} items")
                     return
 
-                # If the response is a client error (4xx), we wont retry
+                # If the response is a client error (4xx), we won't retry
                 if 400 <= response.status_code < 500:
                     logger.error(
                         f"[non-fatal] Tracing client error {response.status_code}: {response.text}"
@@ -183,7 +183,7 @@ class BatchTraceProcessor(TracingProcessor):
         self._shutdown_event = threading.Event()
 
         # The queue size threshold at which we export immediately.
-        self._export_trigger_size = int(max_queue_size * export_trigger_ratio)
+        self._export_trigger_size = max(1, int(max_queue_size * export_trigger_ratio))
 
         # Track when we next *must* perform a scheduled export
         self._next_export_time = time.time() + self._schedule_delay
@@ -269,8 +269,7 @@ class BatchTraceProcessor(TracingProcessor):
 
     def _export_batches(self, force: bool = False):
         """Drains the queue and exports in batches. If force=True, export everything.
-        Otherwise, export up to `max_batch_size` repeatedly until the queue is empty or below a
-        certain threshold.
+        Otherwise, export up to `max_batch_size` repeatedly until the queue is completely empty.
         """
         while True:
             items_to_export: list[Span[Any] | Trace] = []

agents/tracing/traces.py

@@ -10,7 +10,7 @@ from .processor_interface import TracingProcessor
 from .scope import Scope
 
 
-class Trace:
+class Trace(abc.ABC):
     """
     A trace is the root level object that tracing creates. It represents a logical "workflow".
     """

agents/usage.py

@@ -1,6 +1,7 @@
-from dataclasses import dataclass, field
+from dataclasses import field
 
 from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
+from pydantic.dataclasses import dataclass
 
 
 @dataclass