openai-agents 0.2.10__py3-none-any.whl → 0.3.0__py3-none-any.whl

agents/realtime/openai_realtime.py

@@ -5,59 +5,87 @@ import base64
 import inspect
 import json
 import os
+from collections.abc import Mapping
 from datetime import datetime
-from typing import Annotated, Any, Callable, Literal, Union
+from typing import Annotated, Any, Callable, Literal, Union, cast
 
 import pydantic
 import websockets
-from openai.types.beta.realtime.conversation_item import (
+from openai.types.realtime import realtime_audio_config as _rt_audio_config
+from openai.types.realtime.conversation_item import (
     ConversationItem,
     ConversationItem as OpenAIConversationItem,
 )
-from openai.types.beta.realtime.conversation_item_content import (
-    ConversationItemContent as OpenAIConversationItemContent,
-)
-from openai.types.beta.realtime.conversation_item_create_event import (
+from openai.types.realtime.conversation_item_create_event import (
     ConversationItemCreateEvent as OpenAIConversationItemCreateEvent,
 )
-from openai.types.beta.realtime.conversation_item_retrieve_event import (
+from openai.types.realtime.conversation_item_retrieve_event import (
     ConversationItemRetrieveEvent as OpenAIConversationItemRetrieveEvent,
 )
-from openai.types.beta.realtime.conversation_item_truncate_event import (
+from openai.types.realtime.conversation_item_truncate_event import (
     ConversationItemTruncateEvent as OpenAIConversationItemTruncateEvent,
 )
-from openai.types.beta.realtime.input_audio_buffer_append_event import (
+from openai.types.realtime.input_audio_buffer_append_event import (
     InputAudioBufferAppendEvent as OpenAIInputAudioBufferAppendEvent,
 )
-from openai.types.beta.realtime.input_audio_buffer_commit_event import (
+from openai.types.realtime.input_audio_buffer_commit_event import (
     InputAudioBufferCommitEvent as OpenAIInputAudioBufferCommitEvent,
 )
-from openai.types.beta.realtime.realtime_client_event import (
+from openai.types.realtime.realtime_audio_formats import (
+    AudioPCM,
+    AudioPCMA,
+    AudioPCMU,
+)
+from openai.types.realtime.realtime_client_event import (
     RealtimeClientEvent as OpenAIRealtimeClientEvent,
 )
-from openai.types.beta.realtime.realtime_server_event import (
+from openai.types.realtime.realtime_conversation_item_assistant_message import (
+    RealtimeConversationItemAssistantMessage,
+)
+from openai.types.realtime.realtime_conversation_item_function_call_output import (
+    RealtimeConversationItemFunctionCallOutput,
+)
+from openai.types.realtime.realtime_conversation_item_system_message import (
+    RealtimeConversationItemSystemMessage,
+)
+from openai.types.realtime.realtime_conversation_item_user_message import (
+    Content,
+    RealtimeConversationItemUserMessage,
+)
+from openai.types.realtime.realtime_function_tool import (
+    RealtimeFunctionTool as OpenAISessionFunction,
+)
+from openai.types.realtime.realtime_server_event import (
     RealtimeServerEvent as OpenAIRealtimeServerEvent,
 )
-from openai.types.beta.realtime.response_audio_delta_event import ResponseAudioDeltaEvent
-from openai.types.beta.realtime.response_cancel_event import (
+from openai.types.realtime.realtime_session_create_request import (
+    RealtimeSessionCreateRequest as OpenAISessionCreateRequest,
+)
+from openai.types.realtime.realtime_tracing_config import (
+    TracingConfiguration as OpenAITracingConfiguration,
+)
+from openai.types.realtime.realtime_transcription_session_create_request import (
+    RealtimeTranscriptionSessionCreateRequest as OpenAIRealtimeTranscriptionSessionCreateRequest,
+)
+from openai.types.realtime.response_audio_delta_event import ResponseAudioDeltaEvent
+from openai.types.realtime.response_cancel_event import (
     ResponseCancelEvent as OpenAIResponseCancelEvent,
 )
-from openai.types.beta.realtime.response_create_event import (
+from openai.types.realtime.response_create_event import (
     ResponseCreateEvent as OpenAIResponseCreateEvent,
 )
-from openai.types.beta.realtime.session_update_event import (
-    Session as OpenAISessionObject,
-    SessionTool as OpenAISessionTool,
-    SessionTracing as OpenAISessionTracing,
-    SessionTracingTracingConfiguration as OpenAISessionTracingConfiguration,
+from openai.types.realtime.session_update_event import (
     SessionUpdateEvent as OpenAISessionUpdateEvent,
 )
-from pydantic import BaseModel, Field, TypeAdapter
+from openai.types.responses.response_prompt import ResponsePrompt
+from pydantic import Field, TypeAdapter
 from typing_extensions import assert_never
 from websockets.asyncio.client import ClientConnection
 
 from agents.handoffs import Handoff
+from agents.prompts import Prompt
 from agents.realtime._default_tracker import ModelAudioTracker
+from agents.realtime.audio_formats import to_realtime_audio_format
 from agents.tool import FunctionTool, Tool
 from agents.util._types import MaybeAwaitable
 
@@ -103,17 +131,23 @@ from .model_inputs import (
     RealtimeModelSendUserInput,
 )
 
+# Avoid direct imports of non-exported names by referencing via module
+OpenAIRealtimeAudioConfig = _rt_audio_config.RealtimeAudioConfig
+OpenAIRealtimeAudioInput = _rt_audio_config.RealtimeAudioConfigInput  # type: ignore[attr-defined]
+OpenAIRealtimeAudioOutput = _rt_audio_config.RealtimeAudioConfigOutput  # type: ignore[attr-defined]
+
+
 _USER_AGENT = f"Agents/Python {__version__}"
 
 DEFAULT_MODEL_SETTINGS: RealtimeSessionModelSettings = {
     "voice": "ash",
-    "modalities": ["text", "audio"],
+    "modalities": ["audio"],
     "input_audio_format": "pcm16",
     "output_audio_format": "pcm16",
     "input_audio_transcription": {
         "model": "gpt-4o-mini-transcribe",
     },
-    "turn_detection": {"type": "semantic_vad"},
+    "turn_detection": {"type": "semantic_vad", "interrupt_response": True},
 }
 
 
@@ -129,19 +163,8 @@ async def get_api_key(key: str | Callable[[], MaybeAwaitable[str]] | None) -> st
     return os.getenv("OPENAI_API_KEY")
 
 
-class _InputAudioBufferTimeoutTriggeredEvent(BaseModel):
-    type: Literal["input_audio_buffer.timeout_triggered"]
-    event_id: str
-    audio_start_ms: int
-    audio_end_ms: int
-    item_id: str
-
-
 AllRealtimeServerEvents = Annotated[
-    Union[
-        OpenAIRealtimeServerEvent,
-        _InputAudioBufferTimeoutTriggeredEvent,
-    ],
+    Union[OpenAIRealtimeServerEvent,],
     Field(discriminator="type"),
 ]
 
@@ -155,11 +178,16 @@ def get_server_event_type_adapter() -> TypeAdapter[AllRealtimeServerEvents]:
     return ServerEventTypeAdapter
 
 
+# Note: Avoid a module-level union alias for Python 3.9 compatibility.
+# Using a union at runtime (e.g., A | B) in a type alias triggers evaluation
+# during import on 3.9. We instead inline the union in annotations below.
+
+
 class OpenAIRealtimeWebSocketModel(RealtimeModel):
     """A model that uses OpenAI's WebSocket API."""
 
     def __init__(self) -> None:
-        self.model = "gpt-4o-realtime-preview"  # Default model
+        self.model = "gpt-realtime"  # Default model
         self._websocket: ClientConnection | None = None
         self._websocket_task: asyncio.Task[None] | None = None
         self._listeners: list[RealtimeModelListener] = []
@@ -168,7 +196,7 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
         self._ongoing_response: bool = False
         self._tracing_config: RealtimeModelTracingConfig | Literal["auto"] | None = None
         self._playback_tracker: RealtimePlaybackTracker | None = None
-        self._created_session: OpenAISessionObject | None = None
+        self._created_session: OpenAISessionCreateRequest | None = None
         self._server_event_type_adapter = get_server_event_type_adapter()
 
     async def connect(self, options: RealtimeModelConfig) -> None:
@@ -188,15 +216,18 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
         else:
             self._tracing_config = "auto"
 
-        if not api_key:
-            raise UserError("API key is required but was not provided.")
-
         url = options.get("url", f"wss://api.openai.com/v1/realtime?model={self.model}")
 
-        headers = {
-            "Authorization": f"Bearer {api_key}",
-            "OpenAI-Beta": "realtime=v1",
-        }
+        headers: dict[str, str] = {}
+        if options.get("headers") is not None:
+            # For customizing request headers
+            headers.update(options["headers"])
+        else:
+            # OpenAI's Realtime API
+            if not api_key:
+                raise UserError("API key is required but was not provided.")
+
+            headers.update({"Authorization": f"Bearer {api_key}"})
         self._websocket = await websockets.connect(
             url,
             user_agent_header=_USER_AGENT,
@@ -214,7 +245,11 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
             converted_tracing_config = _ConversionHelper.convert_tracing_config(tracing_config)
             await self._send_raw_message(
                 OpenAISessionUpdateEvent(
-                    session=OpenAISessionObject(tracing=converted_tracing_config),
+                    session=OpenAISessionCreateRequest(
+                        model=self.model,
+                        type="realtime",
+                        tracing=converted_tracing_config,
+                    ),
                     type="session.update",
                 )
             )
@@ -296,8 +331,8 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
     async def _send_raw_message(self, event: OpenAIRealtimeClientEvent) -> None:
         """Send a raw message to the model."""
         assert self._websocket is not None, "Not connected"
-
-        await self._websocket.send(event.model_dump_json(exclude_none=True, exclude_unset=True))
+        payload = event.model_dump_json(exclude_none=True, exclude_unset=True)
+        await self._websocket.send(payload)
 
     async def _send_user_input(self, event: RealtimeModelSendUserInput) -> None:
         converted = _ConversionHelper.convert_user_input_to_item_create(event)
@@ -390,10 +425,13 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
                 f"content index: {current_item_content_index}"
             )
 
+        session = self._created_session
         automatic_response_cancellation_enabled = (
-            self._created_session
-            and self._created_session.turn_detection
-            and self._created_session.turn_detection.interrupt_response
+            session
+            and session.audio is not None
+            and session.audio.input is not None
+            and session.audio.input.turn_detection is not None
+            and session.audio.input.turn_detection.interrupt_response is True,
         )
         if not automatic_response_cancellation_enabled:
             await self._cancel_response()
@@ -487,42 +525,103 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
 
     async def _handle_ws_event(self, event: dict[str, Any]):
         await self._emit_event(RealtimeModelRawServerEvent(data=event))
+        # The public interface definedo on this Agents SDK side (e.g., RealtimeMessageItem)
+        # must be the same even after the GA migration, so this part does the conversion
+        if isinstance(event, dict) and event.get("type") in (
+            "response.output_item.added",
+            "response.output_item.done",
+        ):
+            item = event.get("item")
+            if isinstance(item, dict) and item.get("type") == "message":
+                raw_content = item.get("content") or []
+                converted_content: list[dict[str, Any]] = []
+                for part in raw_content:
+                    if not isinstance(part, dict):
+                        continue
+                    if part.get("type") == "audio":
+                        converted_content.append(
+                            {
+                                "type": "audio",
+                                "audio": part.get("audio"),
+                                "transcript": part.get("transcript"),
+                            }
+                        )
+                    elif part.get("type") == "text":
+                        converted_content.append({"type": "text", "text": part.get("text")})
+                status = item.get("status")
+                if status not in ("in_progress", "completed", "incomplete"):
+                    is_done = event.get("type") == "response.output_item.done"
+                    status = "completed" if is_done else "in_progress"
+                # Explicitly type the adapter for mypy
+                type_adapter: TypeAdapter[RealtimeMessageItem] = TypeAdapter(RealtimeMessageItem)
+                message_item: RealtimeMessageItem = type_adapter.validate_python(
+                    {
+                        "item_id": item.get("id", ""),
+                        "type": "message",
+                        "role": item.get("role", "assistant"),
+                        "content": converted_content,
+                        "status": status,
+                    }
+                )
+                await self._emit_event(RealtimeModelItemUpdatedEvent(item=message_item))
+                return
+
         try:
             if "previous_item_id" in event and event["previous_item_id"] is None:
                 event["previous_item_id"] = ""  # TODO (rm) remove
-            parsed: AllRealtimeServerEvents = self._server_event_type_adapter.validate_python(
-                event
-            )
+            parsed: AllRealtimeServerEvents = self._server_event_type_adapter.validate_python(event)
         except pydantic.ValidationError as e:
             logger.error(f"Failed to validate server event: {event}", exc_info=True)
-            await self._emit_event(
-                RealtimeModelErrorEvent(
-                    error=e,
-                )
-            )
+            await self._emit_event(RealtimeModelErrorEvent(error=e))
             return
         except Exception as e:
             event_type = event.get("type", "unknown") if isinstance(event, dict) else "unknown"
             logger.error(f"Failed to validate server event: {event}", exc_info=True)
-            await self._emit_event(
-                RealtimeModelExceptionEvent(
-                    exception=e,
-                    context=f"Failed to validate server event: {event_type}",
-                )
+            exception_event = RealtimeModelExceptionEvent(
+                exception=e,
+                context=f"Failed to validate server event: {event_type}",
             )
+            await self._emit_event(exception_event)
             return
 
-        if parsed.type == "response.audio.delta":
+        if parsed.type == "response.output_audio.delta":
             await self._handle_audio_delta(parsed)
-        elif parsed.type == "response.audio.done":
-            await self._emit_event(
-                RealtimeModelAudioDoneEvent(
-                    item_id=parsed.item_id,
-                    content_index=parsed.content_index,
-                )
+        elif parsed.type == "response.output_audio.done":
+            audio_done_event = RealtimeModelAudioDoneEvent(
+                item_id=parsed.item_id,
+                content_index=parsed.content_index,
             )
+            await self._emit_event(audio_done_event)
         elif parsed.type == "input_audio_buffer.speech_started":
-            await self._send_interrupt(RealtimeModelSendInterrupt())
+            # On VAD speech start, immediately stop local playback so the user can
+            # barge‑in without overlapping assistant audio.
+            last_audio = self._audio_state_tracker.get_last_audio_item()
+            if last_audio is not None:
+                item_id, content_index = last_audio
+                await self._emit_event(
+                    RealtimeModelAudioInterruptedEvent(item_id=item_id, content_index=content_index)
+                )
+
+                # Reset trackers so subsequent playback state queries don't
+                # reference audio that has been interrupted client‑side.
+                self._audio_state_tracker.on_interrupted()
+                if self._playback_tracker:
+                    self._playback_tracker.on_interrupted()
+
+                # If server isn't configured to auto‑interrupt/cancel, cancel the
+                # response to prevent further audio.
+                session = self._created_session
+                automatic_response_cancellation_enabled = (
+                    session
+                    and session.audio is not None
+                    and session.audio.input is not None
+                    and session.audio.input.turn_detection is not None
+                    and session.audio.input.turn_detection.interrupt_response is True,
+                )
+                if not automatic_response_cancellation_enabled:
+                    await self._cancel_response()
+            # Avoid sending conversation.item.truncate here; when GA is set to
+            # interrupt on VAD start, the server will handle truncation.
         elif parsed.type == "response.created":
             self._ongoing_response = True
             await self._emit_event(RealtimeModelTurnStartedEvent())
@@ -531,15 +630,16 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
             await self._emit_event(RealtimeModelTurnEndedEvent())
         elif parsed.type == "session.created":
             await self._send_tracing_config(self._tracing_config)
-            self._update_created_session(parsed.session)  # type: ignore
+            self._update_created_session(parsed.session)
         elif parsed.type == "session.updated":
-            self._update_created_session(parsed.session)  # type: ignore
+            self._update_created_session(parsed.session)
         elif parsed.type == "error":
             await self._emit_event(RealtimeModelErrorEvent(error=parsed.error))
         elif parsed.type == "conversation.item.deleted":
             await self._emit_event(RealtimeModelItemDeletedEvent(item_id=parsed.item_id))
         elif (
-            parsed.type == "conversation.item.created"
+            parsed.type == "conversation.item.added"
+            or parsed.type == "conversation.item.created"
             or parsed.type == "conversation.item.retrieved"
         ):
             previous_item_id = (
@@ -564,7 +664,7 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
                         item_id=parsed.item_id, transcript=parsed.transcript
                     )
                 )
-        elif parsed.type == "response.audio_transcript.delta":
+        elif parsed.type == "response.output_audio_transcript.delta":
             await self._emit_event(
                 RealtimeModelTranscriptDeltaEvent(
                     item_id=parsed.item_id, delta=parsed.delta, response_id=parsed.response_id
@@ -572,7 +672,7 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
             )
         elif (
             parsed.type == "conversation.item.input_audio_transcription.delta"
-            or parsed.type == "response.text.delta"
+            or parsed.type == "response.output_text.delta"
             or parsed.type == "response.function_call_arguments.delta"
         ):
             # No support for partials yet
@@ -583,18 +683,115 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
         ):
             await self._handle_output_item(parsed.item)
         elif parsed.type == "input_audio_buffer.timeout_triggered":
-            await self._emit_event(RealtimeModelInputAudioTimeoutTriggeredEvent(
-                item_id=parsed.item_id,
-                audio_start_ms=parsed.audio_start_ms,
-                audio_end_ms=parsed.audio_end_ms,
-            ))
+            await self._emit_event(
+                RealtimeModelInputAudioTimeoutTriggeredEvent(
+                    item_id=parsed.item_id,
+                    audio_start_ms=parsed.audio_start_ms,
+                    audio_end_ms=parsed.audio_end_ms,
+                )
+            )
+
+    def _update_created_session(
+        self,
+        session: OpenAISessionCreateRequest
+        | OpenAIRealtimeTranscriptionSessionCreateRequest
+        | Mapping[str, object]
+        | pydantic.BaseModel,
+    ) -> None:
+        # Only store/playback-format information for realtime sessions (not transcription-only)
+        normalized_session = self._normalize_session_payload(session)
+        if not normalized_session:
+            return
+
+        self._created_session = normalized_session
+        normalized_format = self._extract_audio_format(normalized_session)
+        if normalized_format is None:
+            return
+
+        self._audio_state_tracker.set_audio_format(normalized_format)
+        if self._playback_tracker:
+            self._playback_tracker.set_audio_format(normalized_format)
+
+    @staticmethod
+    def _normalize_session_payload(
+        session: OpenAISessionCreateRequest
+        | OpenAIRealtimeTranscriptionSessionCreateRequest
+        | Mapping[str, object]
+        | pydantic.BaseModel,
+    ) -> OpenAISessionCreateRequest | None:
+        if isinstance(session, OpenAISessionCreateRequest):
+            return session
+
+        if isinstance(session, OpenAIRealtimeTranscriptionSessionCreateRequest):
+            return None
+
+        session_payload: Mapping[str, object]
+        if isinstance(session, pydantic.BaseModel):
+            session_payload = cast(Mapping[str, object], session.model_dump())
+        elif isinstance(session, Mapping):
+            session_payload = session
+        else:
+            return None
+
+        if OpenAIRealtimeWebSocketModel._is_transcription_session(session_payload):
+            return None
 
-    def _update_created_session(self, session: OpenAISessionObject) -> None:
-        self._created_session = session
-        if session.output_audio_format:
-            self._audio_state_tracker.set_audio_format(session.output_audio_format)
-            if self._playback_tracker:
-                self._playback_tracker.set_audio_format(session.output_audio_format)
+        try:
+            return OpenAISessionCreateRequest.model_validate(session_payload)
+        except pydantic.ValidationError:
+            return None
+
+    @staticmethod
+    def _is_transcription_session(payload: Mapping[str, object]) -> bool:
+        try:
+            OpenAIRealtimeTranscriptionSessionCreateRequest.model_validate(payload)
+        except pydantic.ValidationError:
+            return False
+        else:
+            return True
+
+    @staticmethod
+    def _extract_audio_format(session: OpenAISessionCreateRequest) -> str | None:
+        audio = session.audio
+        if not audio or not audio.output or not audio.output.format:
+            return None
+
+        return OpenAIRealtimeWebSocketModel._normalize_audio_format(audio.output.format)
+
+    @staticmethod
+    def _normalize_audio_format(fmt: object) -> str:
+        if isinstance(fmt, AudioPCM):
+            return "pcm16"
+        if isinstance(fmt, AudioPCMU):
+            return "g711_ulaw"
+        if isinstance(fmt, AudioPCMA):
+            return "g711_alaw"
+
+        fmt_type = OpenAIRealtimeWebSocketModel._read_format_type(fmt)
+        if isinstance(fmt_type, str) and fmt_type:
+            return fmt_type
+
+        return str(fmt)
+
+    @staticmethod
+    def _read_format_type(fmt: object) -> str | None:
+        if isinstance(fmt, str):
+            return fmt
+
+        if isinstance(fmt, Mapping):
+            type_value = fmt.get("type")
+            return type_value if isinstance(type_value, str) else None
+
+        if isinstance(fmt, pydantic.BaseModel):
+            type_value = fmt.model_dump().get("type")
+            return type_value if isinstance(type_value, str) else None
+
+        try:
+            type_value = fmt.type  # type: ignore[attr-defined]
+        except AttributeError:
+            return None
+
+        return type_value if isinstance(type_value, str) else None
 
     async def _update_session_config(self, model_settings: RealtimeSessionModelSettings) -> None:
         session_config = self._get_session_config(model_settings)
@@ -604,51 +801,95 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
 
     def _get_session_config(
         self, model_settings: RealtimeSessionModelSettings
-    ) -> OpenAISessionObject:
+    ) -> OpenAISessionCreateRequest:
         """Get the session config."""
-        return OpenAISessionObject(
-            instructions=model_settings.get("instructions", None),
-            model=(
-                model_settings.get("model_name", self.model)  # type: ignore
-                or DEFAULT_MODEL_SETTINGS.get("model_name")
-            ),
-            voice=model_settings.get("voice", DEFAULT_MODEL_SETTINGS.get("voice")),
-            speed=model_settings.get("speed", None),
-            modalities=model_settings.get("modalities", DEFAULT_MODEL_SETTINGS.get("modalities")),
-            input_audio_format=model_settings.get(
-                "input_audio_format",
-                DEFAULT_MODEL_SETTINGS.get("input_audio_format"),  # type: ignore
-            ),
-            output_audio_format=model_settings.get(
-                "output_audio_format",
-                DEFAULT_MODEL_SETTINGS.get("output_audio_format"),  # type: ignore
-            ),
-            input_audio_transcription=model_settings.get(
-                "input_audio_transcription",
-                DEFAULT_MODEL_SETTINGS.get("input_audio_transcription"),  # type: ignore
-            ),
-            turn_detection=model_settings.get(
-                "turn_detection",
-                DEFAULT_MODEL_SETTINGS.get("turn_detection"),  # type: ignore
-            ),
-            tool_choice=model_settings.get(
-                "tool_choice",
-                DEFAULT_MODEL_SETTINGS.get("tool_choice"),  # type: ignore
-            ),
-            tools=self._tools_to_session_tools(
-                tools=model_settings.get("tools", []), handoffs=model_settings.get("handoffs", [])
+        model_name = (model_settings.get("model_name") or self.model) or "gpt-realtime"
+
+        voice = model_settings.get("voice", DEFAULT_MODEL_SETTINGS.get("voice"))
+        speed = model_settings.get("speed")
+        modalities = model_settings.get("modalities", DEFAULT_MODEL_SETTINGS.get("modalities"))
+
+        input_audio_format = model_settings.get(
+            "input_audio_format",
+            DEFAULT_MODEL_SETTINGS.get("input_audio_format"),
+        )
+        input_audio_transcription = model_settings.get(
+            "input_audio_transcription",
+            DEFAULT_MODEL_SETTINGS.get("input_audio_transcription"),
+        )
+        turn_detection = model_settings.get(
+            "turn_detection",
+            DEFAULT_MODEL_SETTINGS.get("turn_detection"),
+        )
+        output_audio_format = model_settings.get(
+            "output_audio_format",
+            DEFAULT_MODEL_SETTINGS.get("output_audio_format"),
+        )
+
+        input_audio_config = None
+        if any(
+            value is not None
+            for value in [input_audio_format, input_audio_transcription, turn_detection]
+        ):
+            input_audio_config = OpenAIRealtimeAudioInput(
+                format=to_realtime_audio_format(input_audio_format),
+                transcription=cast(Any, input_audio_transcription),
+                turn_detection=cast(Any, turn_detection),
+            )
+
+        output_audio_config = None
+        if any(value is not None for value in [output_audio_format, speed, voice]):
+            output_audio_config = OpenAIRealtimeAudioOutput(
+                format=to_realtime_audio_format(output_audio_format),
+                speed=speed,
+                voice=voice,
+            )
+
+        audio_config = None
+        if input_audio_config or output_audio_config:
+            audio_config = OpenAIRealtimeAudioConfig(
+                input=input_audio_config,
+                output=output_audio_config,
+            )
+
+        prompt: ResponsePrompt | None = None
+        if model_settings.get("prompt") is not None:
+            _passed_prompt: Prompt = model_settings["prompt"]
+            variables: dict[str, Any] | None = _passed_prompt.get("variables")
+            prompt = ResponsePrompt(
+                id=_passed_prompt["id"],
+                variables=variables,
+                version=_passed_prompt.get("version"),
+            )
+
+        # Construct full session object. `type` will be excluded at serialization time for updates.
+        return OpenAISessionCreateRequest(
+            model=model_name,
+            type="realtime",
+            instructions=model_settings.get("instructions"),
+            prompt=prompt,
+            output_modalities=modalities,
+            audio=audio_config,
+            max_output_tokens=cast(Any, model_settings.get("max_output_tokens")),
+            tool_choice=cast(Any, model_settings.get("tool_choice")),
+            tools=cast(
+                Any,
+                self._tools_to_session_tools(
+                    tools=model_settings.get("tools", []),
+                    handoffs=model_settings.get("handoffs", []),
+                ),
             ),
         )
 
     def _tools_to_session_tools(
         self, tools: list[Tool], handoffs: list[Handoff]
-    ) -> list[OpenAISessionTool]:
-        converted_tools: list[OpenAISessionTool] = []
+    ) -> list[OpenAISessionFunction]:
+        converted_tools: list[OpenAISessionFunction] = []
         for tool in tools:
             if not isinstance(tool, FunctionTool):
                 raise UserError(f"Tool {tool.name} is unsupported. Must be a function tool.")
             converted_tools.append(
-                OpenAISessionTool(
+                OpenAISessionFunction(
                     name=tool.name,
                     description=tool.description,
                     parameters=tool.params_json_schema,
@@ -658,7 +899,7 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
 
         for handoff in handoffs:
             converted_tools.append(
-                OpenAISessionTool(
+                OpenAISessionFunction(
                     name=handoff.tool_name,
                     description=handoff.tool_description,
                     parameters=handoff.input_json_schema,
@@ -674,15 +915,32 @@ class _ConversionHelper:
     def conversation_item_to_realtime_message_item(
         cls, item: ConversationItem, previous_item_id: str | None
     ) -> RealtimeMessageItem:
+        if not isinstance(
+            item,
+            (
+                RealtimeConversationItemUserMessage,
+                RealtimeConversationItemAssistantMessage,
+                RealtimeConversationItemSystemMessage,
+            ),
+        ):
+            raise ValueError("Unsupported conversation item type for message conversion.")
+        content: list[dict[str, Any]] = []
+        for each in item.content:
+            c = each.model_dump()
+            if each.type == "output_text":
+                # For backward-compatibility of assistant message items
+                c["type"] = "text"
+            elif each.type == "output_audio":
+                # For backward-compatibility of assistant message items
+                c["type"] = "audio"
+            content.append(c)
         return TypeAdapter(RealtimeMessageItem).validate_python(
             {
                 "item_id": item.id or "",
                 "previous_item_id": previous_item_id,
                 "type": item.type,
                 "role": item.role,
-                "content": (
-                    [content.model_dump() for content in item.content] if item.content else []
-                ),
+                "content": content,
                 "status": "in_progress",
             },
         )
@@ -702,12 +960,12 @@ class _ConversionHelper:
     @classmethod
     def convert_tracing_config(
         cls, tracing_config: RealtimeModelTracingConfig | Literal["auto"] | None
-    ) -> OpenAISessionTracing | None:
+    ) -> OpenAITracingConfiguration | Literal["auto"] | None:
         if tracing_config is None:
             return None
         elif tracing_config == "auto":
             return "auto"
-        return OpenAISessionTracingConfiguration(
+        return OpenAITracingConfiguration(
             group_id=tracing_config.get("group_id"),
             metadata=tracing_config.get("metadata"),
             workflow_name=tracing_config.get("workflow_name"),
@@ -720,22 +978,53 @@ class _ConversionHelper:
         user_input = event.user_input
 
         if isinstance(user_input, dict):
-            return OpenAIConversationItem(
+            content: list[Content] = []
+            for item in user_input.get("content", []):
+                try:
+                    if not isinstance(item, dict):
+                        continue
+                    t = item.get("type")
+                    if t == "input_text":
+                        _txt = item.get("text")
+                        text_val = _txt if isinstance(_txt, str) else None
+                        content.append(Content(type="input_text", text=text_val))
+                    elif t == "input_image":
+                        iu = item.get("image_url")
+                        if isinstance(iu, str) and iu:
+                            d = item.get("detail")
+                            detail_val = cast(
+                                Literal["auto", "low", "high"] | None,
+                                d if isinstance(d, str) and d in ("auto", "low", "high") else None,
+                            )
+                            if detail_val is None:
+                                content.append(
+                                    Content(
+                                        type="input_image",
+                                        image_url=iu,
+                                    )
+                                )
+                            else:
+                                content.append(
+                                    Content(
+                                        type="input_image",
+                                        image_url=iu,
+                                        detail=detail_val,
+                                    )
+                                )
+                    # ignore unknown types for forward-compat
+                except Exception:
+                    # best-effort; skip malformed parts
+                    continue
+            return RealtimeConversationItemUserMessage(
                 type="message",
                 role="user",
-                content=[
-                    OpenAIConversationItemContent(
-                        type="input_text",
-                        text=item.get("text"),
-                    )
-                    for item in user_input.get("content", [])
-                ],
+                content=content,
             )
         else:
-            return OpenAIConversationItem(
+            return RealtimeConversationItemUserMessage(
                 type="message",
                 role="user",
-                content=[OpenAIConversationItemContent(type="input_text", text=user_input)],
+                content=[Content(type="input_text", text=user_input)],
             )
 
     @classmethod
@@ -761,7 +1050,7 @@ class _ConversionHelper:
     def convert_tool_output(cls, event: RealtimeModelSendToolOutput) -> OpenAIRealtimeClientEvent:
         return OpenAIConversationItemCreateEvent(
             type="conversation.item.create",
-            item=OpenAIConversationItem(
+            item=RealtimeConversationItemFunctionCallOutput(
                 type="function_call_output",
                 output=event.output,
                 call_id=event.tool_call.call_id,