openai-agents 0.2.8__py3-none-any.whl → 0.6.8__py3-none-any.whl

agents/realtime/config.py

@@ -1,13 +1,15 @@
 from __future__ import annotations
 
-from typing import (
-    Any,
-    Literal,
-    Union,
-)
+from collections.abc import Mapping
+from typing import Any, Literal, Union
 
+from openai.types.realtime.realtime_audio_formats import (
+    RealtimeAudioFormats as OpenAIRealtimeAudioFormats,
+)
 from typing_extensions import NotRequired, TypeAlias, TypedDict
 
+from agents.prompts import Prompt
+
 from ..guardrail import OutputGuardrail
 from ..handoffs import Handoff
 from ..model_settings import ToolChoice
@@ -15,19 +17,28 @@ from ..tool import Tool
 
 RealtimeModelName: TypeAlias = Union[
     Literal[
+        "gpt-realtime",
+        "gpt-realtime-2025-08-28",
         "gpt-4o-realtime-preview",
         "gpt-4o-mini-realtime-preview",
         "gpt-4o-realtime-preview-2025-06-03",
         "gpt-4o-realtime-preview-2024-12-17",
         "gpt-4o-realtime-preview-2024-10-01",
         "gpt-4o-mini-realtime-preview-2024-12-17",
+        "gpt-realtime-mini",
+        "gpt-realtime-mini-2025-10-06",
     ],
     str,
 ]
 """The name of a realtime model."""
 
 
-RealtimeAudioFormat: TypeAlias = Union[Literal["pcm16", "g711_ulaw", "g711_alaw"], str]
+RealtimeAudioFormat: TypeAlias = Union[
+    Literal["pcm16", "g711_ulaw", "g711_alaw"],
+    str,
+    Mapping[str, Any],
+    OpenAIRealtimeAudioFormats,
+]
 """The audio format for realtime audio streams."""
 
 
@@ -54,6 +65,13 @@ class RealtimeInputAudioTranscriptionConfig(TypedDict):
     """An optional prompt to guide transcription."""
 
 
+class RealtimeInputAudioNoiseReductionConfig(TypedDict):
+    """Noise reduction configuration for input audio."""
+
+    type: NotRequired[Literal["near_field", "far_field"]]
+    """Noise reduction mode to apply to input audio."""
+
+
 class RealtimeTurnDetectionConfig(TypedDict):
     """Turn detection config. Allows extra vendor keys if needed."""
 
@@ -78,6 +96,33 @@ class RealtimeTurnDetectionConfig(TypedDict):
     threshold: NotRequired[float]
     """The threshold for voice activity detection."""
 
+    idle_timeout_ms: NotRequired[int]
+    """Threshold for server-vad to trigger a response if the user is idle for this duration."""
+
+
+class RealtimeAudioInputConfig(TypedDict, total=False):
+    """Configuration for audio input in realtime sessions."""
+
+    format: RealtimeAudioFormat | OpenAIRealtimeAudioFormats
+    noise_reduction: RealtimeInputAudioNoiseReductionConfig | None
+    transcription: RealtimeInputAudioTranscriptionConfig
+    turn_detection: RealtimeTurnDetectionConfig
+
+
+class RealtimeAudioOutputConfig(TypedDict, total=False):
+    """Configuration for audio output in realtime sessions."""
+
+    format: RealtimeAudioFormat | OpenAIRealtimeAudioFormats
+    voice: str
+    speed: float
+
+
+class RealtimeAudioConfig(TypedDict, total=False):
+    """Audio configuration for realtime sessions."""
+
+    input: RealtimeAudioInputConfig
+    output: RealtimeAudioOutputConfig
+
 
 class RealtimeSessionModelSettings(TypedDict):
     """Model settings for a realtime model session."""
@@ -88,24 +133,36 @@ class RealtimeSessionModelSettings(TypedDict):
     instructions: NotRequired[str]
     """System instructions for the model."""
 
+    prompt: NotRequired[Prompt]
+    """The prompt to use for the model."""
+
     modalities: NotRequired[list[Literal["text", "audio"]]]
     """The modalities the model should support."""
 
+    output_modalities: NotRequired[list[Literal["text", "audio"]]]
+    """The output modalities the model should support."""
+
+    audio: NotRequired[RealtimeAudioConfig]
+    """The audio configuration for the session."""
+
     voice: NotRequired[str]
     """The voice to use for audio output."""
 
     speed: NotRequired[float]
     """The speed of the model's responses."""
 
-    input_audio_format: NotRequired[RealtimeAudioFormat]
+    input_audio_format: NotRequired[RealtimeAudioFormat | OpenAIRealtimeAudioFormats]
     """The format for input audio streams."""
 
-    output_audio_format: NotRequired[RealtimeAudioFormat]
+    output_audio_format: NotRequired[RealtimeAudioFormat | OpenAIRealtimeAudioFormats]
     """The format for output audio streams."""
 
     input_audio_transcription: NotRequired[RealtimeInputAudioTranscriptionConfig]
     """Configuration for transcribing input audio."""
 
+    input_audio_noise_reduction: NotRequired[RealtimeInputAudioNoiseReductionConfig | None]
+    """Noise reduction configuration for input audio."""
+
     turn_detection: NotRequired[RealtimeTurnDetectionConfig]
     """Configuration for detecting conversation turns."""
 
@@ -161,6 +218,9 @@ class RealtimeRunConfig(TypedDict):
     tracing_disabled: NotRequired[bool]
     """Whether tracing is disabled for this run."""
 
+    async_tool_calls: NotRequired[bool]
+    """Whether function tool calls should run asynchronously. Defaults to True."""
+
     # TODO (rm) Add history audio storage config
 
 
@@ -174,6 +234,14 @@ class RealtimeUserInputText(TypedDict):
     """The text content from the user."""
 
 
+class RealtimeUserInputImage(TypedDict, total=False):
+    """An image input from the user (Realtime)."""
+
+    type: Literal["input_image"]
+    image_url: str
+    detail: NotRequired[Literal["auto", "low", "high"] | str]
+
+
 class RealtimeUserInputMessage(TypedDict):
     """A message input from the user."""
 
@@ -183,8 +251,8 @@ class RealtimeUserInputMessage(TypedDict):
     role: Literal["user"]
     """The role identifier for user messages."""
 
-    content: list[RealtimeUserInputText]
-    """List of text content items in the message."""
+    content: list[RealtimeUserInputText | RealtimeUserInputImage]
+    """List of content items (text and image) in the message."""
 
 
 RealtimeUserInput: TypeAlias = Union[str, RealtimeUserInputMessage]

agents/realtime/events.py

@@ -69,6 +69,10 @@ class RealtimeToolStart:
     """The agent that updated."""
 
     tool: Tool
+    """The tool being called."""
+
+    arguments: str
+    """The arguments passed to the tool as a JSON string."""
 
     info: RealtimeEventInfo
     """Common info for all events, such as the context."""
@@ -86,6 +90,9 @@ class RealtimeToolEnd:
     tool: Tool
     """The tool that was called."""
 
+    arguments: str
+    """The arguments passed to the tool as a JSON string."""
+
     output: Any
     """The output of the tool call."""
 
@@ -216,6 +223,16 @@ class RealtimeGuardrailTripped:
     type: Literal["guardrail_tripped"] = "guardrail_tripped"
 
 
+@dataclass
+class RealtimeInputAudioTimeoutTriggered:
+    """Called when the model detects a period of inactivity/silence from the user."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["input_audio_timeout_triggered"] = "input_audio_timeout_triggered"
+
+
 RealtimeSessionEvent: TypeAlias = Union[
     RealtimeAgentStartEvent,
     RealtimeAgentEndEvent,
@@ -230,5 +247,6 @@ RealtimeSessionEvent: TypeAlias = Union[
     RealtimeHistoryUpdated,
     RealtimeHistoryAdded,
     RealtimeGuardrailTripped,
+    RealtimeInputAudioTimeoutTriggered,
 ]
 """An event emitted by the realtime session."""

agents/realtime/handoffs.py

@@ -13,10 +13,10 @@ from ..strict_schema import ensure_strict_json_schema
 from ..tracing.spans import SpanError
 from ..util import _error_tracing, _json
 from ..util._types import MaybeAwaitable
+from . import RealtimeAgent
 
 if TYPE_CHECKING:
     from ..agent import AgentBase
-    from . import RealtimeAgent
 
 
 # The handoff input type is the type of data passed when the agent is called via a handoff.
@@ -74,7 +74,7 @@ def realtime_handoff(
     """Create a handoff from a RealtimeAgent.
 
     Args:
-        agent: The RealtimeAgent to handoff to, or a function that returns a RealtimeAgent.
+        agent: The RealtimeAgent to handoff to.
         tool_name_override: Optional override for the name of the tool that represents the handoff.
         tool_description_override: Optional override for the description of the tool that
             represents the handoff.

agents/realtime/items.py

@@ -34,6 +34,22 @@ class InputAudio(BaseModel):
     model_config = ConfigDict(extra="allow")
 
 
+class InputImage(BaseModel):
+    """Image input content for realtime messages."""
+
+    type: Literal["input_image"] = "input_image"
+    """The type identifier for image input."""
+
+    image_url: str | None = None
+    """Data/remote URL string (data:... or https:...)."""
+
+    detail: str | None = None
+    """Optional detail hint (e.g., 'auto', 'high', 'low')."""
+
+    # Allow extra data (e.g., `detail`)
+    model_config = ConfigDict(extra="allow")
+
+
 class AssistantText(BaseModel):
     """Text content from the assistant in realtime responses."""
 
@@ -100,7 +116,7 @@ class UserMessageItem(BaseModel):
     role: Literal["user"] = "user"
     """The role identifier for user messages."""
 
-    content: list[Annotated[InputText | InputAudio, Field(discriminator="type")]]
+    content: list[Annotated[InputText | InputAudio | InputImage, Field(discriminator="type")]]
     """List of content items, can be text or audio."""
 
     # Allow extra data

agents/realtime/model.py

@@ -118,6 +118,12 @@ class RealtimeModelConfig(TypedDict):
     the OpenAI Realtime model will use the default OpenAI WebSocket URL.
     """
 
+    headers: NotRequired[dict[str, str]]
+    """The headers to use when connecting. If unset, the model will use a sane default.
+    Note that, when you set this, authorization header won't be set under the hood.
+    e.g., {"api-key": "your api key here"} for Azure OpenAI Realtime WebSocket connections.
+    """
+
     initial_model_settings: NotRequired[RealtimeSessionModelSettings]
     """The initial model settings to use when connecting."""
 
@@ -133,6 +139,13 @@ class RealtimeModelConfig(TypedDict):
     is played to the user.
     """
 
+    call_id: NotRequired[str]
+    """Attach to an existing realtime call instead of creating a new session.
+
+    When provided, the transport connects using the `call_id` query string parameter rather than a
+    model name. This is used for SIP-originated calls that are accepted via the Realtime Calls API.
+    """
+
 
 class RealtimeModel(abc.ABC):
     """Interface for connecting to a realtime model and sending/receiving events."""

agents/realtime/model_events.py

@@ -85,6 +85,17 @@ class RealtimeModelInputAudioTranscriptionCompletedEvent:
     type: Literal["input_audio_transcription_completed"] = "input_audio_transcription_completed"
 
 
+@dataclass
+class RealtimeModelInputAudioTimeoutTriggeredEvent:
+    """Input audio timeout triggered."""
+
+    item_id: str
+    audio_start_ms: int
+    audio_end_ms: int
+
+    type: Literal["input_audio_timeout_triggered"] = "input_audio_timeout_triggered"
+
+
 @dataclass
 class RealtimeModelTranscriptDeltaEvent:
     """Partial transcript update."""
@@ -174,6 +185,7 @@ RealtimeModelEvent: TypeAlias = Union[
     RealtimeModelAudioEvent,
     RealtimeModelAudioInterruptedEvent,
     RealtimeModelAudioDoneEvent,
+    RealtimeModelInputAudioTimeoutTriggeredEvent,
     RealtimeModelInputAudioTranscriptionCompletedEvent,
     RealtimeModelTranscriptDeltaEvent,
     RealtimeModelItemUpdatedEvent,

agents/realtime/model_inputs.py

@@ -24,12 +24,26 @@ class RealtimeModelInputTextContent(TypedDict):
     text: str
 
 
+class RealtimeModelInputImageContent(TypedDict, total=False):
+    """An image to be sent to the model.
+
+    The Realtime API expects `image_url` to be a string data/remote URL.
+    """
+
+    type: Literal["input_image"]
+    image_url: str
+    """String URL (data:... or https:...)."""
+
+    detail: NotRequired[str]
+    """Optional detail hint such as 'high', 'low', or 'auto'."""
+
+
 class RealtimeModelUserInputMessage(TypedDict):
     """A message to be sent to the model."""
 
     type: Literal["message"]
     role: Literal["user"]
-    content: list[RealtimeModelInputTextContent]
+    content: list[RealtimeModelInputTextContent | RealtimeModelInputImageContent]
 
 
 RealtimeModelUserInput: TypeAlias = Union[str, RealtimeModelUserInputMessage]
@@ -81,6 +95,9 @@ class RealtimeModelSendToolOutput:
 class RealtimeModelSendInterrupt:
     """Send an interrupt to the model."""
 
+    force_response_cancel: bool = False
+    """Force sending a response.cancel event even if automatic cancellation is enabled."""
+
 
 @dataclass
 class RealtimeModelSendSessionUpdate: