openai-agents 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

agents/realtime/agent.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import dataclasses
+import inspect
+from collections.abc import Awaitable
+from dataclasses import dataclass
+from typing import Any, Callable, Generic, cast
+
+from ..agent import AgentBase
+from ..lifecycle import AgentHooksBase, RunHooksBase
+from ..logger import logger
+from ..run_context import RunContextWrapper, TContext
+from ..util._types import MaybeAwaitable
+
+RealtimeAgentHooks = AgentHooksBase[TContext, "RealtimeAgent[TContext]"]
+"""Agent hooks for `RealtimeAgent`s."""
+
+RealtimeRunHooks = RunHooksBase[TContext, "RealtimeAgent[TContext]"]
+"""Run hooks for `RealtimeAgent`s."""
+
+
+@dataclass
+class RealtimeAgent(AgentBase, Generic[TContext]):
+    """A specialized agent instance that is meant to be used within a `RealtimeSession` to build
+    voice agents. Due to the nature of this agent, some configuration options are not supported
+    that are supported by regular `Agent` instances. For example:
+    - `model` choice is not supported, as all RealtimeAgents will be handled by the same model
+      within a `RealtimeSession`.
+    - `modelSettings` is not supported, as all RealtimeAgents will be handled by the same model
+      within a `RealtimeSession`.
+    - `outputType` is not supported, as RealtimeAgents do not support structured outputs.
+    - `toolUseBehavior` is not supported, as all RealtimeAgents will be handled by the same model
+      within a `RealtimeSession`.
+    - `voice` can be configured on an `Agent` level; however, it cannot be changed after the first
+      agent within a `RealtimeSession` has spoken.
+
+    See `AgentBase` for base parameters that are shared with `Agent`s.
+    """
+
+    instructions: (
+        str
+        | Callable[
+            [RunContextWrapper[TContext], RealtimeAgent[TContext]],
+            MaybeAwaitable[str],
+        ]
+        | None
+    ) = None
+    """The instructions for the agent. Will be used as the "system prompt" when this agent is
+    invoked. Describes what the agent should do, and how it responds.
+
+    Can either be a string, or a function that dynamically generates instructions for the agent. If
+    you provide a function, it will be called with the context and the agent instance. It must
+    return a string.
+    """
+
+    hooks: RealtimeAgentHooks | None = None
+    """A class that receives callbacks on various lifecycle events for this agent.
+    """
+
+    def clone(self, **kwargs: Any) -> RealtimeAgent[TContext]:
+        """Make a copy of the agent, with the given arguments changed. For example, you could do:
+        ```
+        new_agent = agent.clone(instructions="New instructions")
+        ```
+        """
+        return dataclasses.replace(self, **kwargs)
+
+    async def get_system_prompt(self, run_context: RunContextWrapper[TContext]) -> str | None:
+        """Get the system prompt for the agent."""
+        if isinstance(self.instructions, str):
+            return self.instructions
+        elif callable(self.instructions):
+            if inspect.iscoroutinefunction(self.instructions):
+                return await cast(Awaitable[str], self.instructions(run_context, self))
+            else:
+                return cast(str, self.instructions(run_context, self))
+        elif self.instructions is not None:
+            logger.error(f"Instructions must be a string or a function, got {self.instructions}")
+
+        return None

agents/realtime/config.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from typing import (
+    Any,
+    Literal,
+    Union,
+)
+
+from typing_extensions import NotRequired, TypeAlias, TypedDict
+
+from ..guardrail import OutputGuardrail
+from ..model_settings import ToolChoice
+from ..tool import Tool
+
+RealtimeModelName: TypeAlias = Union[
+    Literal[
+        "gpt-4o-realtime-preview",
+        "gpt-4o-mini-realtime-preview",
+        "gpt-4o-realtime-preview-2025-06-03",
+        "gpt-4o-realtime-preview-2024-12-17",
+        "gpt-4o-realtime-preview-2024-10-01",
+        "gpt-4o-mini-realtime-preview-2024-12-17",
+    ],
+    str,
+]
+"""The name of a realtime model."""
+
+
+RealtimeAudioFormat: TypeAlias = Union[Literal["pcm16", "g711_ulaw", "g711_alaw"], str]
+
+
+class RealtimeClientMessage(TypedDict):
+    """A raw message to be sent to the model."""
+
+    type: str  # explicitly required
+    other_data: NotRequired[dict[str, Any]]
+    """Merged into the message body."""
+
+
+class RealtimeInputAudioTranscriptionConfig(TypedDict):
+    language: NotRequired[str]
+    model: NotRequired[Literal["gpt-4o-transcribe", "gpt-4o-mini-transcribe", "whisper-1"] | str]
+    prompt: NotRequired[str]
+
+
+class RealtimeTurnDetectionConfig(TypedDict):
+    """Turn detection config. Allows extra vendor keys if needed."""
+
+    type: NotRequired[Literal["semantic_vad", "server_vad"]]
+    create_response: NotRequired[bool]
+    eagerness: NotRequired[Literal["auto", "low", "medium", "high"]]
+    interrupt_response: NotRequired[bool]
+    prefix_padding_ms: NotRequired[int]
+    silence_duration_ms: NotRequired[int]
+    threshold: NotRequired[float]
+
+
+class RealtimeSessionModelSettings(TypedDict):
+    """Model settings for a realtime model session."""
+
+    model_name: NotRequired[RealtimeModelName]
+
+    instructions: NotRequired[str]
+    modalities: NotRequired[list[Literal["text", "audio"]]]
+    voice: NotRequired[str]
+
+    input_audio_format: NotRequired[RealtimeAudioFormat]
+    output_audio_format: NotRequired[RealtimeAudioFormat]
+    input_audio_transcription: NotRequired[RealtimeInputAudioTranscriptionConfig]
+    turn_detection: NotRequired[RealtimeTurnDetectionConfig]
+
+    tool_choice: NotRequired[ToolChoice]
+    tools: NotRequired[list[Tool]]
+
+    tracing: NotRequired[RealtimeModelTracingConfig | None]
+
+
+class RealtimeGuardrailsSettings(TypedDict):
+    """Settings for output guardrails in realtime sessions."""
+
+    debounce_text_length: NotRequired[int]
+    """
+    The minimum number of characters to accumulate before running guardrails on transcript
+    deltas. Defaults to 100. Guardrails run every time the accumulated text reaches
+    1x, 2x, 3x, etc. times this threshold.
+    """
+
+
+class RealtimeModelTracingConfig(TypedDict):
+    """Configuration for tracing in realtime model sessions."""
+
+    workflow_name: NotRequired[str]
+    """The workflow name to use for tracing."""
+
+    group_id: NotRequired[str]
+    """A group identifier to use for tracing, to link multiple traces together."""
+
+    metadata: NotRequired[dict[str, Any]]
+    """Additional metadata to include with the trace."""
+
+
+class RealtimeRunConfig(TypedDict):
+    model_settings: NotRequired[RealtimeSessionModelSettings]
+
+    output_guardrails: NotRequired[list[OutputGuardrail[Any]]]
+    """List of output guardrails to run on the agent's responses."""
+
+    guardrails_settings: NotRequired[RealtimeGuardrailsSettings]
+    """Settings for guardrail execution."""
+
+    tracing_disabled: NotRequired[bool]
+    """Whether tracing is disabled for this run."""
+
+    # TODO (rm) Add history audio storage config
+
+
+class RealtimeUserInputText(TypedDict):
+    type: Literal["input_text"]
+    text: str
+
+
+class RealtimeUserInputMessage(TypedDict):
+    type: Literal["message"]
+    role: Literal["user"]
+    content: list[RealtimeUserInputText]
+
+
+RealtimeUserInput: TypeAlias = Union[str, RealtimeUserInputMessage]

agents/realtime/events.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal, Union
+
+from typing_extensions import TypeAlias
+
+from ..guardrail import OutputGuardrailResult
+from ..run_context import RunContextWrapper
+from ..tool import Tool
+from .agent import RealtimeAgent
+from .items import RealtimeItem
+from .model_events import RealtimeModelAudioEvent, RealtimeModelEvent
+
+
+@dataclass
+class RealtimeEventInfo:
+    context: RunContextWrapper
+    """The context for the event."""
+
+
+@dataclass
+class RealtimeAgentStartEvent:
+    """A new agent has started."""
+
+    agent: RealtimeAgent
+    """The new agent."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["agent_start"] = "agent_start"
+
+
+@dataclass
+class RealtimeAgentEndEvent:
+    """An agent has ended."""
+
+    agent: RealtimeAgent
+    """The agent that ended."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["agent_end"] = "agent_end"
+
+
+@dataclass
+class RealtimeHandoffEvent:
+    """An agent has handed off to another agent."""
+
+    from_agent: RealtimeAgent
+    """The agent that handed off."""
+
+    to_agent: RealtimeAgent
+    """The agent that was handed off to."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["handoff"] = "handoff"
+
+
+@dataclass
+class RealtimeToolStart:
+    """An agent is starting a tool call."""
+
+    agent: RealtimeAgent
+    """The agent that updated."""
+
+    tool: Tool
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["tool_start"] = "tool_start"
+
+
+@dataclass
+class RealtimeToolEnd:
+    """An agent has ended a tool call."""
+
+    agent: RealtimeAgent
+    """The agent that ended the tool call."""
+
+    tool: Tool
+    """The tool that was called."""
+
+    output: Any
+    """The output of the tool call."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["tool_end"] = "tool_end"
+
+
+@dataclass
+class RealtimeRawModelEvent:
+    """Forwards raw events from the model layer."""
+
+    data: RealtimeModelEvent
+    """The raw data from the model layer."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["raw_model_event"] = "raw_model_event"
+
+
+@dataclass
+class RealtimeAudioEnd:
+    """Triggered when the agent stops generating audio."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["audio_end"] = "audio_end"
+
+
+@dataclass
+class RealtimeAudio:
+    """Triggered when the agent generates new audio to be played."""
+
+    audio: RealtimeModelAudioEvent
+    """The audio event from the model layer."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["audio"] = "audio"
+
+
+@dataclass
+class RealtimeAudioInterrupted:
+    """Triggered when the agent is interrupted. Can be listened to by the user to stop audio
+    playback or give visual indicators to the user.
+    """
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["audio_interrupted"] = "audio_interrupted"
+
+
+@dataclass
+class RealtimeError:
+    """An error has occurred."""
+
+    error: Any
+    """The error that occurred."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["error"] = "error"
+
+
+@dataclass
+class RealtimeHistoryUpdated:
+    """The history has been updated. Contains the full history of the session."""
+
+    history: list[RealtimeItem]
+    """The full history of the session."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["history_updated"] = "history_updated"
+
+
+@dataclass
+class RealtimeHistoryAdded:
+    """A new item has been added to the history."""
+
+    item: RealtimeItem
+    """The new item that was added to the history."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["history_added"] = "history_added"
+
+
+@dataclass
+class RealtimeGuardrailTripped:
+    """A guardrail has been tripped and the agent has been interrupted."""
+
+    guardrail_results: list[OutputGuardrailResult]
+    """The results from all triggered guardrails."""
+
+    message: str
+    """The message that was being generated when the guardrail was triggered."""
+
+    info: RealtimeEventInfo
+    """Common info for all events, such as the context."""
+
+    type: Literal["guardrail_tripped"] = "guardrail_tripped"
+
+
+RealtimeSessionEvent: TypeAlias = Union[
+    RealtimeAgentStartEvent,
+    RealtimeAgentEndEvent,
+    RealtimeHandoffEvent,
+    RealtimeToolStart,
+    RealtimeToolEnd,
+    RealtimeRawModelEvent,
+    RealtimeAudioEnd,
+    RealtimeAudio,
+    RealtimeAudioInterrupted,
+    RealtimeError,
+    RealtimeHistoryUpdated,
+    RealtimeHistoryAdded,
+    RealtimeGuardrailTripped,
+]
+"""An event emitted by the realtime session."""

agents/realtime/items.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from typing import Annotated, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class InputText(BaseModel):
+    type: Literal["input_text"] = "input_text"
+    text: str | None = None
+
+    # Allow extra data
+    model_config = ConfigDict(extra="allow")
+
+
+class InputAudio(BaseModel):
+    type: Literal["input_audio"] = "input_audio"
+    audio: str | None = None
+    transcript: str | None = None
+
+    # Allow extra data
+    model_config = ConfigDict(extra="allow")
+
+
+class AssistantText(BaseModel):
+    type: Literal["text"] = "text"
+    text: str | None = None
+
+    # Allow extra data
+    model_config = ConfigDict(extra="allow")
+
+
+class SystemMessageItem(BaseModel):
+    item_id: str
+    previous_item_id: str | None = None
+    type: Literal["message"] = "message"
+    role: Literal["system"] = "system"
+    content: list[InputText]
+
+    # Allow extra data
+    model_config = ConfigDict(extra="allow")
+
+
+class UserMessageItem(BaseModel):
+    item_id: str
+    previous_item_id: str | None = None
+    type: Literal["message"] = "message"
+    role: Literal["user"] = "user"
+    content: list[Annotated[InputText | InputAudio, Field(discriminator="type")]]
+
+    # Allow extra data
+    model_config = ConfigDict(extra="allow")
+
+
+class AssistantMessageItem(BaseModel):
+    item_id: str
+    previous_item_id: str | None = None
+    type: Literal["message"] = "message"
+    role: Literal["assistant"] = "assistant"
+    status: Literal["in_progress", "completed", "incomplete"] | None = None
+    content: list[AssistantText]
+
+    # Allow extra data
+    model_config = ConfigDict(extra="allow")
+
+
+RealtimeMessageItem = Annotated[
+    Union[SystemMessageItem, UserMessageItem, AssistantMessageItem],
+    Field(discriminator="role"),
+]
+
+
+class RealtimeToolCallItem(BaseModel):
+    item_id: str
+    previous_item_id: str | None = None
+    type: Literal["function_call"] = "function_call"
+    status: Literal["in_progress", "completed"]
+    arguments: str
+    name: str
+    output: str | None = None
+
+    # Allow extra data
+    model_config = ConfigDict(extra="allow")
+
+
+RealtimeItem = Union[RealtimeMessageItem, RealtimeToolCallItem]
+
+
+class RealtimeResponse(BaseModel):
+    id: str
+    output: list[RealtimeMessageItem]

agents/realtime/model.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import abc
+from typing import Callable
+
+from typing_extensions import NotRequired, TypedDict
+
+from ..util._types import MaybeAwaitable
+from .config import (
+    RealtimeSessionModelSettings,
+)
+from .model_events import RealtimeModelEvent
+from .model_inputs import RealtimeModelSendEvent
+
+
+class RealtimeModelListener(abc.ABC):
+    """A listener for realtime transport events."""
+
+    @abc.abstractmethod
+    async def on_event(self, event: RealtimeModelEvent) -> None:
+        """Called when an event is emitted by the realtime transport."""
+        pass
+
+
+class RealtimeModelConfig(TypedDict):
+    """Options for connecting to a realtime model."""
+
+    api_key: NotRequired[str | Callable[[], MaybeAwaitable[str]]]
+    """The API key (or function that returns a key) to use when connecting. If unset, the model will
+    try to use a sane default. For example, the OpenAI Realtime model will try to use the
+    `OPENAI_API_KEY`  environment variable.
+    """
+
+    url: NotRequired[str]
+    """The URL to use when connecting. If unset, the model will use a sane default. For example,
+    the OpenAI Realtime model will use the default OpenAI WebSocket URL.
+    """
+
+    initial_model_settings: NotRequired[RealtimeSessionModelSettings]
+    """The initial model settings to use when connecting."""
+
+
+class RealtimeModel(abc.ABC):
+    """Interface for connecting to a realtime model and sending/receiving events."""
+
+    @abc.abstractmethod
+    async def connect(self, options: RealtimeModelConfig) -> None:
+        """Establish a connection to the model and keep it alive."""
+        pass
+
+    @abc.abstractmethod
+    def add_listener(self, listener: RealtimeModelListener) -> None:
+        """Add a listener to the model."""
+        pass
+
+    @abc.abstractmethod
+    def remove_listener(self, listener: RealtimeModelListener) -> None:
+        """Remove a listener from the model."""
+        pass
+
+    @abc.abstractmethod
+    async def send_event(self, event: RealtimeModelSendEvent) -> None:
+        """Send an event to the model."""
+        pass
+
+    @abc.abstractmethod
+    async def close(self) -> None:
+        """Close the session."""
+        pass