openai-agents 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

agents/voice/__init__.py

@@ -0,0 +1,51 @@
+from .events import VoiceStreamEvent, VoiceStreamEventAudio, VoiceStreamEventLifecycle
+from .exceptions import STTWebsocketConnectionError
+from .input import AudioInput, StreamedAudioInput
+from .model import (
+    StreamedTranscriptionSession,
+    STTModel,
+    STTModelSettings,
+    TTSModel,
+    TTSModelSettings,
+    VoiceModelProvider,
+)
+from .models.openai_model_provider import OpenAIVoiceModelProvider
+from .models.openai_stt import OpenAISTTModel, OpenAISTTTranscriptionSession
+from .models.openai_tts import OpenAITTSModel
+from .pipeline import VoicePipeline
+from .pipeline_config import VoicePipelineConfig
+from .result import StreamedAudioResult
+from .utils import get_sentence_based_splitter
+from .workflow import (
+    SingleAgentVoiceWorkflow,
+    SingleAgentWorkflowCallbacks,
+    VoiceWorkflowBase,
+    VoiceWorkflowHelper,
+)
+
+__all__ = [
+    "AudioInput",
+    "StreamedAudioInput",
+    "STTModel",
+    "STTModelSettings",
+    "TTSModel",
+    "TTSModelSettings",
+    "VoiceModelProvider",
+    "StreamedAudioResult",
+    "SingleAgentVoiceWorkflow",
+    "OpenAIVoiceModelProvider",
+    "OpenAISTTModel",
+    "OpenAITTSModel",
+    "VoiceStreamEventAudio",
+    "VoiceStreamEventLifecycle",
+    "VoiceStreamEvent",
+    "VoicePipeline",
+    "VoicePipelineConfig",
+    "get_sentence_based_splitter",
+    "VoiceWorkflowHelper",
+    "VoiceWorkflowBase",
+    "SingleAgentWorkflowCallbacks",
+    "StreamedTranscriptionSession",
+    "OpenAISTTTranscriptionSession",
+    "STTWebsocketConnectionError",
+]

agents/voice/events.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal, Union
+
+from typing_extensions import TypeAlias
+
+from .imports import np, npt
+
+
+@dataclass
+class VoiceStreamEventAudio:
+    """Streaming event from the VoicePipeline"""
+
+    data: npt.NDArray[np.int16 | np.float32] | None
+    """The audio data."""
+
+    type: Literal["voice_stream_event_audio"] = "voice_stream_event_audio"
+    """The type of event."""
+
+
+@dataclass
+class VoiceStreamEventLifecycle:
+    """Streaming event from the VoicePipeline"""
+
+    event: Literal["turn_started", "turn_ended", "session_ended"]
+    """The event that occurred."""
+
+    type: Literal["voice_stream_event_lifecycle"] = "voice_stream_event_lifecycle"
+    """The type of event."""
+
+
+@dataclass
+class VoiceStreamEventError:
+    """Streaming event from the VoicePipeline"""
+
+    error: Exception
+    """The error that occurred."""
+
+    type: Literal["voice_stream_event_error"] = "voice_stream_event_error"
+    """The type of event."""
+
+
+VoiceStreamEvent: TypeAlias = Union[
+    VoiceStreamEventAudio, VoiceStreamEventLifecycle, VoiceStreamEventError
+]
+"""An event from the `VoicePipeline`, streamed via `StreamedAudioResult.stream()`."""

agents/voice/exceptions.py

@@ -0,0 +1,8 @@
+from ..exceptions import AgentsException
+
+
+class STTWebsocketConnectionError(AgentsException):
+    """Exception raised when the STT websocket connection fails."""
+
+    def __init__(self, message: str):
+        self.message = message

agents/voice/imports.py

@@ -0,0 +1,11 @@
+try:
+    import numpy as np
+    import numpy.typing as npt
+    import websockets
+except ImportError as _e:
+    raise ImportError(
+        "`numpy` + `websockets` are required to use voice. You can install them via the optional "
+        "dependency group: `pip install 'openai-agents[voice]'`."
+    ) from _e
+
+__all__ = ["np", "npt", "websockets"]

agents/voice/input.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import io
+import wave
+from dataclasses import dataclass
+
+from ..exceptions import UserError
+from .imports import np, npt
+
+DEFAULT_SAMPLE_RATE = 24000
+
+
+def _buffer_to_audio_file(
+    buffer: npt.NDArray[np.int16 | np.float32],
+    frame_rate: int = DEFAULT_SAMPLE_RATE,
+    sample_width: int = 2,
+    channels: int = 1,
+) -> tuple[str, io.BytesIO, str]:
+    if buffer.dtype == np.float32:
+        # convert to int16
+        buffer = np.clip(buffer, -1.0, 1.0)
+        buffer = (buffer * 32767).astype(np.int16)
+    elif buffer.dtype != np.int16:
+        raise UserError("Buffer must be a numpy array of int16 or float32")
+
+    audio_file = io.BytesIO()
+    with wave.open(audio_file, "w") as wav_file:
+        wav_file.setnchannels(channels)
+        wav_file.setsampwidth(sample_width)
+        wav_file.setframerate(frame_rate)
+        wav_file.writeframes(buffer.tobytes())
+        audio_file.seek(0)
+
+    # (filename, bytes, content_type)
+    return ("audio.wav", audio_file, "audio/wav")
+
+
+@dataclass
+class AudioInput:
+    """Static audio to be used as input for the VoicePipeline."""
+
+    buffer: npt.NDArray[np.int16 | np.float32]
+    """
+    A buffer containing the audio data for the agent. Must be a numpy array of int16 or float32.
+    """
+
+    frame_rate: int = DEFAULT_SAMPLE_RATE
+    """The sample rate of the audio data. Defaults to 24000."""
+
+    sample_width: int = 2
+    """The sample width of the audio data. Defaults to 2."""
+
+    channels: int = 1
+    """The number of channels in the audio data. Defaults to 1."""
+
+    def to_audio_file(self) -> tuple[str, io.BytesIO, str]:
+        """Returns a tuple of (filename, bytes, content_type)"""
+        return _buffer_to_audio_file(self.buffer, self.frame_rate, self.sample_width, self.channels)
+
+    def to_base64(self) -> str:
+        """Returns the audio data as a base64 encoded string."""
+        if self.buffer.dtype == np.float32:
+            # convert to int16
+            self.buffer = np.clip(self.buffer, -1.0, 1.0)
+            self.buffer = (self.buffer * 32767).astype(np.int16)
+        elif self.buffer.dtype != np.int16:
+            raise UserError("Buffer must be a numpy array of int16 or float32")
+
+        return base64.b64encode(self.buffer.tobytes()).decode("utf-8")
+
+
+class StreamedAudioInput:
+    """Audio input represented as a stream of audio data. You can pass this to the `VoicePipeline`
+    and then push audio data into the queue using the `add_audio` method.
+    """
+
+    def __init__(self):
+        self.queue: asyncio.Queue[npt.NDArray[np.int16 | np.float32]] = asyncio.Queue()
+
+    async def add_audio(self, audio: npt.NDArray[np.int16 | np.float32]):
+        """Adds more audio data to the stream.
+
+        Args:
+            audio: The audio data to add. Must be a numpy array of int16 or float32.
+        """
+        await self.queue.put(audio)

agents/voice/model.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import abc
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import Any, Callable, Literal
+
+from .imports import np, npt
+from .input import AudioInput, StreamedAudioInput
+from .utils import get_sentence_based_splitter
+
+DEFAULT_TTS_INSTRUCTIONS = (
+    "You will receive partial sentences. Do not complete the sentence, just read out the text."
+)
+DEFAULT_TTS_BUFFER_SIZE = 120
+
+
+@dataclass
+class TTSModelSettings:
+    """Settings for a TTS model."""
+
+    voice: (
+        Literal["alloy", "ash", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer"] | None
+    ) = None
+    """
+    The voice to use for the TTS model. If not provided, the default voice for the respective model
+    will be used.
+    """
+
+    buffer_size: int = 120
+    """The minimal size of the chunks of audio data that are being streamed out."""
+
+    dtype: npt.DTypeLike = np.int16
+    """The data type for the audio data to be returned in."""
+
+    transform_data: (
+        Callable[[npt.NDArray[np.int16 | np.float32]], npt.NDArray[np.int16 | np.float32]] | None
+    ) = None
+    """
+    A function to transform the data from the TTS model. This is useful if you want the resulting
+    audio stream to have the data in a specific shape already.
+    """
+
+    instructions: str = (
+        "You will receive partial sentences. Do not complete the sentence just read out the text."
+    )
+    """
+    The instructions to use for the TTS model. This is useful if you want to control the tone of the
+    audio output.
+    """
+
+    text_splitter: Callable[[str], tuple[str, str]] = get_sentence_based_splitter()
+    """
+    A function to split the text into chunks. This is useful if you want to split the text into
+    chunks before sending it to the TTS model rather than waiting for the whole text to be
+    processed.
+    """
+
+    speed: float | None = None
+    """The speed with which the TTS model will read the text. Between 0.25 and 4.0."""
+
+
+class TTSModel(abc.ABC):
+    """A text-to-speech model that can convert text into audio output."""
+
+    @property
+    @abc.abstractmethod
+    def model_name(self) -> str:
+        """The name of the TTS model."""
+        pass
+
+    @abc.abstractmethod
+    def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]:
+        """Given a text string, produces a stream of audio bytes, in PCM format.
+
+        Args:
+            text: The text to convert to audio.
+
+        Returns:
+            An async iterator of audio bytes, in PCM format.
+        """
+        pass
+
+
+class StreamedTranscriptionSession(abc.ABC):
+    """A streamed transcription of audio input."""
+
+    @abc.abstractmethod
+    def transcribe_turns(self) -> AsyncIterator[str]:
+        """Yields a stream of text transcriptions. Each transcription is a turn in the conversation.
+
+        This method is expected to return only after `close()` is called.
+        """
+        pass
+
+    @abc.abstractmethod
+    async def close(self) -> None:
+        """Closes the session."""
+        pass
+
+
+@dataclass
+class STTModelSettings:
+    """Settings for a speech-to-text model."""
+
+    prompt: str | None = None
+    """Instructions for the model to follow."""
+
+    language: str | None = None
+    """The language of the audio input."""
+
+    temperature: float | None = None
+    """The temperature of the model."""
+
+    turn_detection: dict[str, Any] | None = None
+    """The turn detection settings for the model when using streamed audio input."""
+
+
+class STTModel(abc.ABC):
+    """A speech-to-text model that can convert audio input into text."""
+
+    @property
+    @abc.abstractmethod
+    def model_name(self) -> str:
+        """The name of the STT model."""
+        pass
+
+    @abc.abstractmethod
+    async def transcribe(
+        self,
+        input: AudioInput,
+        settings: STTModelSettings,
+        trace_include_sensitive_data: bool,
+        trace_include_sensitive_audio_data: bool,
+    ) -> str:
+        """Given an audio input, produces a text transcription.
+
+        Args:
+            input: The audio input to transcribe.
+            settings: The settings to use for the transcription.
+            trace_include_sensitive_data: Whether to include sensitive data in traces.
+            trace_include_sensitive_audio_data: Whether to include sensitive audio data in traces.
+
+        Returns:
+            The text transcription of the audio input.
+        """
+        pass
+
+    @abc.abstractmethod
+    async def create_session(
+        self,
+        input: StreamedAudioInput,
+        settings: STTModelSettings,
+        trace_include_sensitive_data: bool,
+        trace_include_sensitive_audio_data: bool,
+    ) -> StreamedTranscriptionSession:
+        """Creates a new transcription session, which you can push audio to, and receive a stream
+        of text transcriptions.
+
+        Args:
+            input: The audio input to transcribe.
+            settings: The settings to use for the transcription.
+            trace_include_sensitive_data: Whether to include sensitive data in traces.
+            trace_include_sensitive_audio_data: Whether to include sensitive audio data in traces.
+
+        Returns:
+            A new transcription session.
+        """
+        pass
+
+
+class VoiceModelProvider(abc.ABC):
+    """The base interface for a voice model provider.
+
+    A model provider is responsible for creating speech-to-text and text-to-speech models, given a
+    name.
+    """
+
+    @abc.abstractmethod
+    def get_stt_model(self, model_name: str | None) -> STTModel:
+        """Get a speech-to-text model by name.
+
+        Args:
+            model_name: The name of the model to get.
+
+        Returns:
+            The speech-to-text model.
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_tts_model(self, model_name: str | None) -> TTSModel:
+        """Get a text-to-speech model by name."""

agents/voice/models/openai_model_provider.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import httpx
+from openai import AsyncOpenAI, DefaultAsyncHttpxClient
+
+from ...models import _openai_shared
+from ..model import STTModel, TTSModel, VoiceModelProvider
+from .openai_stt import OpenAISTTModel
+from .openai_tts import OpenAITTSModel
+
+_http_client: httpx.AsyncClient | None = None
+
+
+# If we create a new httpx client for each request, that would mean no sharing of connection pools,
+# which would mean worse latency and resource usage. So, we share the client across requests.
+def shared_http_client() -> httpx.AsyncClient:
+    global _http_client
+    if _http_client is None:
+        _http_client = DefaultAsyncHttpxClient()
+    return _http_client
+
+
+DEFAULT_STT_MODEL = "gpt-4o-transcribe"
+DEFAULT_TTS_MODEL = "gpt-4o-mini-tts"
+
+
+class OpenAIVoiceModelProvider(VoiceModelProvider):
+    """A voice model provider that uses OpenAI models."""
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        openai_client: AsyncOpenAI | None = None,
+        organization: str | None = None,
+        project: str | None = None,
+    ) -> None:
+        """Create a new OpenAI voice model provider.
+
+        Args:
+            api_key: The API key to use for the OpenAI client. If not provided, we will use the
+                default API key.
+            base_url: The base URL to use for the OpenAI client. If not provided, we will use the
+                default base URL.
+            openai_client: An optional OpenAI client to use. If not provided, we will create a new
+                OpenAI client using the api_key and base_url.
+            organization: The organization to use for the OpenAI client.
+            project: The project to use for the OpenAI client.
+        """
+        if openai_client is not None:
+            assert api_key is None and base_url is None, (
+                "Don't provide api_key or base_url if you provide openai_client"
+            )
+            self._client: AsyncOpenAI | None = openai_client
+        else:
+            self._client = None
+            self._stored_api_key = api_key
+            self._stored_base_url = base_url
+            self._stored_organization = organization
+            self._stored_project = project
+
+    # We lazy load the client in case you never actually use OpenAIProvider(). Otherwise
+    # AsyncOpenAI() raises an error if you don't have an API key set.
+    def _get_client(self) -> AsyncOpenAI:
+        if self._client is None:
+            self._client = _openai_shared.get_default_openai_client() or AsyncOpenAI(
+                api_key=self._stored_api_key or _openai_shared.get_default_openai_key(),
+                base_url=self._stored_base_url,
+                organization=self._stored_organization,
+                project=self._stored_project,
+                http_client=shared_http_client(),
+            )
+
+        return self._client
+
+    def get_stt_model(self, model_name: str | None) -> STTModel:
+        """Get a speech-to-text model by name.
+
+        Args:
+            model_name: The name of the model to get.
+
+        Returns:
+            The speech-to-text model.
+        """
+        return OpenAISTTModel(model_name or DEFAULT_STT_MODEL, self._get_client())
+
+    def get_tts_model(self, model_name: str | None) -> TTSModel:
+        """Get a text-to-speech model by name.
+
+        Args:
+            model_name: The name of the model to get.
+
+        Returns:
+            The text-to-speech model.
+        """
+        return OpenAITTSModel(model_name or DEFAULT_TTS_MODEL, self._get_client())