openai-agents 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

agents/__init__.py

@@ -73,8 +73,11 @@ from .tracing import (
     Span,
     SpanData,
     SpanError,
+    SpeechGroupSpanData,
+    SpeechSpanData,
     Trace,
     TracingProcessor,
+    TranscriptionSpanData,
     add_trace_processor,
     agent_span,
     custom_span,
@@ -89,7 +92,10 @@ from .tracing import (
     set_trace_processors,
     set_tracing_disabled,
     set_tracing_export_api_key,
+    speech_group_span,
+    speech_span,
     trace,
+    transcription_span,
 )
 from .usage import Usage
 
@@ -211,6 +217,9 @@ __all__ = [
     "handoff_span",
     "set_trace_processors",
     "set_tracing_disabled",
+    "speech_group_span",
+    "transcription_span",
+    "speech_span",
     "trace",
     "Trace",
     "TracingProcessor",
@@ -223,6 +232,9 @@ __all__ = [
     "GenerationSpanData",
     "GuardrailSpanData",
     "HandoffSpanData",
+    "SpeechGroupSpanData",
+    "SpeechSpanData",
+    "TranscriptionSpanData",
     "set_default_openai_key",
     "set_default_openai_client",
     "set_default_openai_api",

agents/models/openai_provider.py

@@ -34,6 +34,19 @@ class OpenAIProvider(ModelProvider):
         project: str | None = None,
         use_responses: bool | None = None,
     ) -> None:
+        """Create a new OpenAI provider.
+
+        Args:
+            api_key: The API key to use for the OpenAI client. If not provided, we will use the
+                default API key.
+            base_url: The base URL to use for the OpenAI client. If not provided, we will use the
+                default base URL.
+            openai_client: An optional OpenAI client to use. If not provided, we will create a new
+                OpenAI client using the api_key and base_url.
+            organization: The organization to use for the OpenAI client.
+            project: The project to use for the OpenAI client.
+            use_responses: Whether to use the OpenAI responses API.
+        """
         if openai_client is not None:
             assert api_key is None and base_url is None, (
                 "Don't provide api_key or base_url if you provide openai_client"

agents/tracing/__init__.py

@@ -10,7 +10,10 @@ from .create import (
     guardrail_span,
     handoff_span,
     response_span,
+    speech_group_span,
+    speech_span,
     trace,
+    transcription_span,
 )
 from .processor_interface import TracingProcessor
 from .processors import default_exporter, default_processor
@@ -24,6 +27,9 @@ from .span_data import (
     HandoffSpanData,
     ResponseSpanData,
     SpanData,
+    SpeechGroupSpanData,
+    SpeechSpanData,
+    TranscriptionSpanData,
 )
 from .spans import Span, SpanError
 from .traces import Trace
@@ -54,9 +60,15 @@ __all__ = [
     "GuardrailSpanData",
     "HandoffSpanData",
     "ResponseSpanData",
+    "SpeechGroupSpanData",
+    "SpeechSpanData",
+    "TranscriptionSpanData",
     "TracingProcessor",
     "gen_trace_id",
     "gen_span_id",
+    "speech_group_span",
+    "speech_span",
+    "transcription_span",
 ]
 
 

agents/tracing/create.py

@@ -13,6 +13,9 @@ from .span_data import (
     GuardrailSpanData,
     HandoffSpanData,
     ResponseSpanData,
+    SpeechGroupSpanData,
+    SpeechSpanData,
+    TranscriptionSpanData,
 )
 from .spans import Span
 from .traces import Trace
@@ -181,7 +184,11 @@ def generation_span(
     """
     return GLOBAL_TRACE_PROVIDER.create_span(
         span_data=GenerationSpanData(
-            input=input, output=output, model=model, model_config=model_config, usage=usage
+            input=input,
+            output=output,
+            model=model,
+            model_config=model_config,
+            usage=usage,
         ),
         span_id=span_id,
         parent=parent,
@@ -304,3 +311,116 @@ def guardrail_span(
         parent=parent,
         disabled=disabled,
     )
+
+
+def transcription_span(
+    model: str | None = None,
+    input: str | None = None,
+    input_format: str | None = "pcm",
+    output: str | None = None,
+    model_config: Mapping[str, Any] | None = None,
+    span_id: str | None = None,
+    parent: Trace | Span[Any] | None = None,
+    disabled: bool = False,
+) -> Span[TranscriptionSpanData]:
+    """Create a new transcription span. The span will not be started automatically, you should
+    either do `with transcription_span() ...` or call `span.start()` + `span.finish()` manually.
+
+    Args:
+        model: The name of the model used for the speech-to-text.
+        input: The audio input of the speech-to-text transcription, as a base64 encoded string of
+            audio bytes.
+        input_format: The format of the audio input (defaults to "pcm").
+        output: The output of the speech-to-text transcription.
+        model_config: The model configuration (hyperparameters) used.
+        span_id: The ID of the span. Optional. If not provided, we will generate an ID. We
+            recommend using `util.gen_span_id()` to generate a span ID, to guarantee that IDs are
+            correctly formatted.
+        parent: The parent span or trace. If not provided, we will automatically use the current
+            trace/span as the parent.
+        disabled: If True, we will return a Span but the Span will not be recorded.
+
+    Returns:
+        The newly created speech-to-text span.
+    """
+    return GLOBAL_TRACE_PROVIDER.create_span(
+        span_data=TranscriptionSpanData(
+            input=input,
+            input_format=input_format,
+            output=output,
+            model=model,
+            model_config=model_config,
+        ),
+        span_id=span_id,
+        parent=parent,
+        disabled=disabled,
+    )
+
+
+def speech_span(
+    model: str | None = None,
+    input: str | None = None,
+    output: str | None = None,
+    output_format: str | None = "pcm",
+    model_config: Mapping[str, Any] | None = None,
+    first_content_at: str | None = None,
+    span_id: str | None = None,
+    parent: Trace | Span[Any] | None = None,
+    disabled: bool = False,
+) -> Span[SpeechSpanData]:
+    """Create a new speech span. The span will not be started automatically, you should either do
+    `with speech_span() ...` or call `span.start()` + `span.finish()` manually.
+
+    Args:
+        model: The name of the model used for the text-to-speech.
+        input: The text input of the text-to-speech.
+        output: The audio output of the text-to-speech as base64 encoded string of PCM audio bytes.
+        output_format: The format of the audio output (defaults to "pcm").
+        model_config: The model configuration (hyperparameters) used.
+        first_content_at: The time of the first byte of the audio output.
+        span_id: The ID of the span. Optional. If not provided, we will generate an ID. We
+            recommend using `util.gen_span_id()` to generate a span ID, to guarantee that IDs are
+            correctly formatted.
+        parent: The parent span or trace. If not provided, we will automatically use the current
+            trace/span as the parent.
+        disabled: If True, we will return a Span but the Span will not be recorded.
+    """
+    return GLOBAL_TRACE_PROVIDER.create_span(
+        span_data=SpeechSpanData(
+            model=model,
+            input=input,
+            output=output,
+            output_format=output_format,
+            model_config=model_config,
+            first_content_at=first_content_at,
+        ),
+        span_id=span_id,
+        parent=parent,
+        disabled=disabled,
+    )
+
+
+def speech_group_span(
+    input: str | None = None,
+    span_id: str | None = None,
+    parent: Trace | Span[Any] | None = None,
+    disabled: bool = False,
+) -> Span[SpeechGroupSpanData]:
+    """Create a new speech group span. The span will not be started automatically, you should
+    either do `with speech_group_span() ...` or call `span.start()` + `span.finish()` manually.
+
+    Args:
+        input: The input text used for the speech request.
+        span_id: The ID of the span. Optional. If not provided, we will generate an ID. We
+            recommend using `util.gen_span_id()` to generate a span ID, to guarantee that IDs are
+            correctly formatted.
+        parent: The parent span or trace. If not provided, we will automatically use the current
+            trace/span as the parent.
+        disabled: If True, we will return a Span but the Span will not be recorded.
+    """
+    return GLOBAL_TRACE_PROVIDER.create_span(
+        span_data=SpeechGroupSpanData(input=input),
+        span_id=span_id,
+        parent=parent,
+        disabled=disabled,
+    )

agents/tracing/span_data.py

@@ -186,3 +186,99 @@ class GuardrailSpanData(SpanData):
             "name": self.name,
             "triggered": self.triggered,
         }
+
+
+class TranscriptionSpanData(SpanData):
+    __slots__ = (
+        "input",
+        "output",
+        "model",
+        "model_config",
+    )
+
+    def __init__(
+        self,
+        input: str | None = None,
+        input_format: str | None = "pcm",
+        output: str | None = None,
+        model: str | None = None,
+        model_config: Mapping[str, Any] | None = None,
+    ):
+        self.input = input
+        self.input_format = input_format
+        self.output = output
+        self.model = model
+        self.model_config = model_config
+
+    @property
+    def type(self) -> str:
+        return "transcription"
+
+    def export(self) -> dict[str, Any]:
+        return {
+            "type": self.type,
+            "input": {
+                "data": self.input or "",
+                "format": self.input_format,
+            },
+            "output": self.output,
+            "model": self.model,
+            "model_config": self.model_config,
+        }
+
+
+class SpeechSpanData(SpanData):
+    __slots__ = ("input", "output", "model", "model_config", "first_byte_at")
+
+    def __init__(
+        self,
+        input: str | None = None,
+        output: str | None = None,
+        output_format: str | None = "pcm",
+        model: str | None = None,
+        model_config: Mapping[str, Any] | None = None,
+        first_content_at: str | None = None,
+    ):
+        self.input = input
+        self.output = output
+        self.output_format = output_format
+        self.model = model
+        self.model_config = model_config
+        self.first_content_at = first_content_at
+
+    @property
+    def type(self) -> str:
+        return "speech"
+
+    def export(self) -> dict[str, Any]:
+        return {
+            "type": self.type,
+            "input": self.input,
+            "output": {
+                "data": self.output or "",
+                "format": self.output_format,
+            },
+            "model": self.model,
+            "model_config": self.model_config,
+            "first_content_at": self.first_content_at,
+        }
+
+
+class SpeechGroupSpanData(SpanData):
+    __slots__ = "input"
+
+    def __init__(
+        self,
+        input: str | None = None,
+    ):
+        self.input = input
+
+    @property
+    def type(self) -> str:
+        return "speech-group"
+
+    def export(self) -> dict[str, Any]:
+        return {
+            "type": self.type,
+            "input": self.input,
+        }

agents/tracing/util.py

@@ -15,3 +15,8 @@ def gen_trace_id() -> str:
 def gen_span_id() -> str:
     """Generates a new span ID."""
     return f"span_{uuid.uuid4().hex[:24]}"
+
+
+def gen_group_id() -> str:
+    """Generates a new group ID."""
+    return f"group_{uuid.uuid4().hex[:24]}"

agents/voice/__init__.py

@@ -0,0 +1,51 @@
+from .events import VoiceStreamEvent, VoiceStreamEventAudio, VoiceStreamEventLifecycle
+from .exceptions import STTWebsocketConnectionError
+from .input import AudioInput, StreamedAudioInput
+from .model import (
+    StreamedTranscriptionSession,
+    STTModel,
+    STTModelSettings,
+    TTSModel,
+    TTSModelSettings,
+    VoiceModelProvider,
+)
+from .models.openai_model_provider import OpenAIVoiceModelProvider
+from .models.openai_stt import OpenAISTTModel, OpenAISTTTranscriptionSession
+from .models.openai_tts import OpenAITTSModel
+from .pipeline import VoicePipeline
+from .pipeline_config import VoicePipelineConfig
+from .result import StreamedAudioResult
+from .utils import get_sentence_based_splitter
+from .workflow import (
+    SingleAgentVoiceWorkflow,
+    SingleAgentWorkflowCallbacks,
+    VoiceWorkflowBase,
+    VoiceWorkflowHelper,
+)
+
+__all__ = [
+    "AudioInput",
+    "StreamedAudioInput",
+    "STTModel",
+    "STTModelSettings",
+    "TTSModel",
+    "TTSModelSettings",
+    "VoiceModelProvider",
+    "StreamedAudioResult",
+    "SingleAgentVoiceWorkflow",
+    "OpenAIVoiceModelProvider",
+    "OpenAISTTModel",
+    "OpenAITTSModel",
+    "VoiceStreamEventAudio",
+    "VoiceStreamEventLifecycle",
+    "VoiceStreamEvent",
+    "VoicePipeline",
+    "VoicePipelineConfig",
+    "get_sentence_based_splitter",
+    "VoiceWorkflowHelper",
+    "VoiceWorkflowBase",
+    "SingleAgentWorkflowCallbacks",
+    "StreamedTranscriptionSession",
+    "OpenAISTTTranscriptionSession",
+    "STTWebsocketConnectionError",
+]

agents/voice/events.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal, Union
+
+from typing_extensions import TypeAlias
+
+from .imports import np, npt
+
+
+@dataclass
+class VoiceStreamEventAudio:
+    """Streaming event from the VoicePipeline"""
+
+    data: npt.NDArray[np.int16 | np.float32] | None
+    """The audio data."""
+
+    type: Literal["voice_stream_event_audio"] = "voice_stream_event_audio"
+    """The type of event."""
+
+
+@dataclass
+class VoiceStreamEventLifecycle:
+    """Streaming event from the VoicePipeline"""
+
+    event: Literal["turn_started", "turn_ended", "session_ended"]
+    """The event that occurred."""
+
+    type: Literal["voice_stream_event_lifecycle"] = "voice_stream_event_lifecycle"
+    """The type of event."""
+
+
+@dataclass
+class VoiceStreamEventError:
+    """Streaming event from the VoicePipeline"""
+
+    error: Exception
+    """The error that occurred."""
+
+    type: Literal["voice_stream_event_error"] = "voice_stream_event_error"
+    """The type of event."""
+
+
+VoiceStreamEvent: TypeAlias = Union[
+    VoiceStreamEventAudio, VoiceStreamEventLifecycle, VoiceStreamEventError
+]
+"""An event from the `VoicePipeline`, streamed via `StreamedAudioResult.stream()`."""

agents/voice/exceptions.py

@@ -0,0 +1,8 @@
+from ..exceptions import AgentsException
+
+
+class STTWebsocketConnectionError(AgentsException):
+    """Exception raised when the STT websocket connection fails."""
+
+    def __init__(self, message: str):
+        self.message = message

agents/voice/imports.py

@@ -0,0 +1,11 @@
+try:
+    import numpy as np
+    import numpy.typing as npt
+    import websockets
+except ImportError as _e:
+    raise ImportError(
+        "`numpy` + `websockets` are required to use voice. You can install them via the optional "
+        "dependency group: `pip install openai-agents[voice]`."
+    ) from _e
+
+__all__ = ["np", "npt", "websockets"]

agents/voice/input.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import io
+import wave
+from dataclasses import dataclass
+
+from ..exceptions import UserError
+from .imports import np, npt
+
+DEFAULT_SAMPLE_RATE = 24000
+
+
+def _buffer_to_audio_file(
+    buffer: npt.NDArray[np.int16 | np.float32],
+    frame_rate: int = DEFAULT_SAMPLE_RATE,
+    sample_width: int = 2,
+    channels: int = 1,
+) -> tuple[str, io.BytesIO, str]:
+    if buffer.dtype == np.float32:
+        # convert to int16
+        buffer = np.clip(buffer, -1.0, 1.0)
+        buffer = (buffer * 32767).astype(np.int16)
+    elif buffer.dtype != np.int16:
+        raise UserError("Buffer must be a numpy array of int16 or float32")
+
+    audio_file = io.BytesIO()
+    with wave.open(audio_file, "w") as wav_file:
+        wav_file.setnchannels(channels)
+        wav_file.setsampwidth(sample_width)
+        wav_file.setframerate(frame_rate)
+        wav_file.writeframes(buffer.tobytes())
+        audio_file.seek(0)
+
+    # (filename, bytes, content_type)
+    return ("audio.wav", audio_file, "audio/wav")
+
+
+@dataclass
+class AudioInput:
+    """Static audio to be used as input for the VoicePipeline."""
+
+    buffer: npt.NDArray[np.int16 | np.float32]
+    """
+    A buffer containing the audio data for the agent. Must be a numpy array of int16 or float32.
+    """
+
+    frame_rate: int = DEFAULT_SAMPLE_RATE
+    """The sample rate of the audio data. Defaults to 24000."""
+
+    sample_width: int = 2
+    """The sample width of the audio data. Defaults to 2."""
+
+    channels: int = 1
+    """The number of channels in the audio data. Defaults to 1."""
+
+    def to_audio_file(self) -> tuple[str, io.BytesIO, str]:
+        """Returns a tuple of (filename, bytes, content_type)"""
+        return _buffer_to_audio_file(self.buffer, self.frame_rate, self.sample_width, self.channels)
+
+    def to_base64(self) -> str:
+        """Returns the audio data as a base64 encoded string."""
+        if self.buffer.dtype == np.float32:
+            # convert to int16
+            self.buffer = np.clip(self.buffer, -1.0, 1.0)
+            self.buffer = (self.buffer * 32767).astype(np.int16)
+        elif self.buffer.dtype != np.int16:
+            raise UserError("Buffer must be a numpy array of int16 or float32")
+
+        return base64.b64encode(self.buffer.tobytes()).decode("utf-8")
+
+
+class StreamedAudioInput:
+    """Audio input represented as a stream of audio data. You can pass this to the `VoicePipeline`
+    and then push audio data into the queue using the `add_audio` method.
+    """
+
+    def __init__(self):
+        self.queue: asyncio.Queue[npt.NDArray[np.int16 | np.float32]] = asyncio.Queue()
+
+    async def add_audio(self, audio: npt.NDArray[np.int16 | np.float32]):
+        """Adds more audio data to the stream.
+
+        Args:
+            audio: The audio data to add. Must be a numpy array of int16 or float32.
+        """
+        await self.queue.put(audio)