dv-pipecat-ai 0.0.85.dev818__py3-none-any.whl → 0.0.85.dev858__py3-none-any.whl

pipecat/services/elevenlabs/stt.py

@@ -11,19 +11,43 @@ using segmented audio processing. The service uploads audio files and receives
 transcription results directly.
 """
 
+import base64
 import io
+import json
+from enum import Enum
 from typing import AsyncGenerator, Optional
 
 import aiohttp
 from loguru import logger
 from pydantic import BaseModel
 
-from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
-from pipecat.services.stt_service import SegmentedSTTService
-from pipecat.transcriptions.language import Language
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    ErrorFrame,
+    Frame,
+    InterimTranscriptionFrame,
+    StartFrame,
+    TranscriptionFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService
+from pipecat.transcriptions.language import Language, resolve_language
 from pipecat.utils.time import time_now_iso8601
 from pipecat.utils.tracing.service_decorators import traced_stt
 
+try:
+    from websockets.asyncio.client import connect as websocket_connect
+    from websockets.protocol import State
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use ElevenLabs Realtime STT, you need to `pip install pipecat-ai[elevenlabs]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
 
 def language_to_elevenlabs_language(language: Language) -> Optional[str]:
     """Convert a Language enum to ElevenLabs language code.
@@ -37,7 +61,7 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]:
     Returns:
         The corresponding ElevenLabs language code, or None if not supported.
     """
-    BASE_LANGUAGES = {
+    LANGUAGE_MAP = {
         Language.AF: "afr",  # Afrikaans
         Language.AM: "amh",  # Amharic
         Language.AR: "ara",  # Arabic
@@ -139,15 +163,7 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]:
         Language.ZU: "zul",  # Zulu
     }
 
-    result = BASE_LANGUAGES.get(language)
-
-    # If not found in base languages, try to find the base language from a variant
-    if not result:
-        lang_str = str(language.value)
-        base_code = lang_str.split("-")[0].lower()
-        result = base_code if base_code in BASE_LANGUAGES.values() else None
-
-    return result
+    return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
 
 
 class ElevenLabsSTTService(SegmentedSTTService):
@@ -235,7 +251,7 @@ class ElevenLabsSTTService(SegmentedSTTService):
         Args:
             language: The language to use for speech-to-text transcription.
         """
-        self.logger.info(f"Switching STT language to: [{language}]")
+        logger.info(f"Switching STT language to: [{language}]")
         self._settings["language"] = self.language_to_service_language(language)
 
     async def set_model(self, model: str):
@@ -249,7 +265,7 @@ class ElevenLabsSTTService(SegmentedSTTService):
             This method is provided for interface compatibility.
         """
         await super().set_model(model)
-        self.logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model")
+        logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model")
 
     async def _transcribe_audio(self, audio_data: bytes) -> dict:
         """Upload audio data to ElevenLabs and get transcription result.
@@ -283,7 +299,7 @@ class ElevenLabsSTTService(SegmentedSTTService):
         async with self._session.post(url, data=data, headers=headers) as response:
             if response.status != 200:
                 error_text = await response.text()
-                self.logger.error(f"ElevenLabs transcription error: {error_text}")
+                logger.error(f"ElevenLabs transcription error: {error_text}")
                 raise Exception(f"Transcription failed with status {response.status}: {error_text}")
 
             result = await response.json()
@@ -324,7 +340,7 @@ class ElevenLabsSTTService(SegmentedSTTService):
                 detected_language = result.get("language_code", "eng")
 
                 await self._handle_transcription(text, True, detected_language)
-                self.logger.debug(f"Transcription: [{text}]")
+                logger.debug(f"Transcription: [{text}]")
 
                 yield TranscriptionFrame(
                     text,
@@ -335,5 +351,457 @@ class ElevenLabsSTTService(SegmentedSTTService):
                 )
 
         except Exception as e:
-            self.logger.error(f"ElevenLabs STT error: {e}")
-            yield ErrorFrame(f"ElevenLabs STT error: {str(e)}")
+            logger.error(f"{self} exception: {e}")
+            yield ErrorFrame(error=f"{self} error: {e}")
+
+
+def audio_format_from_sample_rate(sample_rate: int) -> str:
+    """Get the appropriate audio format string for a given sample rate.
+
+    Args:
+        sample_rate: The audio sample rate in Hz.
+
+    Returns:
+        The ElevenLabs audio format string.
+    """
+    match sample_rate:
+        case 8000:
+            return "pcm_8000"
+        case 16000:
+            return "pcm_16000"
+        case 22050:
+            return "pcm_22050"
+        case 24000:
+            return "pcm_24000"
+        case 44100:
+            return "pcm_44100"
+        case 48000:
+            return "pcm_48000"
+    logger.warning(
+        f"ElevenLabsRealtimeSTTService: No audio format available for {sample_rate} sample rate, using pcm_16000"
+    )
+    return "pcm_16000"
+
+
+class CommitStrategy(str, Enum):
+    """Commit strategies for transcript segmentation."""
+
+    MANUAL = "manual"
+    VAD = "vad"
+
+
+class ElevenLabsRealtimeSTTService(WebsocketSTTService):
+    """Speech-to-text service using ElevenLabs' Realtime WebSocket API.
+
+    This service uses ElevenLabs' Realtime Speech-to-Text API to perform transcription
+    with ultra-low latency. It supports both partial (interim) and committed (final)
+    transcripts, and can use either manual commit control or automatic Voice Activity
+    Detection (VAD) for segment boundaries.
+
+    By default, uses manual commit strategy where Pipecat's VAD controls when to
+    commit transcript segments, providing consistency with other STT services.
+    """
+
+    class InputParams(BaseModel):
+        """Configuration parameters for ElevenLabs Realtime STT API.
+
+        Parameters:
+            language_code: ISO-639-1 or ISO-639-3 language code. Leave None for auto-detection.
+            commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD).
+            vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+            vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+            min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+            min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms).
+                Only used when commit_strategy is VAD. None uses ElevenLabs default.
+        """
+
+        language_code: Optional[str] = None
+        commit_strategy: CommitStrategy = CommitStrategy.MANUAL
+        vad_silence_threshold_secs: Optional[float] = None
+        vad_threshold: Optional[float] = None
+        min_speech_duration_ms: Optional[int] = None
+        min_silence_duration_ms: Optional[int] = None
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str = "api.elevenlabs.io",
+        model: str = "scribe_v2_realtime",
+        sample_rate: Optional[int] = None,
+        params: Optional[InputParams] = None,
+        **kwargs,
+    ):
+        """Initialize the ElevenLabs Realtime STT service.
+
+        Args:
+            api_key: ElevenLabs API key for authentication.
+            base_url: Base URL for ElevenLabs WebSocket API.
+            model: Model ID for transcription. Defaults to "scribe_v2_realtime".
+            sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
+            params: Configuration parameters for the STT service.
+            **kwargs: Additional arguments passed to WebsocketSTTService.
+        """
+        super().__init__(
+            sample_rate=sample_rate,
+            **kwargs,
+        )
+
+        params = params or ElevenLabsRealtimeSTTService.InputParams()
+
+        self._api_key = api_key
+        self._base_url = base_url
+        self._model_id = model
+        self._params = params
+        self._audio_format = ""  # initialized in start()
+        self._receive_task = None
+
+    def can_generate_metrics(self) -> bool:
+        """Check if the service can generate processing metrics.
+
+        Returns:
+            True, as ElevenLabs Realtime STT service supports metrics generation.
+        """
+        return True
+
+    async def set_language(self, language: Language):
+        """Set the transcription language.
+
+        Args:
+            language: The language to use for speech-to-text transcription.
+
+        Note:
+            Changing language requires reconnecting to the WebSocket.
+        """
+        logger.info(f"Switching STT language to: [{language}]")
+        self._params.language_code = language.value if isinstance(language, Language) else language
+        # Reconnect with new settings
+        await self._disconnect()
+        await self._connect()
+
+    async def set_model(self, model: str):
+        """Set the STT model.
+
+        Args:
+            model: The model name to use for transcription.
+
+        Note:
+            Changing model requires reconnecting to the WebSocket.
+        """
+        await super().set_model(model)
+        logger.info(f"Switching STT model to: [{model}]")
+        self._model_id = model
+        # Reconnect with new settings
+        await self._disconnect()
+        await self._connect()
+
+    async def start(self, frame: StartFrame):
+        """Start the STT service and establish WebSocket connection.
+
+        Args:
+            frame: Frame indicating service should start.
+        """
+        await super().start(frame)
+        self._audio_format = audio_format_from_sample_rate(self.sample_rate)
+        await self._connect()
+
+    async def stop(self, frame: EndFrame):
+        """Stop the STT service and close WebSocket connection.
+
+        Args:
+            frame: Frame indicating service should stop.
+        """
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        """Cancel the STT service and close WebSocket connection.
+
+        Args:
+            frame: Frame indicating service should be cancelled.
+        """
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def start_metrics(self):
+        """Start performance metrics collection for transcription processing."""
+        await self.start_ttfb_metrics()
+        await self.start_processing_metrics()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and handle speech events.
+
+        Args:
+            frame: The frame to process.
+            direction: Direction of frame flow in the pipeline.
+        """
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, UserStartedSpeakingFrame):
+            # Start metrics when user starts speaking
+            await self.start_metrics()
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            # Send commit when user stops speaking (manual commit mode)
+            if self._params.commit_strategy == CommitStrategy.MANUAL:
+                if self._websocket and self._websocket.state is State.OPEN:
+                    try:
+                        commit_message = {
+                            "message_type": "input_audio_chunk",
+                            "audio_base_64": "",
+                            "commit": True,
+                            "sample_rate": self.sample_rate,
+                        }
+                        await self._websocket.send(json.dumps(commit_message))
+                        logger.trace("Sent manual commit to ElevenLabs")
+                    except Exception as e:
+                        logger.warning(f"Failed to send commit: {e}")
+
+    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        """Process audio data for speech-to-text transcription.
+
+        Args:
+            audio: Raw audio bytes to transcribe.
+
+        Yields:
+            None - transcription results are handled via WebSocket responses.
+        """
+        # Reconnect if connection is closed
+        if not self._websocket or self._websocket.state is State.CLOSED:
+            await self._connect()
+
+        if self._websocket and self._websocket.state is State.OPEN:
+            try:
+                # Encode audio as base64
+                audio_base64 = base64.b64encode(audio).decode("utf-8")
+
+                # Send audio chunk
+                message = {
+                    "message_type": "input_audio_chunk",
+                    "audio_base_64": audio_base64,
+                    "commit": False,
+                    "sample_rate": self.sample_rate,
+                }
+                await self._websocket.send(json.dumps(message))
+            except Exception as e:
+                logger.error(f"Error sending audio: {e}")
+                yield ErrorFrame(f"ElevenLabs Realtime STT error: {str(e)}")
+
+        yield None
+
+    async def _connect(self):
+        """Establish WebSocket connection to ElevenLabs Realtime STT."""
+        await self._connect_websocket()
+
+        if self._websocket and not self._receive_task:
+            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
+
+    async def _disconnect(self):
+        """Close WebSocket connection and cleanup tasks."""
+        if self._receive_task:
+            await self.cancel_task(self._receive_task)
+            self._receive_task = None
+
+        await self._disconnect_websocket()
+
+    async def _connect_websocket(self):
+        """Connect to ElevenLabs Realtime STT WebSocket endpoint."""
+        try:
+            if self._websocket and self._websocket.state is State.OPEN:
+                return
+
+            logger.debug("Connecting to ElevenLabs Realtime STT")
+
+            # Build query parameters
+            params = [f"model_id={self._model_id}"]
+
+            if self._params.language_code:
+                params.append(f"language_code={self._params.language_code}")
+
+            params.append(f"encoding={self._audio_format}")
+            params.append(f"sample_rate={self.sample_rate}")
+            params.append(f"commit_strategy={self._params.commit_strategy.value}")
+
+            # Add VAD parameters if using VAD commit strategy and values are specified
+            if self._params.commit_strategy == CommitStrategy.VAD:
+                if self._params.vad_silence_threshold_secs is not None:
+                    params.append(
+                        f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}"
+                    )
+                if self._params.vad_threshold is not None:
+                    params.append(f"vad_threshold={self._params.vad_threshold}")
+                if self._params.min_speech_duration_ms is not None:
+                    params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}")
+                if self._params.min_silence_duration_ms is not None:
+                    params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}")
+
+            ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}"
+
+            headers = {"xi-api-key": self._api_key}
+
+            self._websocket = await websocket_connect(ws_url, additional_headers=headers)
+            await self._call_event_handler("on_connected")
+            logger.debug("Connected to ElevenLabs Realtime STT")
+        except Exception as e:
+            logger.error(f"{self}: unable to connect to ElevenLabs Realtime STT: {e}")
+            await self.push_error(ErrorFrame(f"Connection error: {str(e)}"))
+
+    async def _disconnect_websocket(self):
+        """Disconnect from ElevenLabs Realtime STT WebSocket."""
+        try:
+            if self._websocket and self._websocket.state is State.OPEN:
+                logger.debug("Disconnecting from ElevenLabs Realtime STT")
+                await self._websocket.close()
+        except Exception as e:
+            logger.error(f"{self} error closing websocket: {e}")
+        finally:
+            self._websocket = None
+            await self._call_event_handler("on_disconnected")
+
+    def _get_websocket(self):
+        """Get the current WebSocket connection.
+
+        Returns:
+            The WebSocket connection.
+
+        Raises:
+            Exception: If WebSocket is not connected.
+        """
+        if self._websocket:
+            return self._websocket
+        raise Exception("Websocket not connected")
+
+    async def _process_messages(self):
+        """Process incoming WebSocket messages."""
+        async for message in self._get_websocket():
+            try:
+                data = json.loads(message)
+                await self._process_response(data)
+            except json.JSONDecodeError:
+                logger.warning(f"Received non-JSON message: {message}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+
+    async def _receive_messages(self):
+        """Continuously receive and process WebSocket messages."""
+        try:
+            await self._process_messages()
+        except Exception as e:
+            logger.warning(f"{self} WebSocket connection closed: {e}")
+            # Connection closed, will reconnect on next audio chunk
+
+    async def _process_response(self, data: dict):
+        """Process a response message from ElevenLabs.
+
+        Args:
+            data: Parsed JSON response data.
+        """
+        message_type = data.get("message_type")
+
+        if message_type == "session_started":
+            logger.debug(f"ElevenLabs session started: {data}")
+
+        elif message_type == "partial_transcript":
+            await self._on_partial_transcript(data)
+
+        elif message_type == "committed_transcript":
+            await self._on_committed_transcript(data)
+
+        elif message_type == "committed_transcript_with_timestamps":
+            await self._on_committed_transcript_with_timestamps(data)
+
+        elif message_type == "input_error":
+            error_msg = data.get("error", "Unknown input error")
+            logger.error(f"ElevenLabs input error: {error_msg}")
+            await self.push_error(ErrorFrame(f"Input error: {error_msg}"))
+
+        elif message_type in ["auth_error", "quota_exceeded", "transcriber_error", "error"]:
+            error_msg = data.get("error", data.get("message", "Unknown error"))
+            logger.error(f"ElevenLabs error ({message_type}): {error_msg}")
+            await self.push_error(ErrorFrame(f"{message_type}: {error_msg}"))
+
+        else:
+            logger.debug(f"Unknown message type: {message_type}")
+
+    async def _on_partial_transcript(self, data: dict):
+        """Handle partial transcript (interim results).
+
+        Args:
+            data: Partial transcript data.
+        """
+        text = data.get("text", "").strip()
+        if not text:
+            return
+
+        await self.stop_ttfb_metrics()
+
+        # Get language if provided
+        language = data.get("language_code")
+
+        logger.trace(f"Partial transcript: [{text}]")
+
+        await self.push_frame(
+            InterimTranscriptionFrame(
+                text,
+                self._user_id,
+                time_now_iso8601(),
+                language,
+                result=data,
+            )
+        )
+
+    @traced_stt
+    async def _handle_transcription(
+        self, transcript: str, is_final: bool, language: Optional[str] = None
+    ):
+        """Handle a transcription result with tracing."""
+        pass
+
+    async def _on_committed_transcript(self, data: dict):
+        """Handle committed transcript (final results).
+
+        Args:
+            data: Committed transcript data.
+        """
+        text = data.get("text", "").strip()
+        if not text:
+            return
+
+        await self.stop_ttfb_metrics()
+        await self.stop_processing_metrics()
+
+        # Get language if provided
+        language = data.get("language_code")
+
+        logger.debug(f"Committed transcript: [{text}]")
+
+        await self._handle_transcription(text, True, language)
+
+        await self.push_frame(
+            TranscriptionFrame(
+                text,
+                self._user_id,
+                time_now_iso8601(),
+                language,
+                result=data,
+            )
+        )
+
+    async def _on_committed_transcript_with_timestamps(self, data: dict):
+        """Handle committed transcript with word-level timestamps.
+
+        Args:
+            data: Committed transcript data with timestamps.
+        """
+        text = data.get("text", "").strip()
+        if not text:
+            return
+
+        logger.debug(f"Committed transcript with timestamps: [{text}]")
+        logger.trace(f"Timestamps: {data.get('words', [])}")
+
+        # This is sent after the committed_transcript, so we don't need to
+        # push another TranscriptionFrame, but we could use the timestamps
+        # for additional processing if needed in the future

pipecat/services/elevenlabs/tts.py

@@ -14,7 +14,17 @@ import asyncio
 import base64
 import json
 import uuid
-from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple, Union
+from typing import (
+    Any,
+    AsyncGenerator,
+    Dict,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 import aiohttp
 from loguru import logger
@@ -157,7 +167,13 @@ def build_elevenlabs_voice_settings(
     Returns:
         Dictionary of voice settings or None if no valid settings are provided.
     """
-    voice_setting_keys = ["stability", "similarity_boost", "style", "use_speaker_boost", "speed"]
+    voice_setting_keys = [
+        "stability",
+        "similarity_boost",
+        "style",
+        "use_speaker_boost",
+        "speed",
+    ]
 
     voice_settings = {}
     for key in voice_setting_keys:
@@ -503,6 +519,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 return
 
             self.logger.debug("Connecting to ElevenLabs")
+            await self.start_connection_metrics()
 
             voice_id = self._voice_id
             model = self.model_name
@@ -530,17 +547,24 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
 
             # Set max websocket message size to 16MB for large audio responses
             self._websocket = await websocket_connect(
-                url, max_size=16 * 1024 * 1024, additional_headers={"xi-api-key": self._api_key}
+                url,
+                max_size=16 * 1024 * 1024,
+                additional_headers={"xi-api-key": self._api_key},
             )
 
+            await self.stop_connection_metrics(success=True, connection_type="websocket")
+            await self.stop_reconnection_metrics(success=True, reason="successful_reconnection")
             await self._call_event_handler("on_connected")
         except Exception as e:
             self.logger.error(f"{self} initialization error: {e}")
+            await self.stop_connection_metrics(success=False, error=str(e), connection_type="websocket")
+            await self.stop_reconnection_metrics(success=False, reason="connection_failed")
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
         try:
+            await self.start_reconnection_metrics()
             await self.stop_all_metrics()
 
             if self._websocket:
@@ -549,7 +573,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 if self._context_id:
                     await self._websocket.send(json.dumps({"close_socket": True}))
                 await self._websocket.close()
-                logger.debug("Disconnected from ElevenLabs")
+                self.logger.debug("Disconnected from ElevenLabs")
         except Exception as e:
             self.logger.error(f"{self} error closing websocket: {e}")
         finally:

pipecat/services/google/llm.py

@@ -760,12 +760,19 @@ class GoogleLLMService(LLMService):
 
         generation_config = GenerateContentConfig(system_instruction=system)
 
-        # Use the new google-genai client's async method
-        response = await self._client.aio.models.generate_content(
-            model=self._model_name,
-            contents=messages,
-            config=generation_config,
-        )
+        await self.start_connection_metrics()
+        
+        try:
+            # Use the new google-genai client's async method
+            response = await self._client.aio.models.generate_content(
+                model=self._model_name,
+                contents=messages,
+                config=generation_config,
+            )
+            await self.stop_connection_metrics(success=True, connection_type="grpc")
+        except Exception as e:
+            await self.stop_connection_metrics(success=False, error=str(e), connection_type="grpc")
+            raise
 
         # Extract text from response
         if response.candidates and response.candidates[0].content:
@@ -849,11 +856,19 @@ class GoogleLLMService(LLMService):
         )
 
         await self.start_ttfb_metrics()
-        return await self._client.aio.models.generate_content_stream(
-            model=self._model_name,
-            contents=messages,
-            config=generation_config,
-        )
+        await self.start_connection_metrics()
+        
+        try:
+            result = await self._client.aio.models.generate_content_stream(
+                model=self._model_name,
+                contents=messages,
+                config=generation_config,
+            )
+            await self.stop_connection_metrics(success=True, connection_type="grpc")
+            return result
+        except Exception as e:
+            await self.stop_connection_metrics(success=False, error=str(e), connection_type="grpc")
+            raise
 
     async def _stream_content_specific_context(
         self, context: OpenAILLMContext