dv-pipecat-ai 0.0.85.dev5__py3-none-any.whl → 0.0.85.dev698__py3-none-any.whl

pipecat/services/elevenlabs/stt.py

@@ -4,12 +4,19 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-"""ElevenLabs speech-to-text service implementation."""
+"""ElevenLabs speech-to-text service implementation.
 
-import asyncio
+This module provides integration with ElevenLabs' Speech-to-Text API for transcription
+using segmented audio processing. The service uploads audio files and receives
+transcription results directly.
+"""
+
+import io
 from typing import AsyncGenerator, Optional
 
+import aiohttp
 from loguru import logger
+from pydantic import BaseModel
 
 from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
 from pipecat.services.stt_service import SegmentedSTTService
@@ -17,345 +24,316 @@ from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
 from pipecat.utils.tracing.service_decorators import traced_stt
 
-try:
-    from elevenlabs.client import ElevenLabs
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error("In order to use ElevenLabs, you need to `pip install pipecat-ai[elevenlabs]`.")
-    raise Exception(f"Missing module: {e}")
-
 
 def language_to_elevenlabs_language(language: Language) -> Optional[str]:
-    """Maps pipecat Language enum to ElevenLabs language codes.
+    """Convert a Language enum to ElevenLabs language code.
+
+    Source:
+        https://elevenlabs.io/docs/capabilities/speech-to-text
 
     Args:
-        language: A Language enum value representing the input language.
+        language: The Language enum value to convert.
 
     Returns:
-        str or None: The corresponding ElevenLabs language code, or None if not supported.
+        The corresponding ElevenLabs language code, or None if not supported.
     """
-    language_map = {
-        # English
-        Language.EN: "eng",
-        Language.EN_US: "eng",
-        Language.EN_GB: "eng",
-        Language.EN_AU: "eng",
-        Language.EN_CA: "eng",
-        Language.EN_IN: "eng",
-        Language.EN_IE: "eng",
-        Language.EN_NZ: "eng",
-        Language.EN_ZA: "eng",
-        Language.EN_SG: "eng",
-        Language.EN_HK: "eng",
-        Language.EN_PH: "eng",
-        Language.EN_KE: "eng",
-        Language.EN_NG: "eng",
-        Language.EN_TZ: "eng",
-        # Spanish
-        Language.ES: "spa",
-        Language.ES_ES: "spa",
-        Language.ES_MX: "spa",
-        Language.ES_AR: "spa",
-        Language.ES_CO: "spa",
-        Language.ES_CL: "spa",
-        Language.ES_VE: "spa",
-        Language.ES_PE: "spa",
-        Language.ES_EC: "spa",
-        Language.ES_GT: "spa",
-        Language.ES_CU: "spa",
-        Language.ES_BO: "spa",
-        Language.ES_DO: "spa",
-        Language.ES_HN: "spa",
-        Language.ES_PY: "spa",
-        Language.ES_SV: "spa",
-        Language.ES_NI: "spa",
-        Language.ES_CR: "spa",
-        Language.ES_PA: "spa",
-        Language.ES_UY: "spa",
-        Language.ES_PR: "spa",
-        Language.ES_US: "spa",
-        Language.ES_GQ: "spa",
-        # French
-        Language.FR: "fra",
-        Language.FR_FR: "fra",
-        Language.FR_CA: "fra",
-        Language.FR_BE: "fra",
-        Language.FR_CH: "fra",
-        # German
-        Language.DE: "deu",
-        Language.DE_DE: "deu",
-        Language.DE_AT: "deu",
-        Language.DE_CH: "deu",
-        # Italian
-        Language.IT: "ita",
-        Language.IT_IT: "ita",
-        # Portuguese
-        Language.PT: "por",
-        Language.PT_PT: "por",
-        Language.PT_BR: "por",
-        # Hindi
-        Language.HI: "hin",
-        Language.HI_IN: "hin",
-        # Arabic
-        Language.AR: "ara",
-        Language.AR_SA: "ara",
-        Language.AR_EG: "ara",
-        Language.AR_AE: "ara",
-        Language.AR_BH: "ara",
-        Language.AR_DZ: "ara",
-        Language.AR_IQ: "ara",
-        Language.AR_JO: "ara",
-        Language.AR_KW: "ara",
-        Language.AR_LB: "ara",
-        Language.AR_LY: "ara",
-        Language.AR_MA: "ara",
-        Language.AR_OM: "ara",
-        Language.AR_QA: "ara",
-        Language.AR_SY: "ara",
-        Language.AR_TN: "ara",
-        Language.AR_YE: "ara",
-        # Japanese
-        Language.JA: "jpn",
-        Language.JA_JP: "jpn",
-        # Korean
-        Language.KO: "kor",
-        Language.KO_KR: "kor",
-        # Chinese
-        Language.ZH: "cmn",
-        Language.ZH_CN: "cmn",
-        Language.ZH_TW: "cmn",
-        Language.ZH_HK: "cmn",
-        # Russian
-        Language.RU: "rus",
-        Language.RU_RU: "rus",
-        # Dutch
-        Language.NL: "nld",
-        Language.NL_NL: "nld",
-        Language.NL_BE: "nld",
-        # Polish
-        Language.PL: "pol",
-        Language.PL_PL: "pol",
-        # Turkish
-        Language.TR: "tur",
-        Language.TR_TR: "tur",
-        # Swedish
-        Language.SV: "swe",
-        Language.SV_SE: "swe",
-        # Norwegian
-        Language.NO: "nor",
-        Language.NB: "nor",
-        Language.NN: "nor",
-        # Danish
-        Language.DA: "dan",
-        Language.DA_DK: "dan",
-        # Finnish
-        Language.FI: "fin",
-        Language.FI_FI: "fin",
-        # Czech
-        Language.CS: "ces",
-        Language.CS_CZ: "ces",
-        # Hungarian
-        Language.HU: "hun",
-        Language.HU_HU: "hun",
-        # Greek
-        Language.EL: "ell",
-        Language.EL_GR: "ell",
-        # Hebrew
-        Language.HE: "heb",
-        Language.HE_IL: "heb",
-        # Thai
-        Language.TH: "tha",
-        Language.TH_TH: "tha",
-        # Vietnamese
-        Language.VI: "vie",
-        Language.VI_VN: "vie",
-        # Indonesian
-        Language.ID: "ind",
-        Language.ID_ID: "ind",
-        # Malay
-        Language.MS: "msa",
-        Language.MS_MY: "msa",
-        # Ukrainian
-        Language.UK: "ukr",
-        Language.UK_UA: "ukr",
-        # Bulgarian
-        Language.BG: "bul",
-        Language.BG_BG: "bul",
-        # Croatian
-        Language.HR: "hrv",
-        Language.HR_HR: "hrv",
-        # Slovak
-        Language.SK: "slk",
-        Language.SK_SK: "slk",
-        # Slovenian
-        Language.SL: "slv",
-        Language.SL_SI: "slv",
-        # Estonian
-        Language.ET: "est",
-        Language.ET_EE: "est",
-        # Latvian
-        Language.LV: "lav",
-        Language.LV_LV: "lav",
-        # Lithuanian
-        Language.LT: "lit",
-        Language.LT_LT: "lit",
-        Language.TA: "tam",  # Tamil
-        Language.TA_IN: "tam",  # Tamil
-        Language.TE: "tel",  # Telugu
-        Language.TE_IN: "tel",  # Telugu
+    BASE_LANGUAGES = {
+        Language.AF: "afr",  # Afrikaans
+        Language.AM: "amh",  # Amharic
+        Language.AR: "ara",  # Arabic
+        Language.HY: "hye",  # Armenian
+        Language.AS: "asm",  # Assamese
+        Language.AST: "ast",  # Asturian
+        Language.AZ: "aze",  # Azerbaijani
+        Language.BE: "bel",  # Belarusian
+        Language.BN: "ben",  # Bengali
+        Language.BS: "bos",  # Bosnian
+        Language.BG: "bul",  # Bulgarian
+        Language.MY: "mya",  # Burmese
+        Language.YUE: "yue",  # Cantonese
+        Language.CA: "cat",  # Catalan
+        Language.CEB: "ceb",  # Cebuano
+        Language.NY: "nya",  # Chichewa
+        Language.HR: "hrv",  # Croatian
+        Language.CS: "ces",  # Czech
+        Language.DA: "dan",  # Danish
+        Language.NL: "nld",  # Dutch
+        Language.EN: "eng",  # English
+        Language.ET: "est",  # Estonian
+        Language.FIL: "fil",  # Filipino
+        Language.FI: "fin",  # Finnish
+        Language.FR: "fra",  # French
+        Language.FF: "ful",  # Fulah
+        Language.GL: "glg",  # Galician
+        Language.LG: "lug",  # Ganda
+        Language.KA: "kat",  # Georgian
+        Language.DE: "deu",  # German
+        Language.EL: "ell",  # Greek
+        Language.GU: "guj",  # Gujarati
+        Language.HA: "hau",  # Hausa
+        Language.HE: "heb",  # Hebrew
+        Language.HI: "hin",  # Hindi
+        Language.HU: "hun",  # Hungarian
+        Language.IS: "isl",  # Icelandic
+        Language.IG: "ibo",  # Igbo
+        Language.ID: "ind",  # Indonesian
+        Language.GA: "gle",  # Irish
+        Language.IT: "ita",  # Italian
+        Language.JA: "jpn",  # Japanese
+        Language.JV: "jav",  # Javanese
+        Language.KEA: "kea",  # Kabuverdianu
         Language.KN: "kan",  # Kannada
-        Language.KN_IN: "kan",  # Kannada
+        Language.KK: "kaz",  # Kazakh
+        Language.KM: "khm",  # Khmer
+        Language.KO: "kor",  # Korean
+        Language.KU: "kur",  # Kurdish
+        Language.KY: "kir",  # Kyrgyz
+        Language.LO: "lao",  # Lao
+        Language.LV: "lav",  # Latvian
+        Language.LN: "lin",  # Lingala
+        Language.LT: "lit",  # Lithuanian
+        Language.LUO: "luo",  # Luo
+        Language.LB: "ltz",  # Luxembourgish
+        Language.MK: "mkd",  # Macedonian
+        Language.MS: "msa",  # Malay
         Language.ML: "mal",  # Malayalam
-        Language.ML_IN: "mal",  # Malayalam
+        Language.MT: "mlt",  # Maltese
+        Language.ZH: "zho",  # Mandarin Chinese
+        Language.MI: "mri",  # Māori
         Language.MR: "mar",  # Marathi
-        Language.MR_IN: "mar",  # Marathi
+        Language.MN: "mon",  # Mongolian
+        Language.NE: "nep",  # Nepali
+        Language.NSO: "nso",  # Northern Sotho
+        Language.NO: "nor",  # Norwegian
+        Language.OC: "oci",  # Occitan
+        Language.OR: "ori",  # Odia
+        Language.PS: "pus",  # Pashto
+        Language.FA: "fas",  # Persian
+        Language.PL: "pol",  # Polish
+        Language.PT: "por",  # Portuguese
+        Language.PA: "pan",  # Punjabi
+        Language.RO: "ron",  # Romanian
+        Language.RU: "rus",  # Russian
+        Language.SR: "srp",  # Serbian
+        Language.SN: "sna",  # Shona
+        Language.SD: "snd",  # Sindhi
+        Language.SK: "slk",  # Slovak
+        Language.SL: "slv",  # Slovenian
+        Language.SO: "som",  # Somali
+        Language.ES: "spa",  # Spanish
+        Language.SW: "swa",  # Swahili
+        Language.SV: "swe",  # Swedish
+        Language.TA: "tam",  # Tamil
+        Language.TG: "tgk",  # Tajik
+        Language.TE: "tel",  # Telugu
+        Language.TH: "tha",  # Thai
+        Language.TR: "tur",  # Turkish
+        Language.UK: "ukr",  # Ukrainian
+        Language.UMB: "umb",  # Umbundu
+        Language.UR: "urd",  # Urdu
+        Language.UZ: "uzb",  # Uzbek
+        Language.VI: "vie",  # Vietnamese
+        Language.CY: "cym",  # Welsh
+        Language.WO: "wol",  # Wolof
+        Language.XH: "xho",  # Xhosa
+        Language.ZU: "zul",  # Zulu
     }
-    return language_map.get(language)
 
+    result = BASE_LANGUAGES.get(language)
 
-class ElevenlabsSTTService(SegmentedSTTService):
-    """ElevenLabs speech-to-text service using Scribe v1 model.
+    # If not found in base languages, try to find the base language from a variant
+    if not result:
+        lang_str = str(language.value)
+        base_code = lang_str.split("-")[0].lower()
+        result = base_code if base_code in BASE_LANGUAGES.values() else None
 
-    This service uses ElevenLabs' batch STT API to transcribe audio segments.
-    It extends SegmentedSTTService to handle VAD-based audio segmentation.
+    return result
 
-    Args:
-        api_key: ElevenLabs API key for authentication.
-        model_id: Model to use for transcription (default: "scribe_v1").
-        language: Default language for transcription.
-        tag_audio_events: Whether to tag audio events like laughter (default: False).
-        diarize: Whether to enable speaker diarization (default: False).
-        **kwargs: Additional arguments passed to SegmentedSTTService.
+
+class ElevenLabsSTTService(SegmentedSTTService):
+    """Speech-to-text service using ElevenLabs' file-based API.
+
+    This service uses ElevenLabs' Speech-to-Text API to perform transcription on audio
+    segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
+    The service uploads audio files to ElevenLabs and receives transcription results directly.
     """
 
+    class InputParams(BaseModel):
+        """Configuration parameters for ElevenLabs STT API.
+
+        Parameters:
+            language: Target language for transcription.
+            tag_audio_events: Whether to include audio events like (laughter), (coughing), in the transcription.
+        """
+
+        language: Optional[Language] = None
+        tag_audio_events: bool = True
+
     def __init__(
         self,
         *,
         api_key: str,
-        model_id: str = "scribe_v1",
-        language: Optional[Language] = None,
-        tag_audio_events: bool = False,
+        aiohttp_session: aiohttp.ClientSession,
+        base_url: str = "https://api.elevenlabs.io",
+        model: str = "scribe_v1",
         sample_rate: Optional[int] = None,
-        diarize: bool = False,
+        params: Optional[InputParams] = None,
         **kwargs,
     ):
-        super().__init__(**kwargs)
+        """Initialize the ElevenLabs STT service.
+
+        Args:
+            api_key: ElevenLabs API key for authentication.
+            aiohttp_session: aiohttp ClientSession for HTTP requests.
+            base_url: Base URL for ElevenLabs API.
+            model: Model ID for transcription. Defaults to "scribe_v1".
+            sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
+            params: Configuration parameters for the STT service.
+            **kwargs: Additional arguments passed to SegmentedSTTService.
+        """
+        super().__init__(
+            sample_rate=sample_rate,
+            **kwargs,
+        )
 
-        self._client = ElevenLabs(api_key=api_key)
-        self._model_id = model_id
-        self._tag_audio_events = tag_audio_events
-        self._diarize = diarize
+        params = params or ElevenLabsSTTService.InputParams()
+
+        self._api_key = api_key
+        self._base_url = base_url
+        self._session = aiohttp_session
+        self._model_id = model
+        self._tag_audio_events = params.tag_audio_events
 
         self._settings = {
-            "language": language,
-            "model_id": self._model_id,
-            "tag_audio_events": self._tag_audio_events,
-            "diarize": self._diarize,
+            "language": self.language_to_service_language(params.language)
+            if params.language
+            else "eng",
         }
-        self.set_model_name(model_id)
 
     def can_generate_metrics(self) -> bool:
-        """Check if this service can generate processing metrics.
+        """Check if the service can generate processing metrics.
 
         Returns:
-            True, as ElevenLabs service supports metrics generation.
+            True, as ElevenLabs STT service supports metrics generation.
         """
         return True
 
     def language_to_service_language(self, language: Language) -> Optional[str]:
-        """Convert from pipecat Language to ElevenLabs language code.
+        """Convert a Language enum to ElevenLabs service-specific language code.
 
         Args:
-            language: The Language enum value to convert.
+            language: The language to convert.
 
         Returns:
-            str or None: The corresponding ElevenLabs language code, or None if not supported.
+            The ElevenLabs-specific language code, or None if not supported.
         """
         return language_to_elevenlabs_language(language)
 
     async def set_language(self, language: Language):
-        """Set the language for transcription.
+        """Set the transcription language.
 
         Args:
-            language: The Language enum value to use for transcription.
+            language: The language to use for speech-to-text transcription.
         """
         self.logger.info(f"Switching STT language to: [{language}]")
-        self._settings["language"] = language
+        self._settings["language"] = self.language_to_service_language(language)
+
+    async def set_model(self, model: str):
+        """Set the STT model.
+
+        Args:
+            model: The model name to use for transcription.
+
+        Note:
+            ElevenLabs STT API does not currently support model selection.
+            This method is provided for interface compatibility.
+        """
+        await super().set_model(model)
+        self.logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model")
+
+    async def _transcribe_audio(self, audio_data: bytes) -> dict:
+        """Upload audio data to ElevenLabs and get transcription result.
+
+        Args:
+            audio_data: Raw audio bytes in WAV format.
+
+        Returns:
+            The transcription result data.
+
+        Raises:
+            Exception: If transcription fails or returns an error.
+        """
+        url = f"{self._base_url}/v1/speech-to-text"
+        headers = {"xi-api-key": self._api_key}
+
+        # Create form data with the audio file
+        data = aiohttp.FormData()
+        data.add_field(
+            "file",
+            io.BytesIO(audio_data),
+            filename="audio.wav",
+            content_type="audio/x-wav",
+        )
+
+        # Add required model_id, language_code, and tag_audio_events
+        data.add_field("model_id", self._model_id)
+        data.add_field("language_code", self._settings["language"])
+        data.add_field("tag_audio_events", str(self._tag_audio_events).lower())
+
+        async with self._session.post(url, data=data, headers=headers) as response:
+            if response.status != 200:
+                error_text = await response.text()
+                self.logger.error(f"ElevenLabs transcription error: {error_text}")
+                raise Exception(f"Transcription failed with status {response.status}: {error_text}")
+
+            result = await response.json()
+            return result
 
     @traced_stt
     async def _handle_transcription(
-        self, transcript: str, is_final: bool, language: Optional[Language] = None
+        self, transcript: str, is_final: bool, language: Optional[str] = None
     ):
         """Handle a transcription result with tracing."""
-        pass
+        await self.stop_ttfb_metrics()
+        await self.stop_processing_metrics()
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
-        """Transcribe the provided audio using ElevenLabs STT.
+        """Transcribe an audio segment using ElevenLabs' STT API.
 
         Args:
-            audio: Audio data (WAV format) to transcribe.
+            audio: Raw audio bytes in WAV format (already converted by base class).
 
         Yields:
-            Frame: TranscriptionFrame containing the transcribed text or ErrorFrame on failure.
+            Frame: TranscriptionFrame containing the transcribed text, or ErrorFrame on failure.
+
+        Note:
+            The audio is already in WAV format from the SegmentedSTTService.
+            Only non-empty transcriptions are yielded.
         """
         try:
             await self.start_processing_metrics()
             await self.start_ttfb_metrics()
 
-            # Get language code for ElevenLabs API
-            params = {
-                "file": audio,
-                "model_id": self._model_id,
-                "tag_audio_events": self._tag_audio_events,
-                "diarize": self._diarize,
-            }
-
-            language = self._settings["language"]
-            if language is not None:
-                elevenlabs_lang = self.language_to_service_language(language)
-                if elevenlabs_lang:
-                    params["language_code"] = elevenlabs_lang
-            else:
-                params["language_code"] = None
-
-            # Call ElevenLabs STT API in thread pool to avoid blocking
-            transcription = await asyncio.to_thread(self._client.speech_to_text.convert, **params)
-
-            await self.stop_ttfb_metrics()
-
-            # Process transcription result
-            if transcription and hasattr(transcription, "text") and transcription.text:
-                transcript_text = transcription.text.strip()
-
-                if transcript_text:
-                    # Determine language if available from response
-                    response_language = language
-                    if hasattr(transcription, "language_code") and transcription.language_code:
-                        # Try to map back from ElevenLabs language code to pipecat Language
-                        try:
-                            # This is a simplified mapping - you might want to create a reverse map
-                            response_language = language  # For now, keep the original
-                        except ValueError:
-                            self.logger.warning(
-                                f"Unknown language detected: {transcription.language_code}"
-                            )
-
-                    # Handle transcription with tracing
-                    await self._handle_transcription(transcript_text, True, response_language)
-
-                    self.logger.debug(f"ElevenLabs transcription: [{transcript_text}]")
-
-                    yield TranscriptionFrame(
-                        text=transcript_text,
-                        user_id="",
-                        timestamp=time_now_iso8601(),
-                        language=response_language,
-                        result=transcription,
-                    )
-
-            await self.stop_processing_metrics()
+            # Upload audio and get transcription result directly
+            result = await self._transcribe_audio(audio)
+
+            # Extract transcription text
+            text = result.get("text", "").strip()
+            if text:
+                # Use the language_code returned by the API
+                detected_language = result.get("language_code", "eng")
+
+                await self._handle_transcription(text, True, detected_language)
+                self.logger.debug(f"Transcription: [{text}]")
+
+                yield TranscriptionFrame(
+                    text,
+                    self._user_id,
+                    time_now_iso8601(),
+                    detected_language,
+                    result=result,
+                )
 
         except Exception as e:
             self.logger.error(f"ElevenLabs STT error: {e}")
-            await self.stop_all_metrics()
             yield ErrorFrame(f"ElevenLabs STT error: {str(e)}")

pipecat/services/elevenlabs/tts.py

@@ -25,9 +25,9 @@ from pipecat.frames.frames import (
     EndFrame,
     ErrorFrame,
     Frame,
+    InterruptionFrame,
     LLMFullResponseEndFrame,
     StartFrame,
-    StartInterruptionFrame,
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
@@ -465,7 +465,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
             direction: The direction to push the frame.
         """
         await super().push_frame(frame, direction)
-        if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
+        if isinstance(frame, (TTSStoppedFrame, InterruptionFrame)):
             self._started = False
             if isinstance(frame, TTSStoppedFrame):
                 await self.add_word_timestamps([("Reset", 0)])
@@ -550,7 +550,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
             return self._websocket
         raise Exception("Websocket not connected")
 
-    async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
+    async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
         """Handle interruption by closing the current context."""
         await super()._handle_interruption(frame, direction)
 
@@ -559,7 +559,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
             logger.trace(f"Closing context {self._context_id} due to interruption")
             try:
                 # ElevenLabs requires that Pipecat manages the contexts and closes them
-                # when they're not longer in use. Since a StartInterruptionFrame is pushed
+                # when they're not longer in use. Since an InterruptionFrame is pushed
                 # every time the user speaks, we'll use this as a trigger to close the context
                 # and reset the state.
                 # Note: We do not need to call remove_audio_context here, as the context is
@@ -858,7 +858,7 @@ class ElevenLabsHttpTTSService(WordTTSService):
             direction: The direction to push the frame.
         """
         await super().push_frame(frame, direction)
-        if isinstance(frame, (StartInterruptionFrame, TTSStoppedFrame)):
+        if isinstance(frame, (InterruptionFrame, TTSStoppedFrame)):
             # Reset timing on interruption or stop
             self._reset_state()