dv-pipecat-ai 0.0.74.dev770__py3-none-any.whl → 0.0.82.dev776__py3-none-any.whl

pipecat/services/riva/tts.py

@@ -4,6 +4,12 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""NVIDIA Riva text-to-speech service implementation.
+
+This module provides integration with NVIDIA Riva's TTS services through
+gRPC API for high-quality speech synthesis.
+"""
+
 import asyncio
 import os
 from typing import AsyncGenerator, Mapping, Optional
@@ -37,7 +43,21 @@ RIVA_TTS_TIMEOUT_SECS = 5
 
 
 class RivaTTSService(TTSService):
+    """NVIDIA Riva text-to-speech service.
+
+    Provides high-quality text-to-speech synthesis using NVIDIA Riva's
+    cloud-based TTS models. Supports multiple voices, languages, and
+    configurable quality settings.
+    """
+
     class InputParams(BaseModel):
+        """Input parameters for Riva TTS configuration.
+
+        Parameters:
+            language: Language code for synthesis. Defaults to US English.
+            quality: Audio quality setting (0-100). Defaults to 20.
+        """
+
         language: Optional[Language] = Language.EN_US
         quality: Optional[int] = 20
 
@@ -55,6 +75,17 @@ class RivaTTSService(TTSService):
         params: Optional[InputParams] = None,
         **kwargs,
     ):
+        """Initialize the NVIDIA Riva TTS service.
+
+        Args:
+            api_key: NVIDIA API key for authentication.
+            server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint.
+            voice_id: Voice model identifier. Defaults to multilingual Ray voice.
+            sample_rate: Audio sample rate. If None, uses service default.
+            model_function_map: Dictionary containing function_id and model_name for the TTS model.
+            params: Additional configuration parameters for TTS synthesis.
+            **kwargs: Additional arguments passed to parent TTSService.
+        """
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         params = params or RivaTTSService.InputParams()
@@ -82,6 +113,13 @@ class RivaTTSService(TTSService):
         )
 
     async def set_model(self, model: str):
+        """Attempt to set the TTS model.
+
+        Note: Model cannot be changed after initialization for Riva service.
+
+        Args:
+            model: The model name to set (operation not supported).
+        """
         logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
         example = {"function_id": "<UUID>", "model_name": "<model_name>"}
         logger.warning(
@@ -90,6 +128,15 @@ class RivaTTSService(TTSService):
 
     @traced_tts
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        """Generate speech from text using NVIDIA Riva TTS.
+
+        Args:
+            text: The text to synthesize into speech.
+
+        Yields:
+            Frame: Audio frames containing the synthesized speech data.
+        """
+
         def read_audio_responses(queue: asyncio.Queue):
             def add_response(r):
                 asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop())
@@ -121,7 +168,7 @@ class RivaTTSService(TTSService):
             await asyncio.to_thread(read_audio_responses, queue)
 
             # Wait for the thread to start.
-            resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS)
+            resp = await asyncio.wait_for(queue.get(), timeout=RIVA_TTS_TIMEOUT_SECS)
             while resp:
                 await self.stop_ttfb_metrics()
                 frame = TTSAudioRawFrame(
@@ -130,7 +177,7 @@ class RivaTTSService(TTSService):
                     num_channels=1,
                 )
                 yield frame
-                resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS)
+                resp = await asyncio.wait_for(queue.get(), timeout=RIVA_TTS_TIMEOUT_SECS)
         except asyncio.TimeoutError:
             logger.error(f"{self} timeout waiting for audio response")
 
@@ -139,6 +186,13 @@ class RivaTTSService(TTSService):
 
 
 class FastPitchTTSService(RivaTTSService):
+    """Deprecated FastPitch TTS service.
+
+    .. deprecated:: 0.0.66
+        This class is deprecated. Use RivaTTSService instead for new implementations.
+        Provides backward compatibility for existing FastPitch TTS integrations.
+    """
+
     def __init__(
         self,
         *,
@@ -153,6 +207,17 @@ class FastPitchTTSService(RivaTTSService):
         params: Optional[RivaTTSService.InputParams] = None,
         **kwargs,
     ):
+        """Initialize the deprecated FastPitch TTS service.
+
+        Args:
+            api_key: NVIDIA API key for authentication.
+            server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint.
+            voice_id: Voice model identifier. Defaults to Female-1 voice.
+            sample_rate: Audio sample rate. If None, uses service default.
+            model_function_map: Dictionary containing function_id and model_name for FastPitch model.
+            params: Additional configuration parameters for TTS synthesis.
+            **kwargs: Additional arguments passed to parent RivaTTSService.
+        """
         super().__init__(
             api_key=api_key,
             server=server,

pipecat/services/sambanova/llm.py

@@ -20,7 +20,6 @@ from pipecat.metrics.metrics import LLMTokenUsage
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.llm_service import FunctionCallFromLLM
 from pipecat.services.openai.llm import OpenAILLMService
-from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator
 from pipecat.utils.tracing.service_decorators import traced_llm
 
 
@@ -29,12 +28,6 @@ class SambaNovaLLMService(OpenAILLMService):  # type: ignore
 
     This service extends OpenAILLMService to connect to SambaNova's API endpoint while
     maintaining full compatibility with OpenAI's interface and functionality.
-
-    Args:
-        api_key: The API key for accessing SambaNova API.
-        model: The model identifier to use. Defaults to "Llama-4-Maverick-17B-128E-Instruct".
-        base_url: The base URL for SambaNova API. Defaults to "https://api.sambanova.ai/v1".
-        **kwargs: Additional keyword arguments passed to OpenAILLMService.
     """
 
     def __init__(
@@ -45,6 +38,14 @@ class SambaNovaLLMService(OpenAILLMService):  # type: ignore
         base_url: str = "https://api.sambanova.ai/v1",
         **kwargs: Dict[Any, Any],
     ) -> None:
+        """Initialize SambaNova LLM service.
+
+        Args:
+            api_key: The API key for accessing SambaNova API.
+            model: The model identifier to use. Defaults to "Llama-4-Maverick-17B-128E-Instruct".
+            base_url: The base URL for SambaNova API. Defaults to "https://api.sambanova.ai/v1".
+            **kwargs: Additional keyword arguments passed to OpenAILLMService.
+        """
         super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
 
     def create_client(
@@ -66,17 +67,20 @@ class SambaNovaLLMService(OpenAILLMService):  # type: ignore
         logger.debug(f"Creating SambaNova client with API {base_url}")
         return super().create_client(api_key, base_url, **kwargs)
 
-    async def get_chat_completions(
+    def build_chat_completion_params(
         self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
-    ) -> Any:
-        """Get chat completions from SambaNova API endpoint.
+    ) -> dict:
+        """Build parameters for SambaNova chat completion request.
+
+        SambaNova doesn't support some OpenAI parameters like frequency_penalty,
+        presence_penalty, and seed.
 
         Args:
-            context: OpenAI LLM context containing tools and configuration.
-            messages: List of chat completion message parameters.
+            context: The LLM context containing tools and configuration.
+            messages: List of chat completion messages to send.
 
         Returns:
-            Chat completion response stream from SambaNova API.
+            Dictionary of parameters for the chat completion request.
         """
         params = {
             "model": self.model_name,
@@ -92,9 +96,7 @@ class SambaNovaLLMService(OpenAILLMService):  # type: ignore
         }
 
         params.update(self._settings["extra"])
-
-        chunks = await self._client.chat.completions.create(**params)
-        return chunks
+        return params
 
     @traced_llm  # type: ignore
     async def _process_context(self, context: OpenAILLMContext) -> AsyncStream[ChatCompletionChunk]:
@@ -124,7 +126,7 @@ class SambaNovaLLMService(OpenAILLMService):  # type: ignore
             context
         )
 
-        async for chunk in WatchdogAsyncIterator(chunk_stream, manager=self.task_manager):
+        async for chunk in chunk_stream:
             if chunk.usage:
                 tokens = LLMTokenUsage(
                     prompt_tokens=chunk.usage.prompt_tokens,

pipecat/services/sambanova/stt.py

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""SambaNova's Speech-to-Text service implementation for real-time transcription."""
+
 from typing import Any, Optional
 
 from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription
@@ -12,16 +14,9 @@ from pipecat.transcriptions.language import Language
 
 class SambaNovaSTTService(BaseWhisperSTTService):  # type: ignore
     """SambaNova Whisper speech-to-text service.
+
     Uses SambaNova's Whisper API to convert audio to text.
     Requires a SambaNova API key set via the api_key parameter or SAMBANOVA_API_KEY environment variable.
-    Args:
-        model: Whisper model to use. Defaults to "Whisper-Large-v3".
-        api_key: SambaNova API key. Defaults to None.
-        base_url: API base URL. Defaults to "https://api.sambanova.ai/v1".
-        language: Language of the audio input. Defaults to English.
-        prompt: Optional text to guide the model's style or continue a previous segment.
-        temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
-        **kwargs: Additional arguments passed to `pipecat.services.whisper.base_stt.BaseWhisperSTTService`.
     """
 
     def __init__(
@@ -35,6 +30,17 @@ class SambaNovaSTTService(BaseWhisperSTTService):  # type: ignore
         temperature: Optional[float] = None,
         **kwargs: Any,
     ) -> None:
+        """Initialize SambaNova STT service.
+
+        Args:
+            model: Whisper model to use. Defaults to "Whisper-Large-v3".
+            api_key: SambaNova API key. Defaults to None.
+            base_url: API base URL. Defaults to "https://api.sambanova.ai/v1".
+            language: Language of the audio input. Defaults to English.
+            prompt: Optional text to guide the model's style or continue a previous segment.
+            temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
+            **kwargs: Additional arguments passed to `pipecat.services.whisper.base_stt.BaseWhisperSTTService`.
+        """
         super().__init__(
             model=model,
             api_key=api_key,

pipecat/services/sarvam/tts.py

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Sarvam AI text-to-speech service implementation."""
+
 import base64
 from typing import AsyncGenerator, Optional
 
@@ -25,7 +27,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts
 
 
 def language_to_sarvam_language(language: Language) -> Optional[str]:
-    """Convert Pipecat Language enum to Sarvam AI language codes."""
+    """Convert Pipecat Language enum to Sarvam AI language codes.
+
+    Args:
+        language: The Language enum value to convert.
+
+    Returns:
+        The corresponding Sarvam AI language code, or None if not supported.
+    """
     LANGUAGE_MAP = {
         Language.BN: "bn-IN",  # Bengali
         Language.EN: "en-IN",  # English (India)
@@ -50,17 +59,8 @@ class SarvamTTSService(TTSService):
     Indian languages. Provides control over voice characteristics like pitch, pace,
     and loudness.
 
-    Args:
-        api_key: Sarvam AI API subscription key.
-        voice_id: Speaker voice ID (e.g., "anushka", "meera").
-        model: TTS model to use ("bulbul:v1" or "bulbul:v2").
-        aiohttp_session: Shared aiohttp session for making requests.
-        base_url: Sarvam AI API base URL.
-        sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000).
-        params: Additional voice and preprocessing parameters.
-
-    Example:
-        ```python
+    Example::
+
         tts = SarvamTTSService(
             api_key="your-api-key",
             voice_id="anushka",
@@ -72,10 +72,19 @@ class SarvamTTSService(TTSService):
                 pace=1.2
             )
         )
-        ```
     """
 
     class InputParams(BaseModel):
+        """Input parameters for Sarvam TTS configuration.
+
+        Parameters:
+            language: Language for synthesis. Defaults to English (India).
+            pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0.
+            pace: Speech pace multiplier (0.3 to 3.0). Defaults to 1.0.
+            loudness: Volume multiplier (0.1 to 3.0). Defaults to 1.0.
+            enable_preprocessing: Whether to enable text preprocessing. Defaults to False.
+        """
+
         language: Optional[Language] = Language.EN
         pitch: Optional[float] = Field(default=0.0, ge=-0.75, le=0.75)
         pace: Optional[float] = Field(default=1.0, ge=0.3, le=3.0)
@@ -94,6 +103,18 @@ class SarvamTTSService(TTSService):
         params: Optional[InputParams] = None,
         **kwargs,
     ):
+        """Initialize the Sarvam TTS service.
+
+        Args:
+            api_key: Sarvam AI API subscription key.
+            voice_id: Speaker voice ID (e.g., "anushka", "meera"). Defaults to "anushka".
+            model: TTS model to use ("bulbul:v1" or "bulbul:v2"). Defaults to "bulbul:v2".
+            aiohttp_session: Shared aiohttp session for making requests.
+            base_url: Sarvam AI API base URL. Defaults to "https://api.sarvam.ai".
+            sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000). If None, uses default.
+            params: Additional voice and preprocessing parameters. If None, uses defaults.
+            **kwargs: Additional arguments passed to parent TTSService.
+        """
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         params = params or SarvamTTSService.InputParams()
@@ -116,17 +137,43 @@ class SarvamTTSService(TTSService):
         self.set_voice(voice_id)
 
     def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Sarvam service supports metrics generation.
+        """
         return True
 
     def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert a Language enum to Sarvam AI language format.
+
+        Args:
+            language: The language to convert.
+
+        Returns:
+            The Sarvam AI-specific language code, or None if not supported.
+        """
         return language_to_sarvam_language(language)
 
     async def start(self, frame: StartFrame):
+        """Start the Sarvam TTS service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
         await super().start(frame)
         self._settings["sample_rate"] = self.sample_rate
 
     @traced_tts
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        """Generate speech from text using Sarvam AI's API.
+
+        Args:
+            text: The text to synthesize into speech.
+
+        Yields:
+            Frame: Audio frames containing the synthesized speech.
+        """
         logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:

pipecat/services/simli/video.py

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Simli video service for real-time avatar generation."""
+
 import asyncio
 
 import numpy as np
@@ -16,9 +18,10 @@ from pipecat.frames.frames import (
     OutputImageRawFrame,
     StartInterruptionFrame,
     TTSAudioRawFrame,
+    TTSStoppedFrame,
+    UserStartedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, StartFrame
-from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator
 
 try:
     from av.audio.frame import AudioFrame
@@ -31,39 +34,68 @@ except ModuleNotFoundError as e:
 
 
 class SimliVideoService(FrameProcessor):
+    """Simli video service for real-time avatar generation.
+
+    Provides real-time avatar video generation by processing audio frames
+    and producing synchronized video output using the Simli API. Handles
+    audio resampling, video frame processing, and connection management.
+    """
+
     def __init__(
         self,
         simli_config: SimliConfig,
         use_turn_server: bool = False,
         latency_interval: int = 0,
+        simli_url: str = "https://api.simli.ai",
+        is_trinity_avatar: bool = False,
     ):
+        """Initialize the Simli video service.
+
+        Args:
+            simli_config: Configuration object for Simli client settings.
+            use_turn_server: Whether to use TURN server for connection. Defaults to False.
+            latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0.
+            simli_url: URL of the simli servers. Can be changed for custom deployments of enterprise users.
+            is_trinity_avatar: boolean to tell simli client that this is a Trinity avatar which reduces latency when using Trinity.
+
+        """
         super().__init__()
-        self._simli_client = SimliClient(simli_config, use_turn_server, latency_interval)
+        self._initialized = False
+        simli_config.maxIdleTime += 5
+        simli_config.maxSessionLength += 5
+        self._simli_client = SimliClient(
+            simli_config,
+            use_turn_server,
+            latency_interval,
+            simliURL=simli_url,
+        )
 
-        self._pipecat_resampler_event = asyncio.Event()
         self._pipecat_resampler: AudioResampler = None
+        self._pipecat_resampler_event = asyncio.Event()
         self._simli_resampler = AudioResampler("s16", "mono", 16000)
 
-        self._initialized = False
         self._audio_task: asyncio.Task = None
         self._video_task: asyncio.Task = None
+        self._is_trinity_avatar = is_trinity_avatar
+        self._previously_interrupted = is_trinity_avatar
+        self._audio_buffer = bytearray()
 
     async def _start_connection(self):
+        """Start the connection to Simli service and begin processing tasks."""
         if not self._initialized:
             await self._simli_client.Initialize()
             self._initialized = True
 
         # Create task to consume and process audio and video
-        if not self._audio_task:
-            self._audio_task = self.create_task(self._consume_and_process_audio())
-
-        if not self._video_task:
-            self._video_task = self.create_task(self._consume_and_process_video())
+        await self._simli_client.sendSilence()
+        self._audio_task = self.create_task(self._consume_and_process_audio())
+        self._video_task = self.create_task(self._consume_and_process_video())
 
     async def _consume_and_process_audio(self):
+        """Consume audio frames from Simli and push them downstream."""
         await self._pipecat_resampler_event.wait()
         audio_iterator = self._simli_client.getAudioStreamIterator()
-        async for audio_frame in WatchdogAsyncIterator(audio_iterator, manager=self.task_manager):
+        async for audio_frame in audio_iterator:
             resampled_frames = self._pipecat_resampler.resample(audio_frame)
             for resampled_frame in resampled_frames:
                 audio_array = resampled_frame.to_ndarray()
@@ -78,9 +110,10 @@ class SimliVideoService(FrameProcessor):
                     )
 
     async def _consume_and_process_video(self):
+        """Consume video frames from Simli and convert them to output frames."""
         await self._pipecat_resampler_event.wait()
         video_iterator = self._simli_client.getVideoStreamIterator(targetFormat="rgb24")
-        async for video_frame in WatchdogAsyncIterator(video_iterator, manager=self.task_manager):
+        async for video_frame in video_iterator:
             # Process the video frame
             convertedFrame: OutputImageRawFrame = OutputImageRawFrame(
                 image=video_frame.to_rgb().to_image().tobytes(),
@@ -91,9 +124,14 @@ class SimliVideoService(FrameProcessor):
             await self.push_frame(convertedFrame)
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and handle Simli video generation.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame processing.
+        """
         await super().process_frame(frame, direction)
         if isinstance(frame, StartFrame):
-            await self.push_frame(frame, direction)
             await self._start_connection()
         elif isinstance(frame, TTSAudioRawFrame):
             # Send audio frame to Simli
@@ -112,21 +150,44 @@ class SimliVideoService(FrameProcessor):
 
                 resampled_frames = self._simli_resampler.resample(old_frame)
                 for resampled_frame in resampled_frames:
-                    await self._simli_client.send(
-                        resampled_frame.to_ndarray().astype(np.int16).tobytes()
-                    )
+                    audioBytes = resampled_frame.to_ndarray().astype(np.int16).tobytes()
+                    if self._previously_interrupted:
+                        self._audio_buffer.extend(audioBytes)
+                        if len(self._audio_buffer) >= 128000:
+                            try:
+                                for flushFrame in self._simli_resampler.resample(None):
+                                    self._audio_buffer.extend(
+                                        flushFrame.to_ndarray().astype(np.int16).tobytes()
+                                    )
+                            finally:
+                                await self._simli_client.playImmediate(self._audio_buffer)
+                                self._previously_interrupted = False
+                                self._audio_buffer = bytearray()
+                    else:
+                        await self._simli_client.send(audioBytes)
+                return
             except Exception as e:
                 logger.exception(f"{self} exception: {e}")
+        elif isinstance(frame, TTSStoppedFrame):
+            try:
+                if self._previously_interrupted and len(self._audio_buffer) > 0:
+                    await self._simli_client.playImmediate(self._audio_buffer)
+                    self._previously_interrupted = False
+                    self._audio_buffer = bytearray()
+            except Exception as e:
+                logger.exception(f"{self} exception: {e}")
+            return
         elif isinstance(frame, (EndFrame, CancelFrame)):
             await self._stop()
-            await self.push_frame(frame, direction)
-        elif isinstance(frame, StartInterruptionFrame):
-            await self._simli_client.clearBuffer()
-            await self.push_frame(frame, direction)
-        else:
-            await self.push_frame(frame, direction)
+        elif isinstance(frame, (StartInterruptionFrame, UserStartedSpeakingFrame)):
+            if not self._previously_interrupted:
+                await self._simli_client.clearBuffer()
+            self._previously_interrupted = self._is_trinity_avatar
+
+        await self.push_frame(frame, direction)
 
     async def _stop(self):
+        """Stop the Simli client and cancel processing tasks."""
         await self._simli_client.stop()
         if self._audio_task:
             await self.cancel_task(self._audio_task)