dv-pipecat-ai 0.0.74.dev770__py3-none-any.whl → 0.0.82.dev776__py3-none-any.whl

pipecat/services/tavus/video.py

@@ -4,7 +4,11 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-"""This module implements Tavus as a sink transport layer"""
+"""Tavus video service implementation for avatar-based video generation.
+
+This module implements Tavus as a sink transport layer, providing video
+avatar functionality through Tavus's streaming API.
+"""
 
 import asyncio
 from typing import Optional
@@ -13,41 +17,37 @@ import aiohttp
 from daily.daily import AudioData, VideoFrame
 from loguru import logger
 
-from pipecat.audio.utils import create_default_resampler
+from pipecat.audio.utils import create_stream_resampler
 from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
     CancelFrame,
     EndFrame,
     Frame,
     OutputAudioRawFrame,
     OutputImageRawFrame,
+    OutputTransportReadyFrame,
+    SpeechOutputAudioRawFrame,
     StartFrame,
     StartInterruptionFrame,
     TTSAudioRawFrame,
+    TTSStartedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup
 from pipecat.services.ai_service import AIService
 from pipecat.transports.services.tavus import TavusCallbacks, TavusParams, TavusTransportClient
-from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
 
 
 class TavusVideoService(AIService):
-    """
-    Service class that proxies audio to Tavus and receives both audio and video in return.
-
-    It uses the `TavusTransportClient` to manage the session and handle communication. When
-    audio is sent, Tavus responds with both audio and video streams, which are then routed
-    through Pipecat’s media pipeline.
-
-    In use cases such as with `DailyTransport`, this results in two distinct virtual rooms:
-        - **Tavus room**: Contains the Tavus Avatar and the Pipecat Bot.
-        - **User room**: Contains the Pipecat Bot and the user.
-
-    Args:
-        api_key (str): Tavus API key used for authentication.
-        replica_id (str): ID of the Tavus voice replica to use for speech synthesis.
-        persona_id (str): ID of the Tavus persona. Defaults to "pipecat-stream" to use the Pipecat TTS voice.
-        session (aiohttp.ClientSession): Async HTTP session used for communication with Tavus.
-        **kwargs: Additional arguments passed to the parent `AIService` class.
+    """Service that proxies audio to Tavus and receives audio and video in return.
+
+    Uses the TavusTransportClient to manage sessions and handle communication.
+    When audio is sent, Tavus responds with both audio and video streams, which
+    are routed through Pipecat's media pipeline.
+
+    In use cases with DailyTransport, this creates two distinct virtual rooms:
+
+    - Tavus room: Contains the Tavus Avatar and the Pipecat Bot
+    - User room: Contains the Pipecat Bot and the user
     """
 
     def __init__(
@@ -59,6 +59,15 @@ class TavusVideoService(AIService):
         session: aiohttp.ClientSession,
         **kwargs,
     ) -> None:
+        """Initialize the Tavus video service.
+
+        Args:
+            api_key: Tavus API key used for authentication.
+            replica_id: ID of the Tavus voice replica to use for speech synthesis.
+            persona_id: ID of the Tavus persona. Defaults to "pipecat-stream" for Pipecat TTS voice.
+            session: Async HTTP session used for communication with Tavus.
+            **kwargs: Additional arguments passed to the parent AIService class.
+        """
         super().__init__(**kwargs)
         self._api_key = api_key
         self._session = session
@@ -69,14 +78,20 @@ class TavusVideoService(AIService):
         self._client: Optional[TavusTransportClient] = None
 
         self._conversation_id: str
-        self._resampler = create_default_resampler()
+        self._resampler = create_stream_resampler()
 
         self._audio_buffer = bytearray()
         self._send_task: Optional[asyncio.Task] = None
         # This is the custom track destination expected by Tavus
         self._transport_destination: Optional[str] = "stream"
+        self._transport_ready = False
 
     async def setup(self, setup: FrameProcessorSetup):
+        """Set up the Tavus video service.
+
+        Args:
+            setup: Frame processor setup configuration.
+        """
         await super().setup(setup)
         callbacks = TavusCallbacks(
             on_participant_joined=self._on_participant_joined,
@@ -99,15 +114,18 @@ class TavusVideoService(AIService):
         await self._client.setup(setup)
 
     async def cleanup(self):
+        """Clean up the service and release resources."""
         await super().cleanup()
         await self._client.cleanup()
         self._client = None
 
     async def _on_participant_left(self, participant, reason):
+        """Handle participant leaving the session."""
         participant_id = participant["id"]
         logger.info(f"Participant left {participant_id}, reason: {reason}")
 
     async def _on_participant_joined(self, participant):
+        """Handle participant joining the session."""
         participant_id = participant["id"]
         logger.info(f"Participant joined {participant_id}")
         if not self._other_participant_has_joined:
@@ -124,32 +142,51 @@ class TavusVideoService(AIService):
     async def _on_participant_video_frame(
         self, participant_id: str, video_frame: VideoFrame, video_source: str
     ):
+        """Handle incoming video frames from participants."""
         frame = OutputImageRawFrame(
             image=video_frame.buffer,
             size=(video_frame.width, video_frame.height),
             format=video_frame.color_format,
         )
         frame.transport_source = video_source
-        await self.push_frame(frame)
+        if self._transport_ready:
+            await self.push_frame(frame)
 
     async def _on_participant_audio_data(
         self, participant_id: str, audio: AudioData, audio_source: str
     ):
-        frame = OutputAudioRawFrame(
+        """Handle incoming audio data from participants."""
+        frame = SpeechOutputAudioRawFrame(
             audio=audio.audio_frames,
             sample_rate=audio.sample_rate,
             num_channels=audio.num_channels,
         )
         frame.transport_source = audio_source
-        await self.push_frame(frame)
+        if self._transport_ready:
+            await self.push_frame(frame)
 
     def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Tavus service supports metrics generation.
+        """
         return True
 
     async def get_persona_name(self) -> str:
+        """Get the name of the current persona.
+
+        Returns:
+            The persona name from the Tavus client.
+        """
         return await self._client.get_persona_name()
 
     async def start(self, frame: StartFrame):
+        """Start the Tavus video service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
         await super().start(frame)
         await self._client.start(frame)
         if self._transport_destination:
@@ -157,16 +194,32 @@ class TavusVideoService(AIService):
         await self._create_send_task()
 
     async def stop(self, frame: EndFrame):
+        """Stop the Tavus video service.
+
+        Args:
+            frame: The end frame.
+        """
         await super().stop(frame)
         await self._end_conversation()
         await self._cancel_send_task()
 
     async def cancel(self, frame: CancelFrame):
+        """Cancel the Tavus video service.
+
+        Args:
+            frame: The cancel frame.
+        """
         await super().cancel(frame)
         await self._end_conversation()
         await self._cancel_send_task()
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process frames through the service.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame processing.
+        """
         await super().process_frame(frame, direction)
 
         if isinstance(frame, StartInterruptionFrame):
@@ -174,29 +227,44 @@ class TavusVideoService(AIService):
             await self.push_frame(frame, direction)
         elif isinstance(frame, TTSAudioRawFrame):
             await self._handle_audio_frame(frame)
+        elif isinstance(frame, OutputTransportReadyFrame):
+            self._transport_ready = True
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, TTSStartedFrame):
+            await self.start_ttfb_metrics()
+        elif isinstance(frame, BotStartedSpeakingFrame):
+            # We constantly receive audio through WebRTC, but most of the time it is silence.
+            # As soon as we receive actual audio, the base output transport will create a
+            # BotStartedSpeakingFrame, which we can use as a signal for the TTFB metrics.
+            await self.stop_ttfb_metrics()
         else:
             await self.push_frame(frame, direction)
 
     async def _handle_interruptions(self):
+        """Handle interruption events by resetting send tasks and notifying client."""
         await self._cancel_send_task()
         await self._create_send_task()
         await self._client.send_interrupt_message()
 
     async def _end_conversation(self):
+        """End the current conversation and reset state."""
         await self._client.stop()
         self._other_participant_has_joined = False
 
     async def _create_send_task(self):
+        """Create the audio sending task if it doesn't exist."""
         if not self._send_task:
-            self._queue = WatchdogQueue(self.task_manager)
+            self._queue = asyncio.Queue()
             self._send_task = self.create_task(self._send_task_handler())
 
     async def _cancel_send_task(self):
+        """Cancel the audio sending task if it exists."""
         if self._send_task:
             await self.cancel_task(self._send_task)
             self._send_task = None
 
     async def _handle_audio_frame(self, frame: OutputAudioRawFrame):
+        """Process audio frames for sending to Tavus."""
         sample_rate = self._client.out_sample_rate
         # 40 ms of audio
         chunk_size = int((sample_rate * 2) / 25)
@@ -215,6 +283,7 @@ class TavusVideoService(AIService):
             self._audio_buffer = self._audio_buffer[chunk_size:]
 
     async def _send_task_handler(self):
+        """Handle sending audio frames to the Tavus client."""
         while True:
             frame = await self._queue.get()
             if isinstance(frame, OutputAudioRawFrame) and self._client:

pipecat/services/together/llm.py

@@ -16,12 +16,6 @@ class TogetherLLMService(OpenAILLMService):
 
     This service extends OpenAILLMService to connect to Together.ai's API endpoint while
     maintaining full compatibility with OpenAI's interface and functionality.
-
-    Args:
-        api_key: The API key for accessing Together.ai's API.
-        base_url: The base URL for Together.ai API. Defaults to "https://api.together.xyz/v1".
-        model: The model identifier to use. Defaults to "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo".
-        **kwargs: Additional keyword arguments passed to OpenAILLMService.
     """
 
     def __init__(
@@ -32,6 +26,14 @@ class TogetherLLMService(OpenAILLMService):
         model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
         **kwargs,
     ):
+        """Initialize Together.ai LLM service.
+
+        Args:
+            api_key: The API key for accessing Together.ai's API.
+            base_url: The base URL for Together.ai API. Defaults to "https://api.together.xyz/v1".
+            model: The model identifier to use. Defaults to "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo".
+            **kwargs: Additional keyword arguments passed to OpenAILLMService.
+        """
         super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
 
     def create_client(self, api_key=None, base_url=None, **kwargs):

pipecat/services/tts_service.py

@@ -37,7 +37,6 @@ from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_service import AIService
 from pipecat.services.websocket_service import WebsocketService
 from pipecat.transcriptions.language import Language
-from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
 from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
 from pipecat.utils.text.base_text_filter import BaseTextFilter
 from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator
@@ -50,21 +49,6 @@ class TTSService(AIService):
     Provides common functionality for TTS services including text aggregation,
     filtering, audio generation, and frame management. Supports configurable
     sentence aggregation, silence insertion, and frame processing control.
-
-    Args:
-        aggregate_sentences: Whether to aggregate text into sentences before synthesis.
-        push_text_frames: Whether to push TextFrames and LLMFullResponseEndFrames.
-        push_stop_frames: Whether to automatically push TTSStoppedFrames.
-        stop_frame_timeout_s: Idle time before pushing TTSStoppedFrame when push_stop_frames is True.
-        push_silence_after_stop: Whether to push silence audio after TTSStoppedFrame.
-        silence_time_s: Duration of silence to push when push_silence_after_stop is True.
-        pause_frame_processing: Whether to pause frame processing during audio generation.
-        sample_rate: Output sample rate for generated audio.
-        text_aggregator: Custom text aggregator for processing incoming text.
-        text_filters: Sequence of text filters to apply after aggregation.
-        text_filter: Single text filter (deprecated, use text_filters).
-        transport_destination: Destination for generated audio frames.
-        **kwargs: Additional arguments passed to the parent AIService.
     """
 
     def __init__(
@@ -97,6 +81,27 @@ class TTSService(AIService):
         transport_destination: Optional[str] = None,
         **kwargs,
     ):
+        """Initialize the TTS service.
+
+        Args:
+            aggregate_sentences: Whether to aggregate text into sentences before synthesis.
+            push_text_frames: Whether to push TextFrames and LLMFullResponseEndFrames.
+            push_stop_frames: Whether to automatically push TTSStoppedFrames.
+            stop_frame_timeout_s: Idle time before pushing TTSStoppedFrame when push_stop_frames is True.
+            push_silence_after_stop: Whether to push silence audio after TTSStoppedFrame.
+            silence_time_s: Duration of silence to push when push_silence_after_stop is True.
+            pause_frame_processing: Whether to pause frame processing during audio generation.
+            sample_rate: Output sample rate for generated audio.
+            text_aggregator: Custom text aggregator for processing incoming text.
+            text_filters: Sequence of text filters to apply after aggregation.
+            text_filter: Single text filter (deprecated, use text_filters).
+
+                .. deprecated:: 0.0.59
+                    Use `text_filters` instead, which allows multiple filters.
+
+            transport_destination: Destination for generated audio frames.
+            **kwargs: Additional arguments passed to the parent AIService.
+        """
         super().__init__(**kwargs)
         self._aggregate_sentences: bool = aggregate_sentences
         self._push_text_frames: bool = push_text_frames
@@ -112,9 +117,10 @@ class TTSService(AIService):
         self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator()
         self._text_filters: Sequence[BaseTextFilter] = text_filters or []
         self._transport_destination: Optional[str] = transport_destination
-
         self._tracing_enabled: bool = False
-
+        self._voice_config: Dict[str, Any] = {}
+        self._voice = None
+        self._voice_clone_params = None
 
         if text_filter:
             import warnings
@@ -225,6 +231,7 @@ class TTSService(AIService):
         self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
         if self._push_stop_frames and not self._stop_frame_task:
             self._stop_frame_task = self.create_task(self._stop_frame_handler())
+        self._tracing_enabled = frame.enable_tracing
 
     async def stop(self, frame: EndFrame):
         """Stop the TTS service.
@@ -257,7 +264,7 @@ class TTSService(AIService):
                     self._settings[key] = self.language_to_service_language(value)
             elif key == "model":
                 self.set_model_name(value)
-            elif key == "voice":
+            elif key == "voice" or key == "voice_id":
                 self.set_voice(value)
             elif key == "text_filter":
                 for filter in self._text_filters:
@@ -268,9 +275,20 @@ class TTSService(AIService):
     async def say(self, text: str):
         """Immediately speak the provided text.
 
+        .. deprecated:: 0.0.79
+            Push a `TTSSpeakFrame` instead to ensure frame ordering is maintained.
+
         Args:
             text: The text to speak.
         """
+        import warnings
+
+        warnings.warn(
+            "`TTSService.say()` is deprecated. Push a `TTSSpeakFrame` instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
         await self.queue_frame(TTSSpeakFrame(text))
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -433,7 +451,7 @@ class TTSService(AIService):
         while True:
             try:
                 frame = await asyncio.wait_for(
-                    self._stop_frame_queue.get(), self._stop_frame_timeout_s
+                    self._stop_frame_queue.get(), timeout=self._stop_frame_timeout_s
                 )
                 if isinstance(frame, TTSStartedFrame):
                     has_started = True
@@ -443,8 +461,6 @@ class TTSService(AIService):
                 if has_started:
                     await self.push_frame(TTSStoppedFrame())
                     has_started = False
-            finally:
-                self.reset_watchdog()
 
 
 class WordTTSService(TTSService):
@@ -452,12 +468,14 @@ class WordTTSService(TTSService):
 
     Word timestamps are useful to synchronize audio with text of the spoken
     words. This way only the spoken words are added to the conversation context.
-
-    Args:
-        **kwargs: Additional arguments passed to the parent TTSService.
     """
 
     def __init__(self, **kwargs):
+        """Initialize the Word TTS service.
+
+        Args:
+            **kwargs: Additional arguments passed to the parent TTSService.
+        """
         super().__init__(**kwargs)
         self._initial_word_timestamp = -1
         self._words_task = None
@@ -529,7 +547,7 @@ class WordTTSService(TTSService):
 
     def _create_words_task(self):
         if not self._words_task:
-            self._words_queue = WatchdogQueue(self.task_manager)
+            self._words_queue = asyncio.Queue()
             self._words_task = self.create_task(self._words_task_handler())
 
     async def _stop_words_task(self):
@@ -566,22 +584,23 @@ class WebsocketTTSService(TTSService, WebsocketService):
     Combines TTS functionality with websocket connectivity, providing automatic
     error handling and reconnection capabilities.
 
-    Args:
-        reconnect_on_error: Whether to automatically reconnect on websocket errors.
-        **kwargs: Additional arguments passed to parent classes.
-
     Event handlers:
         on_connection_error: Called when a websocket connection error occurs.
 
-    Example:
-        ```python
+    Example::
+
         @tts.event_handler("on_connection_error")
         async def on_connection_error(tts: TTSService, error: str):
             logger.error(f"TTS connection error: {error}")
-        ```
     """
 
     def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
+        """Initialize the Websocket TTS service.
+
+        Args:
+            reconnect_on_error: Whether to automatically reconnect on websocket errors.
+            **kwargs: Additional arguments passed to parent classes.
+        """
         TTSService.__init__(self, **kwargs)
         WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
         self._register_event_handler("on_connection_error")
@@ -596,12 +615,14 @@ class InterruptibleTTSService(WebsocketTTSService):
 
     Designed for TTS services that don't support word timestamps. Handles interruptions
     by reconnecting the websocket when the bot is speaking and gets interrupted.
-
-    Args:
-        **kwargs: Additional arguments passed to the parent WebsocketTTSService.
     """
 
     def __init__(self, **kwargs):
+        """Initialize the Interruptible TTS service.
+
+        Args:
+            **kwargs: Additional arguments passed to the parent WebsocketTTSService.
+        """
         super().__init__(**kwargs)
 
         # Indicates if the bot is speaking. If the bot is not speaking we don't
@@ -635,22 +656,23 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService):
 
     Combines word timestamp functionality with websocket connectivity.
 
-    Args:
-        reconnect_on_error: Whether to automatically reconnect on websocket errors.
-        **kwargs: Additional arguments passed to parent classes.
-
     Event handlers:
         on_connection_error: Called when a websocket connection error occurs.
 
-    Example:
-        ```python
+    Example::
+
         @tts.event_handler("on_connection_error")
         async def on_connection_error(tts: TTSService, error: str):
             logger.error(f"TTS connection error: {error}")
-        ```
     """
 
     def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
+        """Initialize the Websocket Word TTS service.
+
+        Args:
+            reconnect_on_error: Whether to automatically reconnect on websocket errors.
+            **kwargs: Additional arguments passed to parent classes.
+        """
         WordTTSService.__init__(self, **kwargs)
         WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
         self._register_event_handler("on_connection_error")
@@ -665,12 +687,14 @@ class InterruptibleWordTTSService(WebsocketWordTTSService):
 
     For TTS services that support word timestamps but can't correlate generated
     audio with requested text. Handles interruptions by reconnecting when needed.
-
-    Args:
-        **kwargs: Additional arguments passed to the parent WebsocketWordTTSService.
     """
 
     def __init__(self, **kwargs):
+        """Initialize the Interruptible Word TTS service.
+
+        Args:
+            **kwargs: Additional arguments passed to the parent WebsocketWordTTSService.
+        """
         super().__init__(**kwargs)
 
         # Indicates if the bot is speaking. If the bot is not speaking we don't
@@ -713,12 +737,14 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
     The audio received from the TTS will be played in context order. That is, if
     we requested audio for a context "A" and then audio for context "B", the
     audio from context ID "A" will be played first.
-
-    Args:
-        **kwargs: Additional arguments passed to the parent WebsocketWordTTSService.
     """
 
     def __init__(self, **kwargs):
+        """Initialize the Audio Context Word TTS service.
+
+        Args:
+            **kwargs: Additional arguments passed to the parent WebsocketWordTTSService.
+        """
         super().__init__(**kwargs)
         self._contexts: Dict[str, asyncio.Queue] = {}
         self._audio_context_task = None
@@ -792,7 +818,7 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
             # Indicate no more audio contexts are available. this will end the
             # task cleanly after all contexts have been processed.
             await self._contexts_queue.put(None)
-            await self.wait_for_task(self._audio_context_task)
+            await self._audio_context_task
             self._audio_context_task = None
 
     async def cancel(self, frame: CancelFrame):
@@ -811,7 +837,7 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
 
     def _create_audio_context_task(self):
         if not self._audio_context_task:
-            self._contexts_queue = WatchdogQueue(self.task_manager)
+            self._contexts_queue = asyncio.Queue()
             self._contexts: Dict[str, asyncio.Queue] = {}
             self._audio_context_task = self.create_task(self._audio_context_task_handler())
 
@@ -853,12 +879,10 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
         while running:
             try:
                 frame = await asyncio.wait_for(queue.get(), timeout=AUDIO_CONTEXT_TIMEOUT)
-                self.reset_watchdog()
                 if frame:
                     await self.push_frame(frame)
                 running = frame is not None
             except asyncio.TimeoutError:
-                self.reset_watchdog()
                 # We didn't get audio, so let's consider this context finished.
                 logger.trace(f"{self} time out on audio context {context_id}")
                 break