dv-pipecat-ai 0.0.74.dev770__py3-none-any.whl → 0.0.82.dev776__py3-none-any.whl

pipecat/transports/base_output.py

@@ -4,9 +4,14 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Base output transport implementation for Pipecat.
+
+This module provides the BaseOutputTransport class which handles audio and video
+output processing, including frame buffering, mixing, timing, and media streaming.
+"""
+
 import asyncio
 import itertools
-import sys
 import time
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional
@@ -15,7 +20,7 @@ from loguru import logger
 from PIL import Image
 
 from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer
-from pipecat.audio.utils import create_default_resampler
+from pipecat.audio.utils import create_stream_resampler, is_silence
 from pipecat.frames.frames import (
     BotSpeakingFrame,
     BotStartedSpeakingFrame,
@@ -28,6 +33,8 @@ from pipecat.frames.frames import (
     OutputDTMFFrame,
     OutputDTMFUrgentFrame,
     OutputImageRawFrame,
+    OutputTransportReadyFrame,
+    SpeechOutputAudioRawFrame,
     SpriteFrame,
     StartFrame,
     StartInterruptionFrame,
@@ -39,7 +46,6 @@ from pipecat.frames.frames import (
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.transports.base_transport import TransportParams
-from pipecat.utils.asyncio.watchdog_priority_queue import WatchdogPriorityQueue
 from pipecat.utils.time import nanoseconds_to_seconds
 
 # TODO: When we use GeminiMultimodalLiveLLMService, we need to change this to 0.35 but that creates issue for faster TTS.
@@ -47,7 +53,20 @@ BOT_VAD_STOP_SECS = 0.30
 
 
 class BaseOutputTransport(FrameProcessor):
+    """Base class for output transport implementations.
+
+    Handles audio and video output processing including frame buffering, audio mixing,
+    timing coordination, and media streaming. Supports multiple output destinations
+    and provides interruption handling for real-time communication.
+    """
+
     def __init__(self, params: TransportParams, **kwargs):
+        """Initialize the base output transport.
+
+        Args:
+            params: Transport configuration parameters.
+            **kwargs: Additional arguments passed to parent class.
+        """
         super().__init__(**kwargs)
 
         self._params = params
@@ -68,13 +87,28 @@ class BaseOutputTransport(FrameProcessor):
 
     @property
     def sample_rate(self) -> int:
+        """Get the current audio sample rate.
+
+        Returns:
+            The sample rate in Hz.
+        """
         return self._sample_rate
 
     @property
     def audio_chunk_size(self) -> int:
+        """Get the audio chunk size for output processing.
+
+        Returns:
+            The size of audio chunks in bytes.
+        """
         return self._audio_chunk_size
 
     async def start(self, frame: StartFrame):
+        """Start the output transport and initialize components.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
         self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
 
         # We will write 10ms*CHUNKS of audio at a time (where CHUNKS is the
@@ -84,15 +118,29 @@ class BaseOutputTransport(FrameProcessor):
         self._audio_chunk_size = audio_bytes_10ms * self._params.audio_out_10ms_chunks
 
     async def stop(self, frame: EndFrame):
+        """Stop the output transport and cleanup resources.
+
+        Args:
+            frame: The end frame signaling transport shutdown.
+        """
         for _, sender in self._media_senders.items():
             await sender.stop(frame)
 
     async def cancel(self, frame: CancelFrame):
+        """Cancel the output transport and stop all processing.
+
+        Args:
+            frame: The cancel frame signaling immediate cancellation.
+        """
         for _, sender in self._media_senders.items():
             await sender.cancel(frame)
 
     async def set_transport_ready(self, frame: StartFrame):
-        """To be called when the transport is ready to stream."""
+        """Called when the transport is ready to stream.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
         # Register destinations.
         for destination in self._params.audio_out_destinations:
             await self.register_audio_destination(destination)
@@ -127,28 +175,71 @@ class BaseOutputTransport(FrameProcessor):
             )
             await self._media_senders[destination].start(frame)
 
+        # Sending a frame indicating that the output transport is ready and able to receive frames.
+        await self.push_frame(OutputTransportReadyFrame(), FrameDirection.UPSTREAM)
+
     async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+        """Send a transport message.
+
+        Args:
+            frame: The transport message frame to send.
+        """
         pass
 
     async def register_video_destination(self, destination: str):
+        """Register a video output destination.
+
+        Args:
+            destination: The destination identifier to register.
+        """
         pass
 
     async def register_audio_destination(self, destination: str):
+        """Register an audio output destination.
+
+        Args:
+            destination: The destination identifier to register.
+        """
         pass
 
     async def write_video_frame(self, frame: OutputImageRawFrame):
+        """Write a video frame to the transport.
+
+        Args:
+            frame: The output video frame to write.
+        """
         pass
 
     async def write_audio_frame(self, frame: OutputAudioRawFrame):
+        """Write an audio frame to the transport.
+
+        Args:
+            frame: The output audio frame to write.
+        """
         pass
 
     async def write_dtmf(self, frame: OutputDTMFFrame | OutputDTMFUrgentFrame):
+        """Write a DTMF tone to the transport.
+
+        Args:
+            frame: The DTMF frame to write.
+        """
         pass
 
     async def send_audio(self, frame: OutputAudioRawFrame):
+        """Send an audio frame downstream.
+
+        Args:
+            frame: The audio frame to send.
+        """
         await self.queue_frame(frame, FrameDirection.DOWNSTREAM)
 
     async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
+        """Send an image frame downstream.
+
+        Args:
+            frame: The image frame to send.
+        """
         await self.queue_frame(frame, FrameDirection.DOWNSTREAM)
 
     #
@@ -156,6 +247,12 @@ class BaseOutputTransport(FrameProcessor):
     #
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and handle transport-specific logic.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame flow in the pipeline.
+        """
         await super().process_frame(frame, direction)
 
         #
@@ -201,6 +298,7 @@ class BaseOutputTransport(FrameProcessor):
             await self._handle_frame(frame)
 
     async def _handle_frame(self, frame: Frame):
+        """Handle frames by routing them to appropriate media senders."""
         if frame.transport_destination not in self._media_senders:
             logger.warning(
                 f"{self} destination [{frame.transport_destination}] not registered for frame {frame}"
@@ -227,6 +325,12 @@ class BaseOutputTransport(FrameProcessor):
     #
 
     class MediaSender:
+        """Handles media streaming for a specific destination.
+
+        Manages audio and video output processing including buffering, timing,
+        mixing, and frame delivery for a single output destination.
+        """
+
         def __init__(
             self,
             transport: "BaseOutputTransport",
@@ -236,6 +340,15 @@ class BaseOutputTransport(FrameProcessor):
             audio_chunk_size: int,
             params: TransportParams,
         ):
+            """Initialize the media sender.
+
+            Args:
+                transport: The parent transport instance.
+                destination: The destination identifier for this sender.
+                sample_rate: The audio sample rate in Hz.
+                audio_chunk_size: The size of audio chunks in bytes.
+                params: Transport configuration parameters.
+            """
             self._transport = transport
             self._destination = destination
             self._sample_rate = sample_rate
@@ -249,7 +362,7 @@ class BaseOutputTransport(FrameProcessor):
             self._audio_buffer = bytearray()
 
             # This will be used to resample incoming audio to the output sample rate.
-            self._resampler = create_default_resampler()
+            self._resampler = create_stream_resampler()
 
             # The user can provide a single mixer, to be used by the default
             # destination, or a destination/mixer mapping.
@@ -267,13 +380,28 @@ class BaseOutputTransport(FrameProcessor):
 
         @property
         def sample_rate(self) -> int:
+            """Get the audio sample rate.
+
+            Returns:
+                The sample rate in Hz.
+            """
             return self._sample_rate
 
         @property
         def audio_chunk_size(self) -> int:
+            """Get the audio chunk size.
+
+            Returns:
+                The size of audio chunks in bytes.
+            """
             return self._audio_chunk_size
 
         async def start(self, frame: StartFrame):
+            """Start the media sender and initialize components.
+
+            Args:
+                frame: The start frame containing initialization parameters.
+            """
             self._audio_buffer = bytearray()
 
             # Create all tasks.
@@ -294,8 +422,13 @@ class BaseOutputTransport(FrameProcessor):
                 await self._mixer.start(self._sample_rate)
 
         async def stop(self, frame: EndFrame):
+            """Stop the media sender and cleanup resources.
+
+            Args:
+                frame: The end frame signaling sender shutdown.
+            """
             # Let the sink tasks process the queue until they reach this EndFrame.
-            await self._clock_queue.put((sys.maxsize, frame.id, frame))
+            await self._clock_queue.put((float("inf"), frame.id, frame))
             await self._audio_queue.put(frame)
 
             # At this point we have enqueued an EndFrame and we need to wait for
@@ -303,9 +436,9 @@ class BaseOutputTransport(FrameProcessor):
             # also need to wait for these tasks before cancelling the video task
             # because it might be still rendering.
             if self._audio_task:
-                await self._transport.wait_for_task(self._audio_task)
+                await self._audio_task
             if self._clock_task:
-                await self._transport.wait_for_task(self._clock_task)
+                await self._clock_task
 
             # Stop audio mixer.
             if self._mixer:
@@ -315,12 +448,22 @@ class BaseOutputTransport(FrameProcessor):
             await self._cancel_video_task()
 
         async def cancel(self, frame: CancelFrame):
+            """Cancel the media sender and stop all processing.
+
+            Args:
+                frame: The cancel frame signaling immediate cancellation.
+            """
             # Since we are cancelling everything it doesn't matter what task we cancel first.
             await self._cancel_audio_task()
             await self._cancel_clock_task()
             await self._cancel_video_task()
 
         async def handle_interruptions(self, _: StartInterruptionFrame):
+            """Handle interruption events by restarting tasks and clearing buffers.
+
+            Args:
+                _: The start interruption frame (unused).
+            """
             if not self._transport.interruptions_allowed:
                 return
 
@@ -336,6 +479,11 @@ class BaseOutputTransport(FrameProcessor):
             await self._bot_stopped_speaking()
 
         async def handle_audio_frame(self, frame: OutputAudioRawFrame):
+            """Handle incoming audio frames by buffering and chunking.
+
+            Args:
+                frame: The output audio frame to handle.
+            """
             if not self._params.audio_out_enabled:
                 return
 
@@ -358,6 +506,11 @@ class BaseOutputTransport(FrameProcessor):
                 self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
 
         async def handle_image_frame(self, frame: OutputImageRawFrame | SpriteFrame):
+            """Handle incoming image frames for video output.
+
+            Args:
+                frame: The output image or sprite frame to handle.
+            """
             if not self._params.video_out_enabled:
                 return
 
@@ -369,12 +522,27 @@ class BaseOutputTransport(FrameProcessor):
                 await self._set_video_images(frame.images)
 
         async def handle_timed_frame(self, frame: Frame):
+            """Handle frames with presentation timestamps.
+
+            Args:
+                frame: The frame with timing information to handle.
+            """
             await self._clock_queue.put((frame.pts, frame.id, frame))
 
         async def handle_sync_frame(self, frame: Frame):
+            """Handle frames that need synchronized processing.
+
+            Args:
+                frame: The frame to handle synchronously.
+            """
             await self._audio_queue.put(frame)
 
         async def handle_mixer_control_frame(self, frame: MixerControlFrame):
+            """Handle audio mixer control frames.
+
+            Args:
+                frame: The mixer control frame to handle.
+            """
             if self._mixer:
                 await self._mixer.process_frame(frame)
 
@@ -383,16 +551,19 @@ class BaseOutputTransport(FrameProcessor):
         #
 
         def _create_audio_task(self):
+            """Create the audio processing task."""
             if not self._audio_task:
                 self._audio_queue = asyncio.Queue()
                 self._audio_task = self._transport.create_task(self._audio_task_handler())
 
         async def _cancel_audio_task(self):
+            """Cancel and cleanup the audio processing task."""
             if self._audio_task:
                 await self._transport.cancel_task(self._audio_task)
                 self._audio_task = None
 
         async def _bot_started_speaking(self):
+            """Handle bot started speaking event."""
             if not self._bot_speaking:
                 self._transport.logger.debug(
                     f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking"
@@ -408,6 +579,7 @@ class BaseOutputTransport(FrameProcessor):
                 self._bot_speaking = True
 
         async def _bot_stopped_speaking(self):
+            """Handle bot stopped speaking event."""
             if self._bot_speaking:
                 self._transport.logger.debug(
                     f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking"
@@ -427,6 +599,11 @@ class BaseOutputTransport(FrameProcessor):
                 self._audio_buffer = bytearray()
 
         async def _handle_frame(self, frame: Frame):
+            """Handle various frame types with appropriate processing.
+
+            Args:
+                frame: The frame to handle.
+            """
             if isinstance(frame, OutputImageRawFrame):
                 await self._set_video_image(frame)
             elif isinstance(frame, SpriteFrame):
@@ -437,16 +614,20 @@ class BaseOutputTransport(FrameProcessor):
                 await self._transport.write_dtmf(frame)
 
         def _next_frame(self) -> AsyncGenerator[Frame, None]:
+            """Generate the next frame for audio processing.
+
+            Returns:
+                An async generator yielding frames for processing.
+            """
+
             async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
                 while True:
                     try:
                         frame = await asyncio.wait_for(
                             self._audio_queue.get(), timeout=vad_stop_secs
                         )
-                        self._transport.reset_watchdog()
                         yield frame
                     except asyncio.TimeoutError:
-                        self._transport.reset_watchdog()
                         # Notify the bot stopped speaking upstream if necessary.
                         await self._bot_stopped_speaking()
 
@@ -456,13 +637,11 @@ class BaseOutputTransport(FrameProcessor):
                 while True:
                     try:
                         frame = self._audio_queue.get_nowait()
-                        self._transport.reset_watchdog()
                         if isinstance(frame, OutputAudioRawFrame):
                             frame.audio = await self._mixer.mix(frame.audio)
                         last_frame_time = time.time()
                         yield frame
                     except asyncio.QueueEmpty:
-                        self._transport.reset_watchdog()
                         # Notify the bot stopped speaking upstream if necessary.
                         diff_time = time.time() - last_frame_time
                         if diff_time > vad_stop_secs:
@@ -474,6 +653,11 @@ class BaseOutputTransport(FrameProcessor):
                             num_channels=self._params.audio_out_channels,
                         )
                         yield frame
+                        # Allow other asyncio tasks to execute by adding a small sleep
+                        # Without this sleep, in task cancellation scenarios, this loop would
+                        # continuously return without any delay, leading to 100% CPU utilization
+                        # and preventing cancel/stop signals from being processed properly
+                        await asyncio.sleep(0)
 
             if self._mixer:
                 return with_mixer(BOT_VAD_STOP_SECS)
@@ -481,16 +665,31 @@ class BaseOutputTransport(FrameProcessor):
                 return without_mixer(BOT_VAD_STOP_SECS)
 
         async def _audio_task_handler(self):
+            """Main audio processing task handler."""
             # Push a BotSpeakingFrame every 200ms, we don't really need to push it
             # at every audio chunk. If the audio chunk is bigger than 200ms, push at
             # every audio chunk.
             TOTAL_CHUNK_MS = self._params.audio_out_10ms_chunks * 10
             BOT_SPEAKING_CHUNK_PERIOD = max(int(200 / TOTAL_CHUNK_MS), 1)
             bot_speaking_counter = 0
+            speech_last_speaking_time = 0
+
             async for frame in self._next_frame():
                 # Notify the bot started speaking upstream if necessary and that
                 # it's actually speaking.
+                is_speaking = False
                 if isinstance(frame, TTSAudioRawFrame):
+                    is_speaking = True
+                elif isinstance(frame, SpeechOutputAudioRawFrame):
+                    if not is_silence(frame.audio):
+                        is_speaking = True
+                        speech_last_speaking_time = time.time()
+                    else:
+                        silence_duration = time.time() - speech_last_speaking_time
+                        if silence_duration > BOT_VAD_STOP_SECS:
+                            await self._bot_stopped_speaking()
+
+                if is_speaking:
                     await self._bot_started_speaking()
                     if bot_speaking_counter % BOT_SPEAKING_CHUNK_PERIOD == 0:
                         await self._transport.push_frame(BotSpeakingFrame())
@@ -519,23 +718,36 @@ class BaseOutputTransport(FrameProcessor):
         #
 
         def _create_video_task(self):
+            """Create the video processing task if video output is enabled."""
             if not self._video_task and self._params.video_out_enabled:
                 self._video_queue = asyncio.Queue()
                 self._video_task = self._transport.create_task(self._video_task_handler())
 
         async def _cancel_video_task(self):
+            """Cancel and cleanup the video processing task."""
             # Stop video output task.
             if self._video_task:
                 await self._transport.cancel_task(self._video_task)
                 self._video_task = None
 
         async def _set_video_image(self, image: OutputImageRawFrame):
+            """Set a single video image for cycling output.
+
+            Args:
+                image: The image frame to cycle for video output.
+            """
             self._video_images = itertools.cycle([image])
 
         async def _set_video_images(self, images: List[OutputImageRawFrame]):
+            """Set multiple video images for cycling output.
+
+            Args:
+                images: The list of image frames to cycle for video output.
+            """
             self._video_images = itertools.cycle(images)
 
         async def _video_task_handler(self):
+            """Main video processing task handler."""
             self._video_start_time = None
             self._video_frame_index = 0
             self._video_frame_duration = 1 / self._params.video_out_framerate
@@ -551,6 +763,7 @@ class BaseOutputTransport(FrameProcessor):
                     await asyncio.sleep(self._video_frame_duration)
 
         async def _video_is_live_handler(self):
+            """Handle live video streaming with frame timing."""
             image = await self._video_queue.get()
 
             # We get the start time as soon as we get the first image.
@@ -576,6 +789,12 @@ class BaseOutputTransport(FrameProcessor):
             self._video_queue.task_done()
 
         async def _draw_image(self, frame: OutputImageRawFrame):
+            """Draw/render an image frame with resizing if needed.
+
+            Args:
+                frame: The image frame to draw.
+            """
+
             def resize_frame(frame: OutputImageRawFrame) -> OutputImageRawFrame:
                 desired_size = (self._params.video_out_width, self._params.video_out_height)
 
@@ -602,16 +821,19 @@ class BaseOutputTransport(FrameProcessor):
         #
 
         def _create_clock_task(self):
+            """Create the clock/timing processing task."""
             if not self._clock_task:
-                self._clock_queue = WatchdogPriorityQueue(self._transport.task_manager)
+                self._clock_queue = asyncio.PriorityQueue()
                 self._clock_task = self._transport.create_task(self._clock_task_handler())
 
         async def _cancel_clock_task(self):
+            """Cancel and cleanup the clock processing task."""
             if self._clock_task:
                 await self._transport.cancel_task(self._clock_task)
                 self._clock_task = None
 
         async def _clock_task_handler(self):
+            """Main clock/timing task handler for timed frame delivery."""
             running = True
             while running:
                 timestamp, _, frame = await self._clock_queue.get()