dv-pipecat-ai 0.0.82.dev69__py3-none-any.whl → 0.0.82.dev759__py3-none-any.whl

pipecat/processors/aggregators/vision_image_frame.py

@@ -10,13 +10,22 @@ This module provides frame aggregation functionality to combine text and image
 frames into vision frames for multimodal processing.
 """
 
-from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame
+from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame
+from pipecat.processors.aggregators.openai_llm_context import (
+    OpenAILLMContext,
+    OpenAILLMContextFrame,
+)
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
 class VisionImageFrameAggregator(FrameProcessor):
     """Aggregates consecutive text and image frames into vision frames.
 
+    .. deprecated:: 0.0.85
+        VisionImageRawFrame has been removed in favor of context frames
+        (LLMContextFrame or OpenAILLMContextFrame), so this aggregator is not
+        needed anymore. See the 12* examples for the new recommended pattern.
+
     This aggregator waits for a consecutive TextFrame and an InputImageRawFrame.
     After the InputImageRawFrame arrives it will output a VisionImageRawFrame
     combining both the text and image data for multimodal processing.
@@ -28,6 +37,17 @@ class VisionImageFrameAggregator(FrameProcessor):
         The aggregator starts with no cached text, waiting for the first
         TextFrame to arrive before it can create vision frames.
         """
+        import warnings
+
+        warnings.warn(
+            "VisionImageFrameAggregator is deprecated. "
+            "VisionImageRawFrame has been removed in favor of context frames "
+            "(LLMContextFrame or OpenAILLMContextFrame), so this aggregator is "
+            "not needed anymore. See the 12* examples for the new recommended "
+            "pattern.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         super().__init__()
         self._describe_text = None
 
@@ -47,12 +67,14 @@ class VisionImageFrameAggregator(FrameProcessor):
             self._describe_text = frame.text
         elif isinstance(frame, InputImageRawFrame):
             if self._describe_text:
-                frame = VisionImageRawFrame(
+                context = OpenAILLMContext()
+                context.add_image_frame_message(
                     text=self._describe_text,
                     image=frame.image,
                     size=frame.size,
                     format=frame.format,
                 )
+                frame = OpenAILLMContextFrame(context)
                 await self.push_frame(frame)
                 self._describe_text = None
         else:

pipecat/processors/audio/audio_buffer_processor.py

@@ -137,12 +137,12 @@ class AudioBufferProcessor(FrameProcessor):
         return self._num_channels
 
     def has_audio(self) -> bool:
-        """Check if both user and bot audio buffers contain data.
+        """Check if either user or bot audio buffers contain data.
 
         Returns:
-            True if both buffers contain audio data.
+            True if either buffer contains audio data.
         """
-        return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio(
+        return self._buffer_has_audio(self._user_audio_buffer) or self._buffer_has_audio(
             self._bot_audio_buffer
         )
 
@@ -229,9 +229,12 @@ class AudioBufferProcessor(FrameProcessor):
             # Save time of frame so we can compute silence.
             self._last_bot_frame_at = time.time()
 
-        if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size:
+        if self._buffer_size > 0 and (
+            len(self._user_audio_buffer) >= self._buffer_size
+            or len(self._bot_audio_buffer) >= self._buffer_size
+        ):
             await self._call_on_audio_data_handler()
-            self._reset_recording()
+            self._reset_primary_audio_buffers()
 
         # Process turn recording with preprocessed data.
         if self._enable_turn_audio:
@@ -272,9 +275,15 @@ class AudioBufferProcessor(FrameProcessor):
 
     async def _call_on_audio_data_handler(self):
         """Call the audio data event handlers with buffered audio."""
-        if not self.has_audio() or not self._recording:
+        if not self._recording:
             return
 
+        if len(self._user_audio_buffer) == 0 and len(self._bot_audio_buffer) == 0:
+            return
+
+        self._align_track_buffers()
+        flush_time = time.time()
+
         # Call original handler with merged audio
         merged_audio = self.merge_audio_buffers()
         await self._call_event_handler(
@@ -290,23 +299,49 @@ class AudioBufferProcessor(FrameProcessor):
             self._num_channels,
         )
 
+        self._last_user_frame_at = flush_time
+        self._last_bot_frame_at = flush_time
+
     def _buffer_has_audio(self, buffer: bytearray) -> bool:
         """Check if a buffer contains audio data."""
         return buffer is not None and len(buffer) > 0
 
     def _reset_recording(self):
         """Reset recording state and buffers."""
-        self._reset_audio_buffers()
+        self._reset_all_audio_buffers()
         self._last_user_frame_at = time.time()
         self._last_bot_frame_at = time.time()
 
-    def _reset_audio_buffers(self):
+    def _reset_all_audio_buffers(self):
         """Reset all audio buffers to empty state."""
+        self._reset_primary_audio_buffers()
+        self._reset_turn_audio_buffers()
+
+    def _reset_primary_audio_buffers(self):
+        """Clear user and bot buffers while preserving turn buffers and timestamps."""
         self._user_audio_buffer = bytearray()
         self._bot_audio_buffer = bytearray()
+
+    def _reset_turn_audio_buffers(self):
+        """Clear user and bot turn buffers while preserving primary buffers and timestamps."""
         self._user_turn_audio_buffer = bytearray()
         self._bot_turn_audio_buffer = bytearray()
 
+    def _align_track_buffers(self):
+        """Pad the shorter track with silence so both tracks stay in sync."""
+        user_len = len(self._user_audio_buffer)
+        bot_len = len(self._bot_audio_buffer)
+        if user_len == bot_len:
+            return
+
+        target_len = max(user_len, bot_len)
+        if user_len < target_len:
+            self._user_audio_buffer.extend(b"\x00" * (target_len - user_len))
+            self._last_user_frame_at = max(self._last_user_frame_at, self._last_bot_frame_at)
+        if bot_len < target_len:
+            self._bot_audio_buffer.extend(b"\x00" * (target_len - bot_len))
+            self._last_bot_frame_at = max(self._last_bot_frame_at, self._last_user_frame_at)
+
     async def _resample_input_audio(self, frame: InputAudioRawFrame) -> bytes:
         """Resample audio frame to the target sample rate."""
         return await self._input_resampler.resample(

pipecat/processors/dtmf_aggregator.py

@@ -4,15 +4,13 @@ from pipecat.frames.frames import (
     BotSpeakingFrame,
     CancelFrame,
     DTMFUpdateSettingsFrame,
+    EndDTMFCaptureFrame,
     EndFrame,
     Frame,
     InputDTMFFrame,
+    StartDTMFCaptureFrame,
     StartInterruptionFrame,
-    StartUserIdleProcessorFrame,
-    StopUserIdleProcessorFrame,
     TranscriptionFrame,
-    UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame,
     WaitForDTMFFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -44,95 +42,78 @@ class DTMFAggregator(FrameProcessor):
         self._idle_timeout = timeout
         self._digits = digits
         self._digit_event = asyncio.Event()
-        self._digit_aggregate_task = None
+        self._aggregation_task = None
         self._end_on = end_on if end_on else set()
         self._reset_on = reset_on if reset_on else set()
-        self._stopped_idle_processor = False
-
-    async def _start_idle_processor(self):
-        await self.push_frame(StartUserIdleProcessorFrame(), FrameDirection.UPSTREAM)
-        self._stopped_idle_processor = False
-
-    async def _stop_idle_processor(self):
-        await self.push_frame(StopUserIdleProcessorFrame(), FrameDirection.UPSTREAM)
-        self._stopped_idle_processor = True
+        self._dtmf_capture_active = False
 
     async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
         # Handle DTMF frames.
         await super().process_frame(frame, direction)
-        await self.push_frame(frame, direction)
-        if isinstance(frame, InputDTMFFrame):
-            # Start the digit aggregation task if it's not running yet.
-            if self._digit_aggregate_task is None:
-                self._digit_aggregate_task = self.create_task(self._digit_agg_handler(direction))
-
-            # Append the incoming digit.
-            if frame.button.value in self._reset_on:
-                self._aggregation = ""
-            elif frame.button.value in self._end_on:
-                await self.flush_aggregation(direction)
-                self._aggregation = ""
-            else:
-                self._digit_event.set()
-                self._aggregation += frame.button.value
-
-                # Flush if the aggregated digits reach the specified length.
-                if self._digits and len(self._aggregation) == self._digits:
-                    await self.flush_aggregation(direction)
-                    self._aggregation = ""
-            if self._stopped_idle_processor:
-                await self._start_idle_processor()
 
+        if isinstance(frame, InputDTMFFrame):
+            # Push the DTMF frame downstream first
+            await self.push_frame(frame, direction)
+            # Then handle it for proper frame ordering
+            await self._handle_dtmf_frame(frame)
         elif isinstance(frame, (EndFrame, CancelFrame)):
             # For EndFrame, flush any pending aggregation and stop the digit aggregation task.
             if self._aggregation:
-                await self.flush_aggregation(direction)
-            if self._digit_aggregate_task:
-                await self._stop_digit_aggregate_task()
+                await self.flush_aggregation()
+            if self._aggregation_task:
+                await self._stop_aggregation_task()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, WaitForDTMFFrame):
             self.logger.debug("Received WaitForDTMFFrame: Waiting for DTMF input")
-            if self._digit_aggregate_task is None:
-                self._digit_aggregate_task = self.create_task(
-                    self._digit_agg_handler(direction, raise_timeout=True)
-                )
-                self._digit_event.set()
-            await self._stop_idle_processor()
+            self._create_aggregation_task(raise_timeout=True)
+            self._digit_event.set()  # Trigger the timeout handler
+            await self._start_dtmf_capture()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, StartInterruptionFrame):
-            self.logger.debug("Received StartInterruptionFrame: Starting idle processor")
-            if self._stopped_idle_processor:
-                await self._start_idle_processor()
+            self.logger.debug("Received StartInterruptionFrame")
             if self._aggregation:
-                await self.flush_aggregation(direction)
+                await self.flush_aggregation()
+            await self._end_dtmf_capture()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, BotSpeakingFrame):
-            if self._digit_aggregate_task is not None:
+            # Signal the aggregation task to continue when bot speaks
+            if self._aggregation_task is not None:
                 self._digit_event.set()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, DTMFUpdateSettingsFrame):
             await self._update_settings(frame.settings)
             # Don't pass the settings frame downstream
+        else:
+            # Pass all other frames through
+            await self.push_frame(frame, direction)
 
     async def _update_settings(self, settings: dict) -> None:
         """Update DTMF aggregator settings dynamically.
-        
+
         Args:
             settings: Dictionary containing new DTMF settings
                      Supported keys: timeout, digits, end, reset
         """
         settings_changed = False
-        
+
         if "timeout" in settings and settings["timeout"] is not None:
             new_timeout = float(settings["timeout"])
             if new_timeout != self._idle_timeout:
-                self.logger.debug(f"Updating DTMF timeout from {self._idle_timeout} to {new_timeout}")
+                self.logger.debug(
+                    f"Updating DTMF timeout from {self._idle_timeout} to {new_timeout}"
+                )
                 self._idle_timeout = new_timeout
                 settings_changed = True
-        
+
         if "digits" in settings:
             new_digits = settings["digits"]
             if new_digits != self._digits:
-                self.logger.debug(f"Updating DTMF digits from {self._digits} to {new_digits}")
+                self.logger.debug(
+                    f"Updating DTMF digits from {self._digits} to {new_digits}"
+                )
                 self._digits = new_digits
                 settings_changed = True
-        
+
         if "end" in settings:
             # Convert single string to set if needed
             end_value = settings["end"]
@@ -142,12 +123,14 @@ class DTMFAggregator(FrameProcessor):
                 new_end_on = {end_value} if end_value else set()
             else:
                 new_end_on = set(end_value)
-            
+
             if new_end_on != self._end_on:
-                self.logger.debug(f"Updating DTMF end_on from {self._end_on} to {new_end_on}")
+                self.logger.debug(
+                    f"Updating DTMF end_on from {self._end_on} to {new_end_on}"
+                )
                 self._end_on = new_end_on
                 settings_changed = True
-        
+
         if "reset" in settings:
             # Convert single string to set if needed
             reset_value = settings["reset"]
@@ -157,58 +140,116 @@ class DTMFAggregator(FrameProcessor):
                 new_reset_on = {reset_value} if reset_value else set()
             else:
                 new_reset_on = set(reset_value)
-            
+
             if new_reset_on != self._reset_on:
-                self.logger.debug(f"Updating DTMF reset_on from {self._reset_on} to {new_reset_on}")
+                self.logger.debug(
+                    f"Updating DTMF reset_on from {self._reset_on} to {new_reset_on}"
+                )
                 self._reset_on = new_reset_on
                 settings_changed = True
-        
+
         if settings_changed:
             self.logger.info(f"DTMF settings updated successfully")
 
-    async def _digit_agg_handler(self, direction: FrameDirection, raise_timeout=False):
-        """Idle task that waits for new DTMF activity. If no new digit is received within
-        the timeout period, the current aggregation is flushed.
-        """
+    async def _handle_dtmf_frame(self, frame: InputDTMFFrame):
+        """Handle DTMF input frame processing."""
+        # Create aggregation task if needed
+        if self._aggregation_task is None:
+            self._create_aggregation_task()
+
+        digit_value = frame.button.value
+
+        # Handle reset digits
+        if digit_value in self._reset_on:
+            self._aggregation = ""
+            return
+
+        # Handle end digits
+        if digit_value in self._end_on:
+            if self._aggregation:  # Only flush if we have aggregation
+                await self.flush_aggregation()
+            return
+
+        # Add digit to aggregation
+        self._aggregation += digit_value
+
+        # Signal the aggregation task that a digit was received
+        self._digit_event.set()
+
+        # Check if we reached the digit limit
+        if self._digits and len(self._aggregation) == self._digits:
+            await self.flush_aggregation()
+
+    def _create_aggregation_task(self, raise_timeout: bool = False) -> None:
+        """Creates the aggregation task if it hasn't been created yet."""
+        if not self._aggregation_task:
+            self._aggregation_task = self.create_task(
+                self._aggregation_task_handler(raise_timeout)
+            )
+
+    async def _stop_aggregation_task(self) -> None:
+        """Stops the aggregation task."""
+        if self._aggregation_task:
+            await self.cancel_task(self._aggregation_task)
+            self._aggregation_task = None
+
+    async def _aggregation_task_handler(self, raise_timeout=False):
+        """Background task that handles timeout-based flushing."""
         while True:
             try:
                 # Wait for a new digit signal with a timeout.
-                await asyncio.wait_for(self._digit_event.wait(), timeout=self._idle_timeout)
-            except asyncio.TimeoutError:
-                # No new digit arrived within the timeout period; flush aggregation if non-empty.
-                await self.flush_aggregation(direction, raise_timeout)
-            finally:
-                # Clear the event for the next cycle.
+                await asyncio.wait_for(
+                    self._digit_event.wait(), timeout=self._idle_timeout
+                )
                 self._digit_event.clear()
+            except asyncio.TimeoutError:
+                # No new digit arrived within the timeout period; flush if needed
+                await self.flush_aggregation(raise_timeout=raise_timeout)
 
-    async def flush_aggregation(self, direction: FrameDirection, raise_timeout=False):
+    async def flush_aggregation(self, *, raise_timeout: bool = False):
         """Flush the aggregated digits by emitting a TranscriptionFrame downstream."""
         if self._aggregation:
-            # Todo: Change to different frame type if we decide to handle it in llm processor separately.
+            # Create transcription frame
             aggregated_frame = TranscriptionFrame(
                 f"User inputted: {self._aggregation}.", "", time_now_iso8601()
             )
             aggregated_frame.metadata["push_aggregation"] = True
-            await self.push_frame(StartInterruptionFrame())
-            await self.push_frame(aggregated_frame, direction)
+
+            # Send interruption frame (as per original design)
+            await self.push_frame(StartInterruptionFrame(), FrameDirection.DOWNSTREAM)
+
+            # Push the transcription frame
+            await self.push_frame(aggregated_frame, FrameDirection.DOWNSTREAM)
+
+            # Reset state
             self._aggregation = ""
-        elif raise_timeout and self._stopped_idle_processor:
+            await self._end_dtmf_capture()
+
+        elif raise_timeout and not self._aggregation:
+            # Timeout with no aggregation (WaitForDTMFFrame case)
             transcript_frame = TranscriptionFrame(
                 "User didn't press any digits on the keyboard.", "", time_now_iso8601()
             )
             transcript_frame.metadata["push_aggregation"] = True
-            await self.push_frame(transcript_frame)
-            if self._stopped_idle_processor:
-                await self._start_idle_processor()
+            await self.push_frame(transcript_frame, FrameDirection.DOWNSTREAM)
+            await self._end_dtmf_capture()
+
+    async def _start_dtmf_capture(self):
+        """Signal the start of DTMF capture upstream."""
+        if self._dtmf_capture_active:
+            return
+        await self.push_frame(StartDTMFCaptureFrame(), FrameDirection.UPSTREAM)
+        self._dtmf_capture_active = True
 
-    async def _stop_digit_aggregate_task(self):
-        """Cancels the digit aggregation task if it exists."""
-        if self._digit_aggregate_task:
-            await self.cancel_task(self._digit_aggregate_task)
-            self._digit_aggregate_task = None
+    async def _end_dtmf_capture(self):
+        """Signal the end of DTMF capture upstream."""
+        if not self._dtmf_capture_active:
+            return
+        await self.push_frame(EndDTMFCaptureFrame(), FrameDirection.UPSTREAM)
+        self._dtmf_capture_active = False
 
     async def cleanup(self) -> None:
         """Cleans up resources, ensuring that the digit aggregation task is cancelled."""
         await super().cleanup()
-        if self._digit_aggregate_task:
-            await self._stop_digit_aggregate_task()
+        if self._aggregation_task:
+            await self._stop_aggregation_task()

pipecat/processors/filters/stt_mute_filter.py

@@ -25,14 +25,17 @@ from pipecat.frames.frames import (
     FunctionCallResultFrame,
     InputAudioRawFrame,
     InterimTranscriptionFrame,
+    InterruptionFrame,
     StartFrame,
     StartInterruptionFrame,
+    StartDTMFCaptureFrame,
     STTMuteFrame,
     TranscriptionFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
     VADUserStartedSpeakingFrame,
     VADUserStoppedSpeakingFrame,
+    EndDTMFCaptureFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
@@ -58,6 +61,7 @@ class STTMuteStrategy(Enum):
     FUNCTION_CALL = "function_call"
     ALWAYS = "always"
     CUSTOM = "custom"
+    DTMF_CAPTURE = "dtmf_capture"
 
 
 @dataclass
@@ -120,6 +124,7 @@ class STTMuteFilter(FrameProcessor):
         self._function_call_in_progress = False
         self._is_muted = False  # Initialize as unmuted, will set state on StartFrame if needed
         self._voicemail_detection_enabled = False  # Default to False
+        self._dtmf_capture_active = False
 
     @property
     def is_muted(self) -> bool:
@@ -165,6 +170,10 @@ class STTMuteFilter(FrameProcessor):
                         if should_mute:
                             return True
 
+                case STTMuteStrategy.DTMF_CAPTURE:
+                    if self._dtmf_capture_active:
+                        return True
+
         return False
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -205,12 +214,20 @@ class STTMuteFilter(FrameProcessor):
                 self._first_speech_handled = True
             should_mute = await self._should_mute()
             self.logger.debug(f"BotStoppedSpeaking: should mute={should_mute}")
+        elif isinstance(frame, StartDTMFCaptureFrame):
+            self._dtmf_capture_active = True
+            should_mute = await self._should_mute()
+        elif isinstance(frame, EndDTMFCaptureFrame):
+            self._dtmf_capture_active = False
+            should_mute = await self._should_mute()
         elif isinstance(frame, STTMuteFrame):
+            # TODO: Duplication of frame is actually happening. We get this frame from the downstream and then we again push it downstream. Also we're psuhing is upstream  and again push it upstream in _handle_mute_state.
             should_mute = frame.mute
 
         # Then push the original frame
         # Conditionally include InputAudioRawFrame in suppression tuple based on voicemail_detection_enabled
         suppression_types = (
+            InterruptionFrame,
             StartInterruptionFrame,
             VADUserStartedSpeakingFrame,
             VADUserStoppedSpeakingFrame,