dv-pipecat-ai 0.0.82.dev881__py3-none-any.whl → 0.0.85.dev1__py3-none-any.whl

pipecat/processors/dtmf_aggregator.py

@@ -3,15 +3,14 @@ import asyncio
 from pipecat.frames.frames import (
     BotSpeakingFrame,
     CancelFrame,
+    DTMFUpdateSettingsFrame,
+    EndDTMFCaptureFrame,
     EndFrame,
     Frame,
     InputDTMFFrame,
+    StartDTMFCaptureFrame,
     StartInterruptionFrame,
-    StartUserIdleProcessorFrame,
-    StopUserIdleProcessorFrame,
     TranscriptionFrame,
-    UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame,
     WaitForDTMFFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -43,112 +42,214 @@ class DTMFAggregator(FrameProcessor):
         self._idle_timeout = timeout
         self._digits = digits
         self._digit_event = asyncio.Event()
-        self._digit_aggregate_task = None
+        self._aggregation_task = None
         self._end_on = end_on if end_on else set()
         self._reset_on = reset_on if reset_on else set()
-        self._stopped_idle_processor = False
-
-    async def _start_idle_processor(self):
-        await self.push_frame(StartUserIdleProcessorFrame(), FrameDirection.UPSTREAM)
-        self._stopped_idle_processor = False
-
-    async def _stop_idle_processor(self):
-        await self.push_frame(StopUserIdleProcessorFrame(), FrameDirection.UPSTREAM)
-        self._stopped_idle_processor = True
+        self._dtmf_capture_active = False
 
     async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
         # Handle DTMF frames.
         await super().process_frame(frame, direction)
-        await self.push_frame(frame, direction)
-        if isinstance(frame, InputDTMFFrame):
-            # Start the digit aggregation task if it's not running yet.
-            if self._digit_aggregate_task is None:
-                self._digit_aggregate_task = self.create_task(self._digit_agg_handler(direction))
-
-            # Append the incoming digit.
-            if frame.button.value in self._reset_on:
-                self._aggregation = ""
-            elif frame.button.value in self._end_on:
-                await self.flush_aggregation(direction)
-                self._aggregation = ""
-            else:
-                self._digit_event.set()
-                self._aggregation += frame.button.value
-
-                # Flush if the aggregated digits reach the specified length.
-                if self._digits and len(self._aggregation) == self._digits:
-                    await self.flush_aggregation(direction)
-                    self._aggregation = ""
-            if self._stopped_idle_processor:
-                await self._start_idle_processor()
 
+        if isinstance(frame, InputDTMFFrame):
+            # Push the DTMF frame downstream first
+            await self.push_frame(frame, direction)
+            # Then handle it for proper frame ordering
+            await self._handle_dtmf_frame(frame)
         elif isinstance(frame, (EndFrame, CancelFrame)):
             # For EndFrame, flush any pending aggregation and stop the digit aggregation task.
             if self._aggregation:
-                await self.flush_aggregation(direction)
-            if self._digit_aggregate_task:
-                await self._stop_digit_aggregate_task()
+                await self.flush_aggregation()
+            if self._aggregation_task:
+                await self._stop_aggregation_task()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, WaitForDTMFFrame):
             self.logger.debug("Received WaitForDTMFFrame: Waiting for DTMF input")
-            if self._digit_aggregate_task is None:
-                self._digit_aggregate_task = self.create_task(
-                    self._digit_agg_handler(direction, raise_timeout=True)
-                )
-                self._digit_event.set()
-            await self._stop_idle_processor()
+            self._create_aggregation_task(raise_timeout=True)
+            self._digit_event.set()  # Trigger the timeout handler
+            await self._start_dtmf_capture()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, StartInterruptionFrame):
-            self.logger.debug("Received StartInterruptionFrame: Starting idle processor")
-            if self._stopped_idle_processor:
-                await self._start_idle_processor()
+            self.logger.debug("Received StartInterruptionFrame")
             if self._aggregation:
-                await self.flush_aggregation(direction)
+                await self.flush_aggregation()
+            await self._end_dtmf_capture()
+            await self.push_frame(frame, direction)
         elif isinstance(frame, BotSpeakingFrame):
-            if self._digit_aggregate_task is not None:
+            # Signal the aggregation task to continue when bot speaks
+            if self._aggregation_task is not None:
                 self._digit_event.set()
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, DTMFUpdateSettingsFrame):
+            await self._update_settings(frame.settings)
+            # Don't pass the settings frame downstream
+        else:
+            # Pass all other frames through
+            await self.push_frame(frame, direction)
 
-    async def _digit_agg_handler(self, direction: FrameDirection, raise_timeout=False):
-        """Idle task that waits for new DTMF activity. If no new digit is received within
-        the timeout period, the current aggregation is flushed.
+    async def _update_settings(self, settings: dict) -> None:
+        """Update DTMF aggregator settings dynamically.
+
+        Args:
+            settings: Dictionary containing new DTMF settings
+                     Supported keys: timeout, digits, end, reset
         """
+        settings_changed = False
+
+        if "timeout" in settings and settings["timeout"] is not None:
+            new_timeout = float(settings["timeout"])
+            if new_timeout != self._idle_timeout:
+                self.logger.debug(
+                    f"Updating DTMF timeout from {self._idle_timeout} to {new_timeout}"
+                )
+                self._idle_timeout = new_timeout
+                settings_changed = True
+
+        if "digits" in settings:
+            new_digits = settings["digits"]
+            if new_digits != self._digits:
+                self.logger.debug(
+                    f"Updating DTMF digits from {self._digits} to {new_digits}"
+                )
+                self._digits = new_digits
+                settings_changed = True
+
+        if "end" in settings:
+            # Convert single string to set if needed
+            end_value = settings["end"]
+            if end_value is None:
+                new_end_on = set()
+            elif isinstance(end_value, str):
+                new_end_on = {end_value} if end_value else set()
+            else:
+                new_end_on = set(end_value)
+
+            if new_end_on != self._end_on:
+                self.logger.debug(
+                    f"Updating DTMF end_on from {self._end_on} to {new_end_on}"
+                )
+                self._end_on = new_end_on
+                settings_changed = True
+
+        if "reset" in settings:
+            # Convert single string to set if needed
+            reset_value = settings["reset"]
+            if reset_value is None:
+                new_reset_on = set()
+            elif isinstance(reset_value, str):
+                new_reset_on = {reset_value} if reset_value else set()
+            else:
+                new_reset_on = set(reset_value)
+
+            if new_reset_on != self._reset_on:
+                self.logger.debug(
+                    f"Updating DTMF reset_on from {self._reset_on} to {new_reset_on}"
+                )
+                self._reset_on = new_reset_on
+                settings_changed = True
+
+        if settings_changed:
+            self.logger.info(f"DTMF settings updated successfully")
+
+    async def _handle_dtmf_frame(self, frame: InputDTMFFrame):
+        """Handle DTMF input frame processing."""
+        # Create aggregation task if needed
+        if self._aggregation_task is None:
+            self._create_aggregation_task()
+
+        digit_value = frame.button.value
+
+        # Handle reset digits
+        if digit_value in self._reset_on:
+            self._aggregation = ""
+            return
+
+        # Handle end digits
+        if digit_value in self._end_on:
+            if self._aggregation:  # Only flush if we have aggregation
+                await self.flush_aggregation()
+            return
+
+        # Add digit to aggregation
+        self._aggregation += digit_value
+
+        # Signal the aggregation task that a digit was received
+        self._digit_event.set()
+
+        # Check if we reached the digit limit
+        if self._digits and len(self._aggregation) == self._digits:
+            await self.flush_aggregation()
+
+    def _create_aggregation_task(self, raise_timeout: bool = False) -> None:
+        """Creates the aggregation task if it hasn't been created yet."""
+        if not self._aggregation_task:
+            self._aggregation_task = self.create_task(
+                self._aggregation_task_handler(raise_timeout)
+            )
+
+    async def _stop_aggregation_task(self) -> None:
+        """Stops the aggregation task."""
+        if self._aggregation_task:
+            await self.cancel_task(self._aggregation_task)
+            self._aggregation_task = None
+
+    async def _aggregation_task_handler(self, raise_timeout=False):
+        """Background task that handles timeout-based flushing."""
         while True:
             try:
                 # Wait for a new digit signal with a timeout.
-                await asyncio.wait_for(self._digit_event.wait(), timeout=self._idle_timeout)
-            except asyncio.TimeoutError:
-                # No new digit arrived within the timeout period; flush aggregation if non-empty.
-                await self.flush_aggregation(direction, raise_timeout)
-            finally:
-                # Clear the event for the next cycle.
+                await asyncio.wait_for(
+                    self._digit_event.wait(), timeout=self._idle_timeout
+                )
                 self._digit_event.clear()
+            except asyncio.TimeoutError:
+                # No new digit arrived within the timeout period; flush if needed
+                await self.flush_aggregation(raise_timeout=raise_timeout)
 
-    async def flush_aggregation(self, direction: FrameDirection, raise_timeout=False):
+    async def flush_aggregation(self, *, raise_timeout: bool = False):
         """Flush the aggregated digits by emitting a TranscriptionFrame downstream."""
         if self._aggregation:
-            # Todo: Change to different frame type if we decide to handle it in llm processor separately.
+            # Create transcription frame
             aggregated_frame = TranscriptionFrame(
                 f"User inputted: {self._aggregation}.", "", time_now_iso8601()
             )
             aggregated_frame.metadata["push_aggregation"] = True
-            await self.push_frame(StartInterruptionFrame())
-            await self.push_frame(aggregated_frame, direction)
+
+            # Send interruption frame (as per original design)
+            await self.push_frame(StartInterruptionFrame(), FrameDirection.DOWNSTREAM)
+
+            # Push the transcription frame
+            await self.push_frame(aggregated_frame, FrameDirection.DOWNSTREAM)
+
+            # Reset state
             self._aggregation = ""
-        elif raise_timeout and self._stopped_idle_processor:
+            await self._end_dtmf_capture()
+
+        elif raise_timeout and not self._aggregation:
+            # Timeout with no aggregation (WaitForDTMFFrame case)
             transcript_frame = TranscriptionFrame(
                 "User didn't press any digits on the keyboard.", "", time_now_iso8601()
             )
             transcript_frame.metadata["push_aggregation"] = True
-            await self.push_frame(transcript_frame)
-            if self._stopped_idle_processor:
-                await self._start_idle_processor()
+            await self.push_frame(transcript_frame, FrameDirection.DOWNSTREAM)
+            await self._end_dtmf_capture()
+
+    async def _start_dtmf_capture(self):
+        """Signal the start of DTMF capture upstream."""
+        if self._dtmf_capture_active:
+            return
+        await self.push_frame(StartDTMFCaptureFrame(), FrameDirection.UPSTREAM)
+        self._dtmf_capture_active = True
 
-    async def _stop_digit_aggregate_task(self):
-        """Cancels the digit aggregation task if it exists."""
-        if self._digit_aggregate_task:
-            await self.cancel_task(self._digit_aggregate_task)
-            self._digit_aggregate_task = None
+    async def _end_dtmf_capture(self):
+        """Signal the end of DTMF capture upstream."""
+        if not self._dtmf_capture_active:
+            return
+        await self.push_frame(EndDTMFCaptureFrame(), FrameDirection.UPSTREAM)
+        self._dtmf_capture_active = False
 
     async def cleanup(self) -> None:
         """Cleans up resources, ensuring that the digit aggregation task is cancelled."""
         await super().cleanup()
-        if self._digit_aggregate_task:
-            await self._stop_digit_aggregate_task()
+        if self._aggregation_task:
+            await self._stop_aggregation_task()

pipecat/processors/filters/stt_mute_filter.py

@@ -27,12 +27,14 @@ from pipecat.frames.frames import (
     InterimTranscriptionFrame,
     StartFrame,
     StartInterruptionFrame,
+    StartDTMFCaptureFrame,
     STTMuteFrame,
     TranscriptionFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
     VADUserStartedSpeakingFrame,
     VADUserStoppedSpeakingFrame,
+    EndDTMFCaptureFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
@@ -58,6 +60,7 @@ class STTMuteStrategy(Enum):
     FUNCTION_CALL = "function_call"
     ALWAYS = "always"
     CUSTOM = "custom"
+    DTMF_CAPTURE = "dtmf_capture"
 
 
 @dataclass
@@ -120,6 +123,7 @@ class STTMuteFilter(FrameProcessor):
         self._function_call_in_progress = False
         self._is_muted = False  # Initialize as unmuted, will set state on StartFrame if needed
         self._voicemail_detection_enabled = False  # Default to False
+        self._dtmf_capture_active = False
 
     @property
     def is_muted(self) -> bool:
@@ -165,6 +169,10 @@ class STTMuteFilter(FrameProcessor):
                         if should_mute:
                             return True
 
+                case STTMuteStrategy.DTMF_CAPTURE:
+                    if self._dtmf_capture_active:
+                        return True
+
         return False
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -205,7 +213,14 @@ class STTMuteFilter(FrameProcessor):
                 self._first_speech_handled = True
             should_mute = await self._should_mute()
             self.logger.debug(f"BotStoppedSpeaking: should mute={should_mute}")
+        elif isinstance(frame, StartDTMFCaptureFrame):
+            self._dtmf_capture_active = True
+            should_mute = await self._should_mute()
+        elif isinstance(frame, EndDTMFCaptureFrame):
+            self._dtmf_capture_active = False
+            should_mute = await self._should_mute()
         elif isinstance(frame, STTMuteFrame):
+            # TODO: Duplication of frame is actually happening. We get this frame from the downstream and then we again push it downstream. Also we're psuhing is upstream  and again push it upstream in _handle_mute_state.
             should_mute = frame.mute
 
         # Then push the original frame

pipecat/processors/user_idle_processor.py

@@ -15,17 +15,19 @@ from loguru import logger
 from pipecat.frames.frames import (
     BotSpeakingFrame,
     CancelFrame,
+    EndDTMFCaptureFrame,
     EndFrame,
     Frame,
     FunctionCallCancelFrame,
     FunctionCallInProgressFrame,
     FunctionCallResultFrame,
     InputDTMFFrame,
+    LLMFullResponseStartFrame,
+    LLMFullResponseEndFrame,
+    StartDTMFCaptureFrame,
+    StartFrame,
     StartUserIdleProcessorFrame,
     StopUserIdleProcessorFrame,
-    FunctionCallInProgressFrame,
-    FunctionCallResultFrame,
-    StartFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
@@ -83,9 +85,12 @@ class UserIdleProcessor(FrameProcessor):
         self._timeout = timeout
         self._retry_count = 0
         self._interrupted = False
+        self._function_call_active = False
+        self._dtmf_capture_active = False
         self._conversation_started = False
         self._idle_task = None
         self._idle_event = asyncio.Event()
+        self._llm_in_progress = False
 
     def _wrap_callback(
         self,
@@ -180,10 +185,20 @@ class UserIdleProcessor(FrameProcessor):
                 self._idle_event.set()
             elif isinstance(frame, FunctionCallInProgressFrame):
                 # Function calls can take longer than the timeout, so we want to prevent idle callbacks
+                self._function_call_active = True
                 self._interrupted = True
                 self._idle_event.set()
             elif isinstance(frame, FunctionCallResultFrame):
-                self._interrupted = False
+                self._function_call_active = False
+                self._interrupted = self._dtmf_capture_active
+                self._idle_event.set()
+            elif isinstance(frame, StartDTMFCaptureFrame):
+                self._dtmf_capture_active = True
+                self._interrupted = True
+                self._idle_event.set()
+            elif isinstance(frame, EndDTMFCaptureFrame):
+                self._dtmf_capture_active = False
+                self._interrupted = self._function_call_active
                 self._idle_event.set()
             elif isinstance(frame, StartUserIdleProcessorFrame):
                 if not self._idle_task:
@@ -192,6 +207,18 @@ class UserIdleProcessor(FrameProcessor):
             elif isinstance(frame, StopUserIdleProcessorFrame):
                 self.logger.debug("Stopping Idle Processor")
                 await self._stop()
+            elif isinstance(frame, LLMFullResponseStartFrame):
+                self.logger.debug(
+                    "LLM FullResponseStartFrame received, making llm_in_progress to True"
+                )
+                self._idle_event.set()
+                self._llm_in_progress = True
+            elif isinstance(frame, LLMFullResponseEndFrame):
+                self.logger.debug(
+                    "LLM FullResponseEndFrame received, making llm_in_progress to False"
+                )
+                self._idle_event.set()
+                self._llm_in_progress = False
 
     async def cleanup(self) -> None:
         """Cleans up resources when processor is shutting down."""
@@ -210,7 +237,7 @@ class UserIdleProcessor(FrameProcessor):
             try:
                 await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)
             except asyncio.TimeoutError:
-                if not self._interrupted:
+                if not self._interrupted and not self._llm_in_progress:
                     self._retry_count += 1
                     should_continue = await self._callback(self, self._retry_count)
                     if not should_continue:

pipecat/serializers/__init__.py

@@ -1,5 +1,6 @@
 from .base_serializer import FrameSerializer, FrameSerializerType
 from .convox import ConVoxFrameSerializer
+from .custom import CustomFrameSerializer
 from .exotel import ExotelFrameSerializer
 from .plivo import PlivoFrameSerializer
 from .telnyx import TelnyxFrameSerializer
@@ -7,8 +8,9 @@ from .twilio import TwilioFrameSerializer
 
 __all__ = [
     "FrameSerializer",
-    "FrameSerializerType", 
+    "FrameSerializerType",
     "ConVoxFrameSerializer",
+    "CustomFrameSerializer",
     "ExotelFrameSerializer",
     "PlivoFrameSerializer",
     "TelnyxFrameSerializer",

pipecat/serializers/convox.py

@@ -4,9 +4,11 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""ConVox WebSocket frame serializer for audio streaming and call management."""
+
 import base64
-import datetime
 import json
+from datetime import datetime, timezone
 from typing import Optional
 
 from loguru import logger
@@ -99,6 +101,7 @@ class ConVoxFrameSerializer(FrameSerializer):
         """Serializes a Pipecat frame to ConVox WebSocket format.
 
         Handles conversion of various frame types to ConVox WebSocket messages.
+        For EndFrames, initiates call termination if auto_hang_up is enabled.
 
         Args:
             frame: The Pipecat frame to serialize.
@@ -106,7 +109,15 @@ class ConVoxFrameSerializer(FrameSerializer):
         Returns:
             Serialized data as JSON string, or None if the frame isn't handled.
         """
-        if isinstance(frame, StartInterruptionFrame):
+        if (
+            self._params.auto_hang_up
+            and not self._call_ended
+            and isinstance(frame, (EndFrame, CancelFrame))
+        ):
+            self._call_ended = True
+            # Return the callEnd event to be sent via the WebSocket
+            return await self._send_call_end_event()
+        elif isinstance(frame, StartInterruptionFrame):
             # Clear/interrupt command for ConVox
             message = {
                 "event": "clear",
@@ -138,7 +149,7 @@ class ConVoxFrameSerializer(FrameSerializer):
             payload = base64.b64encode(serialized_data).decode("ascii")
 
             # ConVox expects play_audio event format according to the documentation
-            timestamp = datetime.datetime.now().isoformat()
+            timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
 
             message = {
                 "event": "play_audio",
@@ -164,6 +175,32 @@ class ConVoxFrameSerializer(FrameSerializer):
 
         return None
 
+    async def _send_call_end_event(self):
+        """Send a callEnd event to ConVox to terminate the call.
+
+        This method is called when auto_hang_up is enabled and an EndFrame or
+        CancelFrame is received, similar to the logic in end_call_handler.py.
+        """
+        try:
+            call_end_event = {
+                "event": "callEnd",
+                "details": {
+                    "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+                    "direction": "WSS",
+                    "message": "Event trigger request",
+                },
+            }
+
+            logger.info(
+                f"ConVox auto_hang_up: Sending callEnd event for stream_id: {self._stream_id}, call_id: {self._call_id}"
+            )
+            # Note: The actual sending will be handled by the transport layer
+            # when this method returns the JSON string
+            return json.dumps(call_end_event)
+        except Exception as e:
+            logger.error(f"ConVox auto_hang_up: Failed to create callEnd event: {e}")
+            return None
+
     async def deserialize(self, data: str | bytes) -> Frame | None:
         """Deserializes ConVox WebSocket data to Pipecat frames.