dv-pipecat-ai 0.0.82.dev857__py3-none-any.whl → 0.0.85.dev837__py3-none-any.whl

pipecat/audio/vad/vad_analyzer.py

@@ -11,7 +11,9 @@ data structures for voice activity detection in audio streams. Includes state
 management, parameter configuration, and audio analysis framework.
 """
 
+import asyncio
 from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
 from enum import Enum
 from typing import Optional
 
@@ -85,6 +87,10 @@ class VADAnalyzer(ABC):
         self._smoothing_factor = 0.2
         self._prev_volume = 0
 
+        # Thread executor that will run the model. We only need one thread per
+        # analyzer because one analyzer just handles one audio stream.
+        self._executor = ThreadPoolExecutor(max_workers=1)
+
     @property
     def sample_rate(self) -> int:
         """Get the current sample rate.
@@ -161,7 +167,7 @@ class VADAnalyzer(ABC):
         volume = calculate_audio_volume(audio, self.sample_rate)
         return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
 
-    def analyze_audio(self, buffer) -> VADState:
+    async def analyze_audio(self, buffer: bytes) -> VADState:
         """Analyze audio buffer and return current VAD state.
 
         Processes incoming audio data, maintains internal state, and determines
@@ -173,6 +179,12 @@ class VADAnalyzer(ABC):
         Returns:
             Current VAD state after processing the buffer.
         """
+        loop = asyncio.get_running_loop()
+        state = await loop.run_in_executor(self._executor, self._run_analyzer, buffer)
+        return state
+
+    def _run_analyzer(self, buffer: bytes) -> VADState:
+        """Analyze audio buffer and return current VAD state."""
         self._vad_buffer += buffer
 
         num_required_bytes = self._vad_frames_num_bytes

pipecat/extensions/voicemail/voicemail_detector.py

@@ -21,7 +21,6 @@ from typing import List, Optional
 from loguru import logger
 
 from pipecat.frames.frames import (
-    BotInterruptionFrame,
     EndFrame,
     Frame,
     LLMFullResponseEndFrame,
@@ -37,7 +36,8 @@ from pipecat.frames.frames import (
     UserStoppedSpeakingFrame,
 )
 from pipecat.pipeline.parallel_pipeline import ParallelPipeline
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.aggregators.llm_context import LLMContext
+from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
 from pipecat.services.llm_service import LLMService
 from pipecat.sync.base_notifier import BaseNotifier
@@ -360,7 +360,7 @@ class ClassificationProcessor(FrameProcessor):
             await self._voicemail_notifier.notify()  # Clear buffered TTS frames
 
             # Interrupt the current pipeline to stop any ongoing processing
-            await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
+            await self.push_interruption_task_frame_and_wait()
 
             # Set the voicemail event to trigger the voicemail handler
             self._voicemail_event.clear()
@@ -615,8 +615,8 @@ VOICEMAIL SYSTEM (respond "VOICEMAIL"):
         ]
 
         # Create the LLM context and aggregators for conversation management
-        self._context = OpenAILLMContext(self._messages)
-        self._context_aggregator = llm.create_context_aggregator(self._context)
+        self._context = LLMContext(self._messages)
+        self._context_aggregator = LLMContextAggregatorPair(self._context)
 
         # Create notification system for coordinating between components
         self._gate_notifier = EventNotifier()  # Signals classification completion

pipecat/frames/frames.py

@@ -451,12 +451,14 @@ class TranscriptionMessage:
         content: The message content/text.
         user_id: Optional identifier for the user.
         timestamp: Optional timestamp when the message was created.
+        message_id: Optional unique identifier for tracking and dropping messages.
     """
 
     role: Literal["user", "assistant"]
     content: str
     user_id: Optional[str] = None
     timestamp: Optional[str] = None
+    message_id: Optional[int] = None
 
 
 @dataclass
@@ -510,6 +512,17 @@ class TranscriptionUpdateFrame(DataFrame):
         return f"{self.name}(pts: {pts}, messages: {len(self.messages)})"
 
 
+@dataclass
+class TranscriptDropFrame(DataFrame):
+    """Frame indicating previously emitted transcript chunks should be discarded.
+
+    Parameters:
+        transcript_ids: List of frame/message identifiers to drop.
+    """
+
+    transcript_ids: List[int]
+
+
 @dataclass
 class LLMContextFrame(Frame):
     """Frame containing a universal LLM context.
@@ -672,7 +685,7 @@ class TTSSpeakFrame(DataFrame):
 
 
 @dataclass
-class TransportMessageFrame(DataFrame):
+class OutputTransportMessageFrame(DataFrame):
     """Frame containing transport-specific message data.
 
     Parameters:
@@ -685,6 +698,32 @@ class TransportMessageFrame(DataFrame):
         return f"{self.name}(message: {self.message})"
 
 
+@dataclass
+class TransportMessageFrame(OutputTransportMessageFrame):
+    """Frame containing transport-specific message data.
+
+    .. deprecated:: 0.0.87
+        This frame is deprecated and will be removed in a future version.
+        Instead, use `OutputTransportMessageFrame`.
+
+    Parameters:
+        message: The transport message payload.
+    """
+
+    def __post_init__(self):
+        super().__post_init__()
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "TransportMessageFrame is deprecated and will be removed in a future version. "
+                "Instead, use OutputTransportMessageFrame.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+
 @dataclass
 class DTMFFrame:
     """Base class for DTMF (Dual-Tone Multi-Frequency) keypad frames.
@@ -788,43 +827,6 @@ class FatalErrorFrame(ErrorFrame):
     fatal: bool = field(default=True, init=False)
 
 
-@dataclass
-class EndTaskFrame(SystemFrame):
-    """Frame to request graceful pipeline task closure.
-
-    This is used to notify the pipeline task that the pipeline should be
-    closed nicely (flushing all the queued frames) by pushing an EndFrame
-    downstream. This frame should be pushed upstream.
-    """
-
-    pass
-
-
-@dataclass
-class CancelTaskFrame(SystemFrame):
-    """Frame to request immediate pipeline task cancellation.
-
-    This is used to notify the pipeline task that the pipeline should be
-    stopped immediately by pushing a CancelFrame downstream. This frame
-    should be pushed upstream.
-    """
-
-    pass
-
-
-@dataclass
-class StopTaskFrame(SystemFrame):
-    """Frame to request pipeline task stop while keeping processors running.
-
-    This is used to notify the pipeline task that it should be stopped as
-    soon as possible (flushing all the queued frames) but that the pipeline
-    processors should be kept in a running state. This frame should be pushed
-    upstream.
-    """
-
-    pass
-
-
 @dataclass
 class FrameProcessorPauseUrgentFrame(SystemFrame):
     """Frame to pause frame processing immediately.
@@ -857,7 +859,7 @@ class FrameProcessorResumeUrgentFrame(SystemFrame):
 
 
 @dataclass
-class StartInterruptionFrame(SystemFrame):
+class InterruptionFrame(SystemFrame):
     """Frame indicating user started speaking (interruption detected).
 
     Emitted by the BaseInputTransport to indicate that a user has started
@@ -869,6 +871,34 @@ class StartInterruptionFrame(SystemFrame):
     pass
 
 
+@dataclass
+class StartInterruptionFrame(InterruptionFrame):
+    """Frame indicating user started speaking (interruption detected).
+
+    .. deprecated:: 0.0.85
+        This frame is deprecated and will be removed in a future version.
+        Instead, use `InterruptionFrame`.
+
+    Emitted by the BaseInputTransport to indicate that a user has started
+    speaking (i.e. is interrupting). This is similar to
+    UserStartedSpeakingFrame except that it should be pushed concurrently
+    with other frames (so the order is not guaranteed).
+    """
+
+    def __post_init__(self):
+        super().__post_init__()
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "StartInterruptionFrame is deprecated and will be removed in a future version. "
+                "Instead, use InterruptionFrame.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+
 @dataclass
 class UserStartedSpeakingFrame(SystemFrame):
     """Frame indicating user has started speaking.
@@ -944,20 +974,6 @@ class VADUserStoppedSpeakingFrame(SystemFrame):
     pass
 
 
-@dataclass
-class BotInterruptionFrame(SystemFrame):
-    """Frame indicating the bot should be interrupted.
-
-    Emitted when the bot should be interrupted. This will mainly cause the
-    same actions as if the user interrupted except that the
-    UserStartedSpeakingFrame and UserStoppedSpeakingFrame won't be generated.
-    This frame should be pushed upstreams. It results in the BaseInputTransport
-    starting an interruption by pushing a StartInterruptionFrame downstream.
-    """
-
-    pass
-
-
 @dataclass
 class BotStartedSpeakingFrame(SystemFrame):
     """Frame indicating the bot started speaking.
@@ -1115,8 +1131,8 @@ class STTMuteFrame(SystemFrame):
 
 
 @dataclass
-class TransportMessageUrgentFrame(SystemFrame):
-    """Frame for urgent transport messages that need immediate processing.
+class InputTransportMessageFrame(SystemFrame):
+    """Frame for transport messages received from external sources.
 
     Parameters:
         message: The urgent transport message payload.
@@ -1129,20 +1145,69 @@ class TransportMessageUrgentFrame(SystemFrame):
 
 
 @dataclass
-class InputTransportMessageUrgentFrame(TransportMessageUrgentFrame):
+class InputTransportMessageUrgentFrame(InputTransportMessageFrame):
     """Frame for transport messages received from external sources.
 
-    This frame wraps incoming transport messages to distinguish them from outgoing
-    urgent transport messages (TransportMessageUrgentFrame), preventing infinite
-    message loops in the transport layer. It inherits the message payload from
-    TransportMessageFrame while marking the message as having been received
-    rather than generated locally.
+    .. deprecated:: 0.0.87
+        This frame is deprecated and will be removed in a future version.
+        Instead, use `InputTransportMessageFrame`.
 
-    Used by transport implementations to properly handle bidirectional message
-    flow without creating feedback loops.
+    Parameters:
+        message: The urgent transport message payload.
     """
 
-    pass
+    def __post_init__(self):
+        super().__post_init__()
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "InputTransportMessageUrgentFrame is deprecated and will be removed in a future version. "
+                "Instead, use InputTransportMessageFrame.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+
+@dataclass
+class OutputTransportMessageUrgentFrame(SystemFrame):
+    """Frame for urgent transport messages that need to be sent immediately.
+
+    Parameters:
+        message: The urgent transport message payload.
+    """
+
+    message: Any
+
+    def __str__(self):
+        return f"{self.name}(message: {self.message})"
+
+
+@dataclass
+class TransportMessageUrgentFrame(OutputTransportMessageUrgentFrame):
+    """Frame for urgent transport messages that need to be sent immediately.
+
+    .. deprecated:: 0.0.87
+        This frame is deprecated and will be removed in a future version.
+        Instead, use `OutputTransportMessageUrgentFrame`.
+
+    Parameters:
+        message: The urgent transport message payload.
+    """
+
+    def __post_init__(self):
+        super().__post_init__()
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "TransportMessageUrgentFrame is deprecated and will be removed in a future version. "
+                "Instead, use OutputTransportMessageFrame.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
 
 
 @dataclass
@@ -1253,23 +1318,6 @@ class UserImageRawFrame(InputImageRawFrame):
         return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, request: {self.request})"
 
 
-@dataclass
-class VisionImageRawFrame(InputImageRawFrame):
-    """Image frame for vision/image analysis with associated text prompt.
-
-    An image with an associated text to ask for a description of it.
-
-    Parameters:
-        text: Optional text prompt describing what to analyze in the image.
-    """
-
-    text: Optional[str] = None
-
-    def __str__(self):
-        pts = format_pts(self.pts)
-        return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})"
-
-
 @dataclass
 class InputDTMFFrame(DTMFFrame, SystemFrame):
     """DTMF keypress input frame from transport."""
@@ -1306,6 +1354,116 @@ class SpeechControlParamsFrame(SystemFrame):
     turn_params: Optional[SmartTurnParams] = None
 
 
+@dataclass
+class StartDTMFCaptureFrame(SystemFrame):
+    """System frame indicating the bot is actively collecting DTMF input."""
+
+    pass
+
+
+@dataclass
+class EndDTMFCaptureFrame(SystemFrame):
+    """System frame indicating DTMF collection has finished."""
+
+    pass
+
+
+# Task frames
+#
+
+
+@dataclass
+class TaskFrame(SystemFrame):
+    """Base frame for task frames.
+
+    This is a base class for frames that are meant to be sent and handled
+    upstream by the pipeline task. This might result in a corresponding frame
+    sent downstream (e.g. `InterruptionTaskFrame` / `InterruptionFrame` or
+    `EndTaskFrame` / `EndFrame`).
+
+    """
+
+    pass
+
+
+@dataclass
+class EndTaskFrame(TaskFrame):
+    """Frame to request graceful pipeline task closure.
+
+    This is used to notify the pipeline task that the pipeline should be
+    closed nicely (flushing all the queued frames) by pushing an EndFrame
+    downstream. This frame should be pushed upstream.
+    """
+
+    pass
+
+
+@dataclass
+class CancelTaskFrame(TaskFrame):
+    """Frame to request immediate pipeline task cancellation.
+
+    This is used to notify the pipeline task that the pipeline should be
+    stopped immediately by pushing a CancelFrame downstream. This frame
+    should be pushed upstream.
+    """
+
+    pass
+
+
+@dataclass
+class StopTaskFrame(TaskFrame):
+    """Frame to request pipeline task stop while keeping processors running.
+
+    This is used to notify the pipeline task that it should be stopped as
+    soon as possible (flushing all the queued frames) but that the pipeline
+    processors should be kept in a running state. This frame should be pushed
+    upstream.
+    """
+
+    pass
+
+
+@dataclass
+class InterruptionTaskFrame(TaskFrame):
+    """Frame indicating the bot should be interrupted.
+
+    Emitted when the bot should be interrupted. This will mainly cause the
+    same actions as if the user interrupted except that the
+    UserStartedSpeakingFrame and UserStoppedSpeakingFrame won't be generated.
+    This frame should be pushed upstream.
+    """
+
+    pass
+
+
+@dataclass
+class BotInterruptionFrame(InterruptionTaskFrame):
+    """Frame indicating the bot should be interrupted.
+
+    .. deprecated:: 0.0.85
+        This frame is deprecated and will be removed in a future version.
+        Instead, use `InterruptionTaskFrame`.
+
+    Emitted when the bot should be interrupted. This will mainly cause the
+    same actions as if the user interrupted except that the
+    UserStartedSpeakingFrame and UserStoppedSpeakingFrame won't be generated.
+    This frame should be pushed upstream.
+    """
+
+    def __post_init__(self):
+        super().__post_init__()
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "BotInterruptionFrame is deprecated and will be removed in a future version. "
+                "Instead, use InterruptionTaskFrame.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+
 #
 # Control frames
 #
@@ -1473,6 +1631,17 @@ class STTUpdateSettingsFrame(ServiceUpdateSettingsFrame):
     pass
 
 
+@dataclass
+class DTMFUpdateSettingsFrame(ServiceUpdateSettingsFrame):
+    """Frame for updating DTMF aggregator settings.
+
+    Updates DTMF configuration dynamically during conversation flow.
+    Settings can include: timeout, digits, end, reset parameters.
+    """
+
+    pass
+
+
 @dataclass
 class VADParamsUpdateFrame(SystemFrame):
     """Frame for updating VAD parameters.
@@ -1545,23 +1714,34 @@ class MixerEnableFrame(MixerControlFrame):
     enable: bool
 
 
+@dataclass
+class ServiceSwitcherFrame(ControlFrame):
+    """A base class for frames that affect ServiceSwitcher behavior."""
+
+
 @dataclass
 class StartUserIdleProcessorFrame(SystemFrame):
     """Frame to start the UserIdleProcessor monitoring."""
 
 
 @dataclass
-class ServiceSwitcherFrame(ControlFrame):
-    """A base class for frames that control ServiceSwitcher behavior."""
+class StopUserIdleProcessorFrame(SystemFrame):
+    """Frame to stop the UserIdleProcessor monitoring."""
 
     pass
 
 
 @dataclass
-class StopUserIdleProcessorFrame(SystemFrame):
-    """Frame to stop the UserIdleProcessor monitoring."""
+class ResetUserIdleProcessorFrame(SystemFrame):
+    """Frame signalling the UserIdleProcessor to treat the user as active again.
 
-    pass
+    Parameters:
+        text: Optional transcription text that triggered the reset.
+        is_final: Whether the transcription was marked final by STT.
+    """
+
+    text: str = ""
+    is_final: bool = False
 
 
 @dataclass
@@ -1571,6 +1751,17 @@ class WaitForDTMFFrame(ControlFrame):
     pass
 
 
+@dataclass
+class CallTransferFrame(ControlFrame):
+    """Frame to request call transfer to another number/destination.
+
+    Parameters:
+        target: The phone number or destination to transfer the call to.
+    """
+
+    target: str
+
+
 @dataclass
 class ManuallySwitchServiceFrame(ServiceSwitcherFrame):
     """A frame to request a manual switch in the active service in a ServiceSwitcher.

pipecat/observers/loggers/debug_log_observer.py

@@ -54,7 +54,7 @@ class DebugLogObserver(BaseObserver):
 
         Log frames with specific source/destination filters::
 
-            from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
+            from pipecat.frames.frames import InterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
             from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
             from pipecat.transports.base_output import BaseOutputTransport
             from pipecat.services.stt_service import STTService
@@ -62,8 +62,8 @@ class DebugLogObserver(BaseObserver):
             observers=[
                 DebugLogObserver(
                     frame_types={
-                        # Only log StartInterruptionFrame when source is BaseOutputTransport
-                        StartInterruptionFrame: (BaseOutputTransport, FrameEndpoint.SOURCE),
+                        # Only log InterruptionFrame when source is BaseOutputTransport
+                        InterruptionFrame: (BaseOutputTransport, FrameEndpoint.SOURCE),
                         # Only log UserStartedSpeakingFrame when destination is STTService
                         UserStartedSpeakingFrame: (STTService, FrameEndpoint.DESTINATION),
                         # Log LLMTextFrame regardless of source or destination type

pipecat/observers/loggers/llm_log_observer.py

@@ -11,6 +11,7 @@ from loguru import logger
 from pipecat.frames.frames import (
     FunctionCallInProgressFrame,
     FunctionCallResultFrame,
+    LLMContextFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesFrame,
@@ -79,10 +80,13 @@ class LLMLogObserver(BaseObserver):
                 f"🧠 {arrow} {dst} LLM MESSAGES FRAME: {frame.messages} at {time_sec:.2f}s"
             )
         # Log OpenAILLMContextFrame (input)
-        elif isinstance(frame, OpenAILLMContextFrame):
-            logger.debug(
-                f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {frame.context.messages} at {time_sec:.2f}s"
+        elif isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)):
+            messages = (
+                frame.context.messages
+                if isinstance(frame, OpenAILLMContextFrame)
+                else frame.context.get_messages()
             )
+            logger.debug(f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {messages} at {time_sec:.2f}s")
         # Log function call result (input)
         elif isinstance(frame, FunctionCallResultFrame):
             logger.debug(

pipecat/observers/loggers/user_bot_latency_log_observer.py

@@ -61,17 +61,29 @@ class UserBotLatencyLogObserver(BaseObserver):
         elif isinstance(data.frame, UserStoppedSpeakingFrame):
             self._user_stopped_time = time.time()
         elif isinstance(data.frame, (EndFrame, CancelFrame)):
-            if self._latencies:
-                avg_latency = mean(self._latencies)
-                min_latency = min(self._latencies)
-                max_latency = max(self._latencies)
-                logger.info(
-                    f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING - Avg: {avg_latency:.3f}s, Min: {min_latency:.3f}s, Max: {max_latency:.3f}s"
-                )
+            self._log_summary()
         elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time:
             latency = time.time() - self._user_stopped_time
             self._user_stopped_time = 0
             self._latencies.append(latency)
-            logger.debug(
-                f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s"
-            )
+            self._log_latency(latency)
+
+    def _log_summary(self):
+        if not self._latencies:
+            return
+        avg_latency = mean(self._latencies)
+        min_latency = min(self._latencies)
+        max_latency = max(self._latencies)
+        logger.info(
+            f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING - Avg: {avg_latency:.3f}s, Min: {min_latency:.3f}s, Max: {max_latency:.3f}s"
+        )
+
+    def _log_latency(self, latency: float):
+        """Log the latency.
+
+        Args:
+            latency: The latency to log.
+        """
+        logger.debug(
+            f"⏱️ LATENCY FROM USER STOPPED SPEAKING TO BOT STARTED SPEAKING: {latency:.3f}s"
+        )