dv-pipecat-ai 0.0.82.dev857__py3-none-any.whl → 0.0.85.dev837__py3-none-any.whl

pipecat/processors/frameworks/strands_agents.py

@@ -0,0 +1,170 @@
+"""Strands Agent integration for Pipecat.
+
+This module provides integration with Strands Agents for handling conversational AI
+interactions. It supports both single agent and multi-agent graphs.
+"""
+
+from typing import Optional
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+    Frame,
+    LLMContextFrame,
+    LLMFullResponseEndFrame,
+    LLMFullResponseStartFrame,
+    LLMTextFrame,
+)
+from pipecat.metrics.metrics import LLMTokenUsage
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+try:
+    from strands import Agent
+    from strands.multiagent.graph import Graph
+except ModuleNotFoundError as e:
+    logger.exception("In order to use Strands Agents, you need to `pip install strands-agents`.")
+    raise Exception(f"Missing module: {e}")
+
+
+class StrandsAgentsProcessor(FrameProcessor):
+    """Processor that integrates Strands Agents with Pipecat's frame pipeline.
+
+    This processor takes LLM message frames, extracts the latest user message,
+    and processes it through either a single Strands Agent or a multi-agent Graph.
+    The response is streamed back as text frames with appropriate response markers.
+
+    Supports both single agent streaming and graph-based multi-agent workflows.
+    """
+
+    def __init__(
+        self,
+        agent: Optional[Agent] = None,
+        graph: Optional[Graph] = None,
+        graph_exit_node: Optional[str] = None,
+    ):
+        """Initialize the Strands Agents processor.
+
+        Args:
+            agent: The Strands Agent to use for single-agent processing.
+            graph: The Strands multi-agent Graph to use for graph-based processing.
+            graph_exit_node: The exit node name when using graph-based processing.
+
+        Raises:
+            AssertionError: If neither agent nor graph is provided, or if graph is
+                          provided without a graph_exit_node.
+        """
+        super().__init__()
+        self.agent = agent
+        self.graph = graph
+        self.graph_exit_node = graph_exit_node
+
+        assert self.agent or self.graph, "Either agent or graph must be provided"
+
+        if self.graph:
+            assert self.graph_exit_node, "graph_exit_node must be provided if graph is provided"
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and handle LLM message frames.
+
+        Args:
+            frame: The incoming frame to process.
+            direction: The direction of frame flow in the pipeline.
+        """
+        await super().process_frame(frame, direction)
+        if isinstance(frame, (LLMContextFrame, OpenAILLMContextFrame)):
+            messages = frame.context.get_messages()
+            if messages:
+                last_message = messages[-1]
+                await self._ainvoke(str(last_message["content"]).strip())
+        else:
+            await self.push_frame(frame, direction)
+
+    async def _ainvoke(self, text: str):
+        """Invoke the Strands agent with the provided text and stream results as Pipecat frames.
+
+        Args:
+            text: The user input text to process through the agent or graph.
+        """
+        logger.debug(f"Invoking Strands agent with: {text}")
+        ttfb_tracking = True
+        try:
+            await self.push_frame(LLMFullResponseStartFrame())
+            await self.start_processing_metrics()
+            await self.start_ttfb_metrics()
+
+            if self.graph:
+                # Graph does not stream; await full result then emit assistant text
+                graph_result = await self.graph.invoke_async(text)
+                if ttfb_tracking:
+                    await self.stop_ttfb_metrics()
+                    ttfb_tracking = False
+                try:
+                    node_result = graph_result.results[self.graph_exit_node]
+                    logger.debug(f"Node result: {node_result}")
+                    for agent_result in node_result.get_agent_results():
+                        # Push to TTS service
+                        message = getattr(agent_result, "message", None)
+                        if isinstance(message, dict) and "content" in message:
+                            for block in message["content"]:
+                                if isinstance(block, dict) and "text" in block:
+                                    await self.push_frame(LLMTextFrame(str(block["text"])))
+                        # Update usage metrics
+                        await self._report_usage_metrics(
+                            agent_result.metrics.accumulated_usage.get("inputTokens", 0),
+                            agent_result.metrics.accumulated_usage.get("outputTokens", 0),
+                            agent_result.metrics.accumulated_usage.get("totalTokens", 0),
+                        )
+                except Exception as parse_err:
+                    logger.warning(f"Failed to extract messages from GraphResult: {parse_err}")
+            else:
+                # Agent supports streaming events via async iterator
+                async for event in self.agent.stream_async(text):
+                    # Push to TTS service
+                    if isinstance(event, dict) and "data" in event:
+                        await self.push_frame(LLMTextFrame(str(event["data"])))
+                        if ttfb_tracking:
+                            await self.stop_ttfb_metrics()
+                            ttfb_tracking = False
+
+                    # Update usage metrics
+                    if (
+                        isinstance(event, dict)
+                        and "event" in event
+                        and "metadata" in event["event"]
+                    ):
+                        if "usage" in event["event"]["metadata"]:
+                            usage = event["event"]["metadata"]["usage"]
+                            await self._report_usage_metrics(
+                                usage.get("inputTokens", 0),
+                                usage.get("outputTokens", 0),
+                                usage.get("totalTokens", 0),
+                            )
+        except GeneratorExit:
+            logger.warning(f"{self} generator was closed prematurely")
+        except Exception as e:
+            logger.exception(f"{self} an unknown error occurred: {e}")
+        finally:
+            if ttfb_tracking:
+                await self.stop_ttfb_metrics()
+                ttfb_tracking = False
+            await self.stop_processing_metrics()
+            await self.push_frame(LLMFullResponseEndFrame())
+
+    def can_generate_metrics(self) -> bool:
+        """Check if this service can generate performance metrics.
+
+        Returns:
+            True as this service supports metrics generation.
+        """
+        return True
+
+    async def _report_usage_metrics(
+        self, prompt_tokens: int, completion_tokens: int, total_tokens: int
+    ):
+        tokens = LLMTokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+        )
+        await self.start_llm_usage_metrics(tokens)

pipecat/processors/logger.py

@@ -15,7 +15,7 @@ from pipecat.frames.frames import (
     Frame,
     InputAudioRawFrame,
     OutputAudioRawFrame,
-    TransportMessageFrame,
+    UserSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
@@ -36,9 +36,9 @@ class FrameLogger(FrameProcessor):
         color: Optional[str] = None,
         ignored_frame_types: Tuple[Type[Frame], ...] = (
             BotSpeakingFrame,
+            UserSpeakingFrame,
             InputAudioRawFrame,
             OutputAudioRawFrame,
-            TransportMessageFrame,
         ),
     ):
         """Initialize the frame logger.

pipecat/processors/transcript_processor.py

@@ -19,7 +19,8 @@ from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
     Frame,
-    StartInterruptionFrame,
+    InterruptionFrame,
+    TranscriptDropFrame,
     TranscriptionFrame,
     TranscriptionMessage,
     TranscriptionUpdateFrame,
@@ -44,6 +45,7 @@ class BaseTranscriptProcessor(FrameProcessor):
         super().__init__(**kwargs)
         self._processed_messages: List[TranscriptionMessage] = []
         self._register_event_handler("on_transcript_update")
+        self._register_event_handler("on_transcript_drop")
 
     async def _emit_update(self, messages: List[TranscriptionMessage]):
         """Emit transcript updates for new messages.
@@ -57,6 +59,19 @@ class BaseTranscriptProcessor(FrameProcessor):
             await self._call_event_handler("on_transcript_update", update_frame)
             await self.push_frame(update_frame)
 
+    async def _handle_transcript_drop(self, frame: TranscriptDropFrame):
+        """Handle transcript drop notifications by removing stored messages."""
+        if not frame.transcript_ids:
+            return
+
+        await self._call_event_handler("on_transcript_drop", frame)
+
+        drop_ids = set(frame.transcript_ids)
+        if drop_ids:
+            self._processed_messages = [
+                msg for msg in self._processed_messages if msg.message_id not in drop_ids
+            ]
+
 
 class UserTranscriptProcessor(BaseTranscriptProcessor):
     """Processes user transcription frames into timestamped conversation messages."""
@@ -72,9 +87,15 @@ class UserTranscriptProcessor(BaseTranscriptProcessor):
 
         if isinstance(frame, TranscriptionFrame):
             message = TranscriptionMessage(
-                role="user", user_id=frame.user_id, content=frame.text, timestamp=frame.timestamp
+                role="user",
+                user_id=frame.user_id,
+                content=frame.text,
+                timestamp=frame.timestamp,
+                message_id=frame.id,
             )
             await self._emit_update([message])
+        elif isinstance(frame, TranscriptDropFrame):
+            await self._handle_transcript_drop(frame)
 
         await self.push_frame(frame, direction)
 
@@ -86,7 +107,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
     transcript messages. Utterances are completed when:
 
     - The bot stops speaking (BotStoppedSpeakingFrame)
-    - The bot is interrupted (StartInterruptionFrame)
+    - The bot is interrupted (InterruptionFrame)
     - The pipeline ends (EndFrame)
     """
 
@@ -185,7 +206,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
 
         - TTSTextFrame: Aggregates text for current utterance
         - BotStoppedSpeakingFrame: Completes current utterance
-        - StartInterruptionFrame: Completes current utterance due to interruption
+        - InterruptionFrame: Completes current utterance due to interruption
         - EndFrame: Completes current utterance at pipeline end
         - CancelFrame: Completes current utterance due to cancellation
 
@@ -195,7 +216,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
         """
         await super().process_frame(frame, direction)
 
-        if isinstance(frame, (StartInterruptionFrame, CancelFrame)):
+        if isinstance(frame, (InterruptionFrame, CancelFrame)):
             # Push frame first otherwise our emitted transcription update frame
             # might get cleaned up.
             await self.push_frame(frame, direction)

pipecat/processors/user_idle_processor.py

@@ -15,17 +15,18 @@ from loguru import logger
 from pipecat.frames.frames import (
     BotSpeakingFrame,
     CancelFrame,
+    EndDTMFCaptureFrame,
     EndFrame,
     Frame,
-    FunctionCallCancelFrame,
     FunctionCallInProgressFrame,
     FunctionCallResultFrame,
     InputDTMFFrame,
+    LLMFullResponseStartFrame,
+    LLMFullResponseEndFrame,
+    StartDTMFCaptureFrame,
+    StartFrame,
     StartUserIdleProcessorFrame,
     StopUserIdleProcessorFrame,
-    FunctionCallInProgressFrame,
-    FunctionCallResultFrame,
-    StartFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
@@ -83,9 +84,12 @@ class UserIdleProcessor(FrameProcessor):
         self._timeout = timeout
         self._retry_count = 0
         self._interrupted = False
+        self._function_call_active = False
+        self._dtmf_capture_active = False
         self._conversation_started = False
         self._idle_task = None
         self._idle_event = asyncio.Event()
+        self._llm_in_progress = False
 
     def _wrap_callback(
         self,
@@ -180,10 +184,20 @@ class UserIdleProcessor(FrameProcessor):
                 self._idle_event.set()
             elif isinstance(frame, FunctionCallInProgressFrame):
                 # Function calls can take longer than the timeout, so we want to prevent idle callbacks
+                self._function_call_active = True
                 self._interrupted = True
                 self._idle_event.set()
             elif isinstance(frame, FunctionCallResultFrame):
-                self._interrupted = False
+                self._function_call_active = False
+                self._interrupted = self._dtmf_capture_active
+                self._idle_event.set()
+            elif isinstance(frame, StartDTMFCaptureFrame):
+                self._dtmf_capture_active = True
+                self._interrupted = True
+                self._idle_event.set()
+            elif isinstance(frame, EndDTMFCaptureFrame):
+                self._dtmf_capture_active = False
+                self._interrupted = self._function_call_active
                 self._idle_event.set()
             elif isinstance(frame, StartUserIdleProcessorFrame):
                 if not self._idle_task:
@@ -192,6 +206,18 @@ class UserIdleProcessor(FrameProcessor):
             elif isinstance(frame, StopUserIdleProcessorFrame):
                 self.logger.debug("Stopping Idle Processor")
                 await self._stop()
+            elif isinstance(frame, LLMFullResponseStartFrame):
+                self.logger.debug(
+                    "LLM FullResponseStartFrame received, making llm_in_progress to True"
+                )
+                self._idle_event.set()
+                self._llm_in_progress = True
+            elif isinstance(frame, LLMFullResponseEndFrame):
+                self.logger.debug(
+                    "LLM FullResponseEndFrame received, making llm_in_progress to False"
+                )
+                self._idle_event.set()
+                self._llm_in_progress = False
 
     async def cleanup(self) -> None:
         """Cleans up resources when processor is shutting down."""
@@ -206,15 +232,13 @@ class UserIdleProcessor(FrameProcessor):
 
         Runs in a loop until cancelled or callback indicates completion.
         """
-        while True:
+        running = True
+        while running:
             try:
                 await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)
             except asyncio.TimeoutError:
-                if not self._interrupted:
+                if not self._interrupted and not self._llm_in_progress:
                     self._retry_count += 1
-                    should_continue = await self._callback(self, self._retry_count)
-                    if not should_continue:
-                        await self._stop()
-                        break
+                    running = await self._callback(self, self._retry_count)
             finally:
                 self._idle_event.clear()

pipecat/runner/daily.py

@@ -76,12 +76,14 @@ class DailyRoomConfig(BaseModel):
 async def configure(
     aiohttp_session: aiohttp.ClientSession,
     *,
+    api_key: Optional[str] = None,
     room_exp_duration: Optional[float] = 2.0,
     token_exp_duration: Optional[float] = 2.0,
     sip_caller_phone: Optional[str] = None,
     sip_enable_video: Optional[bool] = False,
     sip_num_endpoints: Optional[int] = 1,
     sip_codecs: Optional[Dict[str, List[str]]] = None,
+    room_properties: Optional[DailyRoomProperties] = None,
 ) -> DailyRoomConfig:
     """Configure Daily room URL and token with optional SIP capabilities.
 
@@ -91,6 +93,7 @@ async def configure(
 
     Args:
         aiohttp_session: HTTP session for making API requests.
+        api_key: Daily API key.
         room_exp_duration: Room expiration time in hours.
         token_exp_duration: Token expiration time in hours.
         sip_caller_phone: Phone number or identifier for SIP display name.
@@ -99,6 +102,10 @@ async def configure(
         sip_num_endpoints: Number of allowed SIP endpoints.
         sip_codecs: Codecs to support for audio and video. If None, uses Daily defaults.
             Example: {"audio": ["OPUS"], "video": ["H264"]}
+        room_properties: Optional DailyRoomProperties to use instead of building from
+            individual parameters. When provided, this overrides room_exp_duration and
+            SIP-related parameters. If not provided, properties are built from the
+            individual parameters as before.
 
     Returns:
         DailyRoomConfig: Object with room_url, token, and optional sip_endpoint.
@@ -115,18 +122,48 @@ async def configure(
         # SIP-enabled room
         sip_config = await configure(session, sip_caller_phone="+15551234567")
         print(f"SIP endpoint: {sip_config.sip_endpoint}")
+
+        # Custom room properties with recording enabled
+        custom_props = DailyRoomProperties(
+            enable_recording="cloud",
+            max_participants=2,
+        )
+        config = await configure(session, room_properties=custom_props)
     """
     # Check for required API key
-    api_key = os.getenv("DAILY_API_KEY")
+    api_key = api_key or os.getenv("DAILY_API_KEY")
     if not api_key:
         raise Exception(
             "DAILY_API_KEY environment variable is required. "
             "Get your API key from https://dashboard.daily.co/developers"
         )
 
+    # Warn if both room_properties and individual parameters are provided
+    if room_properties is not None:
+        individual_params_provided = any(
+            [
+                room_exp_duration != 2.0,
+                token_exp_duration != 2.0,
+                sip_caller_phone is not None,
+                sip_enable_video is not False,
+                sip_num_endpoints != 1,
+                sip_codecs is not None,
+            ]
+        )
+        if individual_params_provided:
+            logger.warning(
+                "Both room_properties and individual parameters (room_exp_duration, token_exp_duration, "
+                "sip_*) were provided. The room_properties will be used and individual parameters "
+                "will be ignored."
+            )
+
     # Determine if SIP mode is enabled
     sip_enabled = sip_caller_phone is not None
 
+    # If room_properties is provided, check if it has SIP configuration
+    if room_properties and room_properties.sip:
+        sip_enabled = True
+
     daily_rest_helper = DailyRESTHelper(
         daily_api_key=api_key,
         daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
@@ -150,27 +187,29 @@ async def configure(
     room_name = f"{room_prefix}-{uuid.uuid4().hex[:8]}"
     logger.info(f"Creating new Daily room: {room_name}")
 
-    # Calculate expiration time
-    expiration_time = time.time() + (room_exp_duration * 60 * 60)
+    # Use provided room_properties or build from parameters
+    if room_properties is None:
+        # Calculate expiration time
+        expiration_time = time.time() + (room_exp_duration * 60 * 60)
 
-    # Create room properties
-    room_properties = DailyRoomProperties(
-        exp=expiration_time,
-        eject_at_room_exp=True,
-    )
-
-    # Add SIP configuration if enabled
-    if sip_enabled:
-        sip_params = DailyRoomSipParams(
-            display_name=sip_caller_phone,
-            video=sip_enable_video,
-            sip_mode="dial-in",
-            num_endpoints=sip_num_endpoints,
-            codecs=sip_codecs,
+        # Create room properties
+        room_properties = DailyRoomProperties(
+            exp=expiration_time,
+            eject_at_room_exp=True,
         )
-        room_properties.sip = sip_params
-        room_properties.enable_dialout = True  # Enable outbound calls if needed
-        room_properties.start_video_off = not sip_enable_video  # Voice-only by default
+
+        # Add SIP configuration if enabled
+        if sip_enabled:
+            sip_params = DailyRoomSipParams(
+                display_name=sip_caller_phone,
+                video=sip_enable_video,
+                sip_mode="dial-in",
+                num_endpoints=sip_num_endpoints,
+                codecs=sip_codecs,
+            )
+            room_properties.sip = sip_params
+            room_properties.enable_dialout = True  # Enable outbound calls if needed
+            room_properties.start_video_off = not sip_enable_video  # Voice-only by default
 
     # Create room parameters
     room_params = DailyRoomParams(name=room_name, properties=room_properties)