dv-pipecat-ai 0.0.93.dev2__tar.gz → 0.0.93.dev3__tar.gz

{dv_pipecat_ai-0.0.93.dev2/src/dv_pipecat_ai.egg-info → dv_pipecat_ai-0.0.93.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-pipecat-ai
-Version: 0.0.93.dev2
+Version: 0.0.93.dev3
 Summary: An open source framework for voice (and multimodal) assistants
 License-Expression: BSD-2-Clause
 Project-URL: Source, https://github.com/pipecat-ai/pipecat

{dv_pipecat_ai-0.0.93.dev2 → dv_pipecat_ai-0.0.93.dev3/src/dv_pipecat_ai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-pipecat-ai
-Version: 0.0.93.dev2
+Version: 0.0.93.dev3
 Summary: An open source framework for voice (and multimodal) assistants
 License-Expression: BSD-2-Clause
 Project-URL: Source, https://github.com/pipecat-ai/pipecat

{dv_pipecat_ai-0.0.93.dev2 → dv_pipecat_ai-0.0.93.dev3}/src/dv_pipecat_ai.egg-info/SOURCES.txt

@@ -98,6 +98,7 @@ src/pipecat/adapters/services/bedrock_adapter.py
 src/pipecat/adapters/services/gemini_adapter.py
 src/pipecat/adapters/services/open_ai_adapter.py
 src/pipecat/adapters/services/open_ai_realtime_adapter.py
+src/pipecat/adapters/services/open_ai_responses_adapter.py
 src/pipecat/audio/__init__.py
 src/pipecat/audio/utils.py
 src/pipecat/audio/dtmf/__init__.py
@@ -378,6 +379,8 @@ src/pipecat/services/openai/realtime/context.py
 src/pipecat/services/openai/realtime/events.py
 src/pipecat/services/openai/realtime/frames.py
 src/pipecat/services/openai/realtime/llm.py
+src/pipecat/services/openai/responses/__init__.py
+src/pipecat/services/openai/responses/llm.py
 src/pipecat/services/openai_realtime/__init__.py
 src/pipecat/services/openai_realtime/azure.py
 src/pipecat/services/openai_realtime/context.py

dv_pipecat_ai-0.0.93.dev3/src/pipecat/adapters/services/open_ai_responses_adapter.py

@@ -0,0 +1,242 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""OpenAI Responses API adapter for Pipecat."""
+
+import copy
+from typing import Any, Dict, List, Optional, TypedDict
+
+from loguru import logger
+from openai._types import NotGiven as OpenAINotGiven
+from openai.types.responses import FunctionToolParam, ResponseInputItemParam
+
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.processors.aggregators.llm_context import (
+    LLMContext,
+    LLMContextMessage,
+    LLMSpecificMessage,
+    NotGiven,
+)
+
+
+class OpenAIResponsesLLMInvocationParams(TypedDict, total=False):
+    """Context-based parameters for invoking OpenAI Responses API."""
+
+    input: List[ResponseInputItemParam]
+    tools: List[FunctionToolParam] | OpenAINotGiven
+    tool_choice: str | dict | OpenAINotGiven
+    instructions: str
+
+
+class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParams]):
+    """OpenAI Responses API adapter for Pipecat.
+
+    Handles:
+
+    - Converting LLMContext messages to Responses API input items
+    - Converting Pipecat's standardized tools schema to Responses API function tool format
+    - Extracting and sanitizing messages from the LLM context for logging
+    """
+
+    def __init__(self):
+        """Initialize the adapter."""
+        super().__init__()
+        self._warned_system_instruction = False
+
+    @property
+    def id_for_llm_specific_messages(self) -> str:
+        """Get the identifier used in LLMSpecificMessage instances."""
+        return "openai_responses"
+
+    def get_llm_invocation_params(
+        self,
+        context: LLMContext,
+        *,
+        system_instruction: Optional[str] = None,
+    ) -> OpenAIResponsesLLMInvocationParams:
+        """Get Responses API invocation parameters from a universal LLM context.
+
+        Args:
+            context: The LLM context containing messages, tools, etc.
+            system_instruction: Optional system instruction from service settings.
+
+        Returns:
+            Dictionary of parameters for the Responses API.
+        """
+        messages = self.get_messages(context)
+        input_items = self._convert_messages_to_input(messages)
+
+        params: OpenAIResponsesLLMInvocationParams = {
+            "input": input_items,
+            "tools": self.from_standard_tools(context.tools),
+            "tool_choice": context.tool_choice,
+        }
+
+        if system_instruction:
+            # Compatibility: The Responses API requires at least one input
+            # message when instructions are provided. Contexts that worked with
+            # OpenAILLMService (system_instruction + empty messages) need the
+            # instructions converted to an initial developer message.
+            if not input_items:
+                params["input"] = [{"role": "developer", "content": system_instruction}]
+            else:
+                params["instructions"] = system_instruction
+
+        return params
+
+    def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[FunctionToolParam]:
+        """Convert function schemas to Responses API function tool format.
+
+        Args:
+            tools_schema: The Pipecat tools schema to convert.
+
+        Returns:
+            List of Responses API function tool definitions.
+        """
+        functions_schema = tools_schema.standard_tools
+        result = []
+        for func in functions_schema:
+            d = func.to_default_dict()
+            tool: FunctionToolParam = {
+                "type": "function",
+                "name": d["name"],
+                "parameters": d.get("parameters", {}),
+                "strict": d.get("strict", None),
+            }
+            if "description" in d:
+                tool["description"] = d["description"]
+            result.append(tool)
+        return result
+
+    def get_messages_for_logging(self, context: LLMContext) -> List[Dict[str, Any]]:
+        """Get messages from context in a format ready for logging.
+
+        Removes or truncates sensitive data like image content for safe logging.
+
+        Args:
+            context: The LLM context containing messages.
+
+        Returns:
+            List of messages in a format ready for logging.
+        """
+        msgs = []
+        for message in self.get_messages(context):
+            msg = copy.deepcopy(message)
+            if "content" in msg:
+                if isinstance(msg["content"], list):
+                    for item in msg["content"]:
+                        if item.get("type") == "image_url":
+                            if item["image_url"]["url"].startswith("data:image/"):
+                                item["image_url"]["url"] = "data:image/..."
+                        if item.get("type") == "input_audio":
+                            item["input_audio"]["data"] = "..."
+            msgs.append(msg)
+        return msgs
+
+    def _convert_messages_to_input(
+        self, messages: List[LLMContextMessage]
+    ) -> List[ResponseInputItemParam]:
+        """Convert LLMContext messages to Responses API input items.
+
+        Args:
+            messages: Messages from the LLMContext.
+
+        Returns:
+            List of Responses API input items.
+        """
+        result: List[ResponseInputItemParam] = []
+        is_first = True
+
+        for message in messages:
+            if isinstance(message, LLMSpecificMessage):
+                result.append(message.message)
+                is_first = False
+                continue
+
+            role = message.get("role")
+
+            if role == "system":
+                if is_first and not self._warned_system_instruction:
+                    logger.warning(
+                        "System messages in LLMContext are converted to 'developer' role for the "
+                        "Responses API. Consider using system_instruction instead, which "
+                        "maps to the 'instructions' parameter."
+                    )
+                    self._warned_system_instruction = True
+                content = message.get("content", "")
+                if isinstance(content, list):
+                    content = self._convert_multimodal_content(content)
+                result.append({"role": "developer", "content": content})
+
+            elif role == "user":
+                content = message.get("content", "")
+                if isinstance(content, list):
+                    content = self._convert_multimodal_content(content)
+                result.append({"role": "user", "content": content})
+
+            elif role == "assistant":
+                tool_calls = message.get("tool_calls")
+                if tool_calls:
+                    for tc in tool_calls:
+                        func = tc.get("function", {})
+                        result.append(
+                            {
+                                "type": "function_call",
+                                "call_id": tc.get("id", ""),
+                                "name": func.get("name", ""),
+                                "arguments": func.get("arguments", ""),
+                            }
+                        )
+                else:
+                    content = message.get("content", "")
+                    if isinstance(content, list):
+                        content = self._convert_multimodal_content(content)
+                    result.append({"role": "assistant", "content": content})
+
+            elif role == "tool":
+                content = message.get("content", "")
+                if not isinstance(content, str):
+                    content = str(content)
+                result.append(
+                    {
+                        "type": "function_call_output",
+                        "call_id": message.get("tool_call_id", ""),
+                        "output": content,
+                    }
+                )
+
+            is_first = False
+
+        return result
+
+    def _convert_multimodal_content(self, content: list) -> list:
+        """Convert multimodal content parts to Responses API format.
+
+        Args:
+            content: List of content parts from the LLMContext message.
+
+        Returns:
+            List of content parts in Responses API format.
+        """
+        result = []
+        for part in content:
+            part_type = part.get("type")
+            if part_type == "text":
+                result.append({"type": "input_text", "text": part.get("text", "")})
+            elif part_type == "image_url":
+                image_url_obj = part.get("image_url", {})
+                result.append(
+                    {
+                        "type": "input_image",
+                        "image_url": image_url_obj.get("url", ""),
+                        "detail": image_url_obj.get("detail", "auto"),
+                    }
+                )
+            else:
+                # Pass through other types as-is (e.g. input_audio when supported)
+                result.append(part)
+        return result

{dv_pipecat_ai-0.0.93.dev2 → dv_pipecat_ai-0.0.93.dev3}/src/pipecat/processors/aggregators/llm_response_universal.py

@@ -13,6 +13,7 @@ LLM processing, and text-to-speech components in conversational AI pipelines.
 
 import asyncio
 import json
+import time
 from abc import abstractmethod
 from typing import Any, Dict, List, Literal, Optional, Set
 
@@ -39,6 +40,7 @@ from pipecat.frames.frames import (
     InterruptionFrame,
     LLMContextAssistantTimestampFrame,
     LLMContextFrame,
+    LLMProcessingInitiatedFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesAppendFrame,
@@ -50,6 +52,7 @@ from pipecat.frames.frames import (
     StartFrame,
     TextFrame,
     TranscriptionFrame,
+    TranscriptDropFrame,
     TTSTextFrame,
     UserImageRawFrame,
     UserStartedSpeakingFrame,
@@ -132,6 +135,7 @@ class LLMContextAggregator(FrameProcessor):
         Args:
             direction: The direction to push the frame (upstream or downstream).
         """
+        await self.push_frame(LLMProcessingInitiatedFrame(), direction)
         frame = self._get_context_frame()
         await self.push_frame(frame, direction)
 
@@ -234,12 +238,21 @@ class LLMUserAggregator(LLMContextAggregator):
         self._aggregation_event = asyncio.Event()
         self._aggregation_task = None
 
+        # Smart aggregation push state — enables immediate push of late
+        # transcriptions and prevents duplicate STT outputs from triggering
+        # duplicate LLM calls.
+        self._latest_final_transcript = ""
+        self._last_user_speaking_time: float = 0
+        self._last_aggregation_push_time: float = 0
+        self._pending_transcription_ids: List[int] = []
+
     async def reset(self):
         """Reset the aggregation state and interruption strategies."""
         await super().reset()
         self._was_bot_speaking = False
         self._seen_interim_results = False
         self._waiting_for_aggregation = False
+        self._pending_transcription_ids.clear()
         [await s.reset() for s in self._interruption_strategies]
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -304,12 +317,22 @@ class LLMUserAggregator(LLMContextAggregator):
         """Process the current aggregation and push it downstream."""
         aggregation = self._aggregation
         await self.reset()
+        # Notify BotSpeakingFrameMonitor to tighten VAD before LLM starts
+        await self.push_frame(LLMProcessingInitiatedFrame())
         self._context.add_message({"role": self.role, "content": aggregation})
         frame = LLMContextFrame(self._context)
         await self.push_frame(frame)
+        self._last_aggregation_push_time = time.time()
 
-    async def push_aggregation(self):
-        """Push the current aggregation based on interruption strategies and conditions."""
+    async def push_aggregation(self, trigger_interruption: bool = False):
+        """Push the current aggregation based on interruption strategies and conditions.
+
+        Args:
+            trigger_interruption: If True and no interruption strategies are
+                configured, push an InterruptionFrame to stop bot speech before
+                submitting the user message. Used by smart push logic for late
+                transcriptions that should interrupt the bot.
+        """
         if len(self._aggregation) > 0:
             if self.interruption_strategies and self._bot_speaking:
                 should_interrupt = await self._should_interrupt_based_on_strategies()
@@ -322,9 +345,12 @@ class LLMUserAggregator(LLMContextAggregator):
                     await self._process_aggregation()
                 else:
                     logger.debug("Interruption conditions not met - not pushing aggregation")
-                    # Don't process aggregation, just reset it
+                    # Don't process aggregation, discard pending transcriptions and reset
+                    await self._discard_pending_transcriptions("interruption_conditions_not_met")
                     await self.reset()
             else:
+                if trigger_interruption:
+                    await self.push_frame(InterruptionFrame(), FrameDirection.DOWNSTREAM)
                 # No interruption config - normal behavior (always push aggregation)
                 await self._process_aggregation()
         # Handles the case where both the user and the bot are not speaking,
@@ -355,6 +381,21 @@ class LLMUserAggregator(LLMContextAggregator):
 
         return any([await should_interrupt(s) for s in self._interruption_strategies])
 
+    async def _discard_pending_transcriptions(self, reason: str):
+        """Notify upstream processors that pending transcripts should be dropped.
+
+        When interruption strategies reject user speech, the words were already
+        emitted to the transcript. This sends TranscriptDropFrame upstream so
+        TranscriptProcessor removes those words from the conversation transcript.
+
+        Args:
+            reason: Descriptive reason for the drop (for logging).
+        """
+        if self._pending_transcription_ids:
+            drop_frame = TranscriptDropFrame(transcript_ids=list(self._pending_transcription_ids))
+            await self.push_frame(drop_frame, FrameDirection.UPSTREAM)
+        self._pending_transcription_ids.clear()
+
     async def _start(self, frame: StartFrame):
         self._create_aggregation_task()
 
@@ -382,6 +423,13 @@ class LLMUserAggregator(LLMContextAggregator):
             await s.append_audio(frame.audio, frame.sample_rate)
 
     async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame):
+        # Clear stale aggregation from previous utterance to prevent it from
+        # leaking into the new one. Also drop pending transcript IDs upstream.
+        if len(self._aggregation) > 0:
+            self._aggregation = ""
+            await self._discard_pending_transcriptions("user_started_speaking")
+        self._latest_final_transcript = ""
+        self._last_user_speaking_time = time.time()
         self._user_speaking = True
         self._waiting_for_aggregation = True
         self._was_bot_speaking = self._bot_speaking
@@ -393,6 +441,7 @@ class LLMUserAggregator(LLMContextAggregator):
             self._emulating_vad = False
 
     async def _handle_user_stopped_speaking(self, _: UserStoppedSpeakingFrame):
+        self._last_user_speaking_time = time.time()
         self._user_speaking = False
         # We just stopped speaking. Let's see if there's some aggregation to
         # push. If the last thing we saw is an interim transcription, let's wait
@@ -421,17 +470,64 @@ class LLMUserAggregator(LLMContextAggregator):
             return
 
         self._aggregation += f" {text}" if self._aggregation else text
+        self._pending_transcription_ids.append(frame.id)
         # We just got a final result, so let's reset interim results.
         self._seen_interim_results = False
-        # Reset aggregation timer.
-        self._aggregation_event.set()
+
+        # Smart aggregation push: when the user is NOT speaking (VAD stopped),
+        # check if this late transcription should be pushed immediately rather
+        # than waiting for the background task timeout (saves 200-500ms TTFB).
+        if not self._user_speaking:
+            time_since_stopped = time.time() - self._last_user_speaking_time
+            should_push = False
+            should_interrupt = False
+
+            # Condition 1: User stopped < 1s ago + new (non-duplicate) text.
+            # Covers normal late transcription delivery from STT.
+            if time_since_stopped < 1.0 and self._latest_final_transcript != text:
+                # If we also pushed aggregation < 1s ago, this is a rapid
+                # correction/addition — interrupt the bot's stale response.
+                if time.time() - self._last_aggregation_push_time < 1.0:
+                    should_interrupt = True
+                should_push = True
+            # Condition 2: User stopped 1-3s ago + bot silent + new text.
+            # Covers very slow STT delivery when nobody is talking.
+            elif (
+                not self._bot_speaking
+                and time_since_stopped < 3.0
+                and self._latest_final_transcript != text
+            ):
+                if time.time() - self._last_aggregation_push_time < 3.0:
+                    should_interrupt = True
+                should_push = True
+            # Condition 3: Frame metadata override from STT or upstream processor.
+            elif frame.metadata.get("push_aggregation", False):
+                should_push = True
+
+            if should_push:
+                if should_interrupt:
+                    # Single-word filter: don't interrupt bot for backchannel
+                    # utterances like "ok", "hmm", "yes".
+                    if len(self._aggregation.split()) > 1:
+                        await self.push_aggregation(should_interrupt)
+                else:
+                    await self.push_aggregation()
+            else:
+                # No immediate push — fall back to background task timeout.
+                self._aggregation_event.set()
+        else:
+            # User is still speaking — just reset the aggregation timer.
+            self._aggregation_event.set()
+
+        self._latest_final_transcript = text
 
     async def _handle_interim_transcription(self, _: InterimTranscriptionFrame):
         self._seen_interim_results = True
 
     def _create_aggregation_task(self):
-        if not self._aggregation_task:
-            self._aggregation_task = self.create_task(self._aggregation_task_handler())
+        pass
+        # if not self._aggregation_task:
+        #     self._aggregation_task = self.create_task(self._aggregation_task_handler())
 
     async def _cancel_aggregation_task(self):
         if self._aggregation_task:
@@ -510,6 +606,7 @@ class LLMUserAggregator(LLMContextAggregator):
             if self._bot_speaking and not self._params.enable_emulated_vad_interruptions:
                 # If emulated VAD interruptions are disabled and bot is speaking, ignore
                 logger.debug("Ignoring user speaking emulation, bot is speaking.")
+                await self._discard_pending_transcriptions("emulated_vad_ignored")
                 await self.reset()
             else:
                 # Either bot is not speaking, or emulated VAD interruptions are enabled

{dv_pipecat_ai-0.0.93.dev2 → dv_pipecat_ai-0.0.93.dev3}/src/pipecat/services/openai/__init__.py

@@ -11,6 +11,7 @@ from pipecat.services import DeprecatedModuleProxy
 from .image import *
 from .llm import *
 from .realtime import *
+from .responses.llm import *
 from .stt import *
 from .tts import *