dv-pipecat-ai 0.0.82.dev815__py3-none-any.whl → 0.0.82.dev857__py3-none-any.whl

pipecat/adapters/services/gemini_adapter.py

@@ -6,20 +6,71 @@
 
 """Gemini LLM adapter for Pipecat."""
 
-from typing import Any, Dict, List, Union
+import base64
+import json
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, TypedDict
+
+from loguru import logger
+from openai import NotGiven
 
 from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
 from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
+from pipecat.processors.aggregators.llm_context import (
+    LLMContext,
+    LLMContextMessage,
+    LLMSpecificMessage,
+    LLMStandardMessage,
+)
+
+try:
+    from google.genai.types import (
+        Blob,
+        Content,
+        FunctionCall,
+        FunctionResponse,
+        Part,
+    )
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error("In order to use Google AI, you need to `pip install pipecat-ai[google]`.")
+    raise Exception(f"Missing module: {e}")
+
+
+class GeminiLLMInvocationParams(TypedDict):
+    """Context-based parameters for invoking Gemini LLM."""
 
+    system_instruction: Optional[str]
+    messages: List[Content]
+    tools: List[Any] | NotGiven
 
-class GeminiLLMAdapter(BaseLLMAdapter):
-    """LLM adapter for Google's Gemini service.
 
-    Provides tool schema conversion functionality to transform standard tool
-    definitions into Gemini's specific function-calling format for use with
-    Gemini LLM models.
+class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
+    """Gemini-specific adapter for Pipecat.
+
+    Handles:
+    - Extracting parameters for Gemini's API from a universal LLM context
+    - Converting Pipecat's standardized tools schema to Gemini's function-calling format.
+    - Extracting and sanitizing messages from the LLM context for logging with Gemini.
     """
 
+    def get_llm_invocation_params(self, context: LLMContext) -> GeminiLLMInvocationParams:
+        """Get Gemini-specific LLM invocation parameters from a universal LLM context.
+
+        Args:
+            context: The LLM context containing messages, tools, etc.
+
+        Returns:
+            Dictionary of parameters for Gemini's API.
+        """
+        messages = self._from_universal_context_messages(self._get_messages(context))
+        return {
+            "system_instruction": messages.system_instruction,
+            "messages": messages.messages,
+            # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
+            "tools": self.from_standard_tools(context.tools),
+        }
+
     def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
         """Convert tool schemas to Gemini's function-calling format.
 
@@ -39,3 +90,222 @@ class GeminiLLMAdapter(BaseLLMAdapter):
             custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, [])
 
         return formatted_standard_tools + custom_gemini_tools
+
+    def get_messages_for_logging(self, context: LLMContext) -> List[Dict[str, Any]]:
+        """Get messages from a universal LLM context in a format ready for logging about Gemini.
+
+        Removes or truncates sensitive data like image content for safe logging.
+
+        Args:
+            context: The LLM context containing messages.
+
+        Returns:
+            List of messages in a format ready for logging about Gemini.
+        """
+        # Get messages in Gemini's format
+        messages = self._from_universal_context_messages(self._get_messages(context)).messages
+
+        # Sanitize messages for logging
+        messages_for_logging = []
+        for message in messages:
+            obj = message.to_json_dict()
+            try:
+                if "parts" in obj:
+                    for part in obj["parts"]:
+                        if "inline_data" in part:
+                            part["inline_data"]["data"] = "..."
+            except Exception as e:
+                logger.debug(f"Error: {e}")
+            messages_for_logging.append(obj)
+        return messages_for_logging
+
+    def _get_messages(self, context: LLMContext) -> List[LLMContextMessage]:
+        return context.get_messages("google")
+
+    @dataclass
+    class ConvertedMessages:
+        """Container for Google-formatted messages converted from universal context."""
+
+        messages: List[Content]
+        system_instruction: Optional[str] = None
+
+    def _from_universal_context_messages(
+        self, universal_context_messages: List[LLMContextMessage]
+    ) -> ConvertedMessages:
+        """Restructures messages to ensure proper Google format and message ordering.
+
+        This method handles conversion of OpenAI-formatted messages to Google format,
+        with special handling for function calls, function responses, and system messages.
+        System messages are added back to the context as user messages when needed.
+
+        The final message order is preserved as:
+
+        1. Function calls (from model)
+        2. Function responses (from user)
+        3. Text messages (converted from system messages)
+
+        Note::
+
+            System messages are only added back when there are no regular text
+            messages in the context, ensuring proper conversation continuity
+            after function calls.
+        """
+        system_instruction = None
+        messages = []
+
+        # Process each message, preserving Google-formatted messages and converting others
+        for message in universal_context_messages:
+            if isinstance(message, LLMSpecificMessage):
+                # Assume that LLMSpecificMessage wraps a message in Google format
+                messages.append(message.message)
+                continue
+
+            # Convert standard format to Google format
+            converted = self._from_standard_message(
+                message, already_have_system_instruction=bool(system_instruction)
+            )
+            if isinstance(converted, Content):
+                # Regular (non-system) message
+                messages.append(converted)
+            else:
+                # System instruction
+                system_instruction = converted
+
+        # Check if we only have function-related messages (no regular text)
+        has_regular_messages = any(
+            len(msg.parts) == 1
+            and getattr(msg.parts[0], "text", None)
+            and not getattr(msg.parts[0], "function_call", None)
+            and not getattr(msg.parts[0], "function_response", None)
+            for msg in messages
+        )
+
+        # Add system instruction back as a user message if we only have function messages
+        if system_instruction and not has_regular_messages:
+            messages.append(Content(role="user", parts=[Part(text=system_instruction)]))
+
+        # Remove any empty messages
+        messages = [m for m in messages if m.parts]
+
+        return self.ConvertedMessages(messages=messages, system_instruction=system_instruction)
+
+    def _from_standard_message(
+        self, message: LLMStandardMessage, already_have_system_instruction: bool
+    ) -> Content | str:
+        """Convert standard universal context message to Google Content object.
+
+        Handles conversion of text, images, and function calls to Google's
+        format.
+        System instructions are returned as a plain string.
+
+        Args:
+            message: Message in standard universal context format.
+            already_have_system_instruction: Whether we already have a system instruction
+
+        Returns:
+            Content object with role and parts, or a plain string for system
+            messages.
+
+        Examples:
+            Standard text message::
+
+                {
+                    "role": "user",
+                    "content": "Hello there"
+                }
+
+            Converts to Google Content with::
+
+                Content(
+                    role="user",
+                    parts=[Part(text="Hello there")]
+                )
+
+            Standard function call message::
+
+                {
+                    "role": "assistant",
+                    "tool_calls": [
+                        {
+                            "function": {
+                                "name": "search",
+                                "arguments": '{"query": "test"}'
+                            }
+                        }
+                    ]
+                }
+
+            Converts to Google Content with::
+
+                Content(
+                    role="model",
+                    parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))]
+                )
+        """
+        role = message["role"]
+        content = message.get("content", [])
+        if role == "system":
+            if already_have_system_instruction:
+                role = "user"  # Convert system message to user role if we already have a system instruction
+            else:
+                # System instructions are returned as plain text
+                if isinstance(content, str):
+                    return content
+                elif isinstance(content, list):
+                    # If content is a list, we assume it's a list of text parts, per the standard
+                    return " ".join(part["text"] for part in content if part.get("type") == "text")
+        elif role == "assistant":
+            role = "model"
+
+        parts = []
+        if message.get("tool_calls"):
+            for tc in message["tool_calls"]:
+                parts.append(
+                    Part(
+                        function_call=FunctionCall(
+                            name=tc["function"]["name"],
+                            args=json.loads(tc["function"]["arguments"]),
+                        )
+                    )
+                )
+        elif role == "tool":
+            role = "model"
+            try:
+                response = json.loads(message["content"])
+                if isinstance(response, dict):
+                    response_dict = response
+                else:
+                    response_dict = {"value": response}
+            except Exception as e:
+                # Response might not be JSON-deserializable.
+                # This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string.
+                response_dict = {"value": message["content"]}
+            parts.append(
+                Part(
+                    function_response=FunctionResponse(
+                        name="tool_call_result",  # seems to work to hard-code the same name every time
+                        response=response_dict,
+                    )
+                )
+            )
+        elif isinstance(content, str):
+            parts.append(Part(text=content))
+        elif isinstance(content, list):
+            for c in content:
+                if c["type"] == "text":
+                    parts.append(Part(text=c["text"]))
+                elif c["type"] == "image_url":
+                    parts.append(
+                        Part(
+                            inline_data=Blob(
+                                mime_type="image/jpeg",
+                                data=base64.b64decode(c["image_url"]["url"].split(",")[1]),
+                            )
+                        )
+                    )
+                elif c["type"] == "input_audio":
+                    input_audio = c["input_audio"]
+                    audio_bytes = base64.b64decode(input_audio["data"])
+                    parts.append(Part(inline_data=Blob(mime_type="audio/wav", data=audio_bytes)))
+
+        return Content(role=role, parts=parts)

pipecat/adapters/services/open_ai_adapter.py

@@ -6,22 +6,63 @@
 
 """OpenAI LLM adapter for Pipecat."""
 
-from typing import List
+import copy
+import json
+from typing import Any, Dict, List, TypedDict
 
-from openai.types.chat import ChatCompletionToolParam
+from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN
+from openai._types import NotGiven as OpenAINotGiven
+from openai.types.chat import (
+    ChatCompletionMessageParam,
+    ChatCompletionToolChoiceOptionParam,
+    ChatCompletionToolParam,
+)
 
 from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.processors.aggregators.llm_context import (
+    LLMContext,
+    LLMContextMessage,
+    LLMContextToolChoice,
+    NotGiven,
+)
 
 
-class OpenAILLMAdapter(BaseLLMAdapter):
-    """Adapter for converting tool schemas to OpenAI's format.
+class OpenAILLMInvocationParams(TypedDict):
+    """Context-based parameters for invoking OpenAI ChatCompletion API."""
 
-    Provides conversion utilities for transforming Pipecat's standard tool
-    schemas into the format expected by OpenAI's ChatCompletion API for
-    function calling capabilities.
+    messages: List[ChatCompletionMessageParam]
+    tools: List[ChatCompletionToolParam] | OpenAINotGiven
+    tool_choice: ChatCompletionToolChoiceOptionParam | OpenAINotGiven
+
+
+class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
+    """OpenAI-specific adapter for Pipecat.
+
+    Handles:
+
+    - Extracting parameters for OpenAI's ChatCompletion API from a universal
+      LLM context
+    - Converting Pipecat's standardized tools schema to OpenAI's function-calling format.
+    - Extracting and sanitizing messages from the LLM context for logging about OpenAI.
     """
 
+    def get_llm_invocation_params(self, context: LLMContext) -> OpenAILLMInvocationParams:
+        """Get OpenAI-specific LLM invocation parameters from a universal LLM context.
+
+        Args:
+            context: The LLM context containing messages, tools, etc.
+
+        Returns:
+            Dictionary of parameters for OpenAI's ChatCompletion API.
+        """
+        return {
+            "messages": self._from_universal_context_messages(self._get_messages(context)),
+            # NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
+            "tools": self.from_standard_tools(context.tools),
+            "tool_choice": context.tool_choice,
+        }
+
     def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ChatCompletionToolParam]:
         """Convert function schemas to OpenAI's function-calling format.
 
@@ -37,3 +78,43 @@ class OpenAILLMAdapter(BaseLLMAdapter):
             ChatCompletionToolParam(type="function", function=func.to_default_dict())
             for func in functions_schema
         ]
+
+    def get_messages_for_logging(self, context: LLMContext) -> List[Dict[str, Any]]:
+        """Get messages from a universal LLM context in a format ready for logging about OpenAI.
+
+        Removes or truncates sensitive data like image content for safe logging.
+
+        Args:
+            context: The LLM context containing messages.
+
+        Returns:
+            List of messages in a format ready for logging about OpenAI.
+        """
+        msgs = []
+        for message in self._get_messages(context):
+            msg = copy.deepcopy(message)
+            if "content" in msg:
+                if isinstance(msg["content"], list):
+                    for item in msg["content"]:
+                        if item["type"] == "image_url":
+                            if item["image_url"]["url"].startswith("data:image/"):
+                                item["image_url"]["url"] = "data:image/..."
+            if "mime_type" in msg and msg["mime_type"].startswith("image/"):
+                msg["data"] = "..."
+            msgs.append(msg)
+        return msgs
+
+    def _get_messages(self, context: LLMContext) -> List[LLMContextMessage]:
+        return context.get_messages("openai")
+
+    def _from_universal_context_messages(
+        self, messages: List[LLMContextMessage]
+    ) -> List[ChatCompletionMessageParam]:
+        # Just a pass-through: messages are already the right type
+        return messages
+
+    def _from_standard_tool_choice(
+        self, tool_choice: LLMContextToolChoice | NotGiven
+    ) -> ChatCompletionToolChoiceOptionParam | OpenAINotGiven:
+        # Just a pass-through: tool_choice is already the right type
+        return tool_choice

pipecat/adapters/services/open_ai_realtime_adapter.py

@@ -6,11 +6,21 @@
 
 """OpenAI Realtime LLM adapter for Pipecat."""
 
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, TypedDict
 
 from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
 from pipecat.adapters.schemas.function_schema import FunctionSchema
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.processors.aggregators.llm_context import LLMContext
+
+
+class OpenAIRealtimeLLMInvocationParams(TypedDict):
+    """Context-based parameters for invoking OpenAI Realtime API.
+
+    This is a placeholder until support for universal LLMContext machinery is added for OpenAI Realtime.
+    """
+
+    pass
 
 
 class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
@@ -20,6 +30,34 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
     OpenAI's Realtime API for function calling capabilities.
     """
 
+    def get_llm_invocation_params(self, context: LLMContext) -> OpenAIRealtimeLLMInvocationParams:
+        """Get OpenAI Realtime-specific LLM invocation parameters from a universal LLM context.
+
+        This is a placeholder until support for universal LLMContext machinery is added for OpenAI Realtime.
+
+        Args:
+            context: The LLM context containing messages, tools, etc.
+
+        Returns:
+            Dictionary of parameters for invoking OpenAI Realtime's API.
+        """
+        raise NotImplementedError("Universal LLMContext is not yet supported for OpenAI Realtime.")
+
+    def get_messages_for_logging(self, context) -> List[Dict[str, Any]]:
+        """Get messages from a universal LLM context in a format ready for logging about OpenAI Realtime.
+
+        Removes or truncates sensitive data like image content for safe logging.
+
+        This is a placeholder until support for universal LLMContext machinery is added for OpenAI Realtime.
+
+        Args:
+            context: The LLM context containing messages.
+
+        Returns:
+            List of messages in a format ready for logging about OpenAI Realtime.
+        """
+        raise NotImplementedError("Universal LLMContext is not yet supported for OpenAI Realtime.")
+
     @staticmethod
     def _to_openai_realtime_function_format(function: FunctionSchema) -> Dict[str, Any]:
         """Convert a function schema to OpenAI Realtime format.

pipecat/audio/dtmf/types.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""This module defines generic type for DTMS.
+
+It defines the `KeypadEntry` enumeration, representing dual-tone multi-frequency
+(DTMF) keypad entries for phone system integration. Each entry corresponds to a
+key on the telephone keypad, facilitating the handling of input in
+telecommunication applications.
+"""
+
+from enum import Enum
+
+
+class KeypadEntry(str, Enum):
+    """DTMF keypad entries for phone system integration.
+
+    Parameters:
+        ONE: Number key 1.
+        TWO: Number key 2.
+        THREE: Number key 3.
+        FOUR: Number key 4.
+        FIVE: Number key 5.
+        SIX: Number key 6.
+        SEVEN: Number key 7.
+        EIGHT: Number key 8.
+        NINE: Number key 9.
+        ZERO: Number key 0.
+        POUND: Pound/hash key (#).
+        STAR: Star/asterisk key (*).
+    """
+
+    ONE = "1"
+    TWO = "2"
+    THREE = "3"
+    FOUR = "4"
+    FIVE = "5"
+    SIX = "6"
+    SEVEN = "7"
+    EIGHT = "8"
+    NINE = "9"
+    ZERO = "0"
+
+    POUND = "#"
+    STAR = "*"

pipecat/audio/dtmf/utils.py

@@ -0,0 +1,70 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""DTMF audio utilities.
+
+This module provides functionality to load DTMF (Dual-Tone Multi-Frequency)
+audio files corresponding to phone keypad entries. Audio data is cached
+in-memory after first load to improve performance on subsequent accesses.
+"""
+
+import asyncio
+import io
+import wave
+from importlib.resources import files
+from typing import Dict, Optional
+
+import aiofiles
+
+from pipecat.audio.dtmf.types import KeypadEntry
+from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
+from pipecat.audio.utils import create_file_resampler
+
+__DTMF_LOCK__ = asyncio.Lock()
+__DTMF_AUDIO__: Dict[KeypadEntry, bytes] = {}
+__DTMF_RESAMPLER__: Optional[BaseAudioResampler] = None
+
+__DTMF_FILE_NAME = {
+    KeypadEntry.POUND: "dtmf-pound.wav",
+    KeypadEntry.STAR: "dtmf-star.wav",
+}
+
+
+async def load_dtmf_audio(button: KeypadEntry, *, sample_rate: int = 8000) -> bytes:
+    """Load audio for DTMF tones associated with the given button.
+
+    Args:
+        button (KeypadEntry): The button for which the DTMF audio is to be loaded.
+        sample_rate (int, optional): The sample rate for the audio. Defaults to 8000.
+
+    Returns:
+        bytes: The audio data for the DTMF tone as bytes.
+    """
+    global __DTMF_AUDIO__, __DTMF_RESAMPLER__
+
+    async with __DTMF_LOCK__:
+        if button in __DTMF_AUDIO__:
+            return __DTMF_AUDIO__[button]
+
+        if not __DTMF_RESAMPLER__:
+            __DTMF_RESAMPLER__ = create_file_resampler()
+
+        dtmf_file_name = __DTMF_FILE_NAME.get(button, f"dtmf-{button.value}.wav")
+        dtmf_file_path = files("pipecat.audio.dtmf").joinpath(dtmf_file_name)
+
+        async with aiofiles.open(dtmf_file_path, "rb") as f:
+            data = await f.read()
+
+        with io.BytesIO(data) as buffer:
+            with wave.open(buffer, "rb") as wf:
+                audio = wf.readframes(wf.getnframes())
+                in_sample_rate = wf.getframerate()
+                resampled_audio = await __DTMF_RESAMPLER__.resample(
+                    audio, in_sample_rate, sample_rate
+                )
+                __DTMF_AUDIO__[button] = resampled_audio
+
+    return __DTMF_AUDIO__[button]