dv-pipecat-ai 0.0.74.dev770__py3-none-any.whl → 0.0.82.dev776__py3-none-any.whl

pipecat/services/anthropic/llm.py

@@ -26,6 +26,7 @@ from pydantic import BaseModel, Field
 
 from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter
 from pipecat.frames.frames import (
+    ErrorFrame,
     Frame,
     FunctionCallCancelFrame,
     FunctionCallInProgressFrame,
@@ -52,11 +53,10 @@ from pipecat.processors.aggregators.openai_llm_context import (
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
-from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator
 from pipecat.utils.tracing.service_decorators import traced_llm
 
 try:
-    from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven
+    from anthropic import NOT_GIVEN, APITimeoutError, AsyncAnthropic, NotGiven
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error("In order to use Anthropic, you need to `pip install pipecat-ai[anthropic]`.")
@@ -101,13 +101,6 @@ class AnthropicLLMService(LLMService):
     Provides inference capabilities with Claude models including support for
     function calling, vision processing, streaming responses, and prompt caching.
     Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex.
-
-    Args:
-        api_key: Anthropic API key for authentication.
-        model: Model name to use. Defaults to "claude-sonnet-4-20250514".
-        params: Optional model parameters for inference.
-        client: Optional custom Anthropic client instance.
-        **kwargs: Additional arguments passed to parent LLMService.
     """
 
     # Overriding the default adapter to use the Anthropic one.
@@ -139,14 +132,29 @@ class AnthropicLLMService(LLMService):
         model: str = "claude-sonnet-4-20250514",
         params: Optional[InputParams] = None,
         client=None,
+        retry_timeout_secs: Optional[float] = 5.0,
+        retry_on_timeout: Optional[bool] = False,
         **kwargs,
     ):
+        """Initialize the Anthropic LLM service.
+
+        Args:
+            api_key: Anthropic API key for authentication.
+            model: Model name to use. Defaults to "claude-sonnet-4-20250514".
+            params: Optional model parameters for inference.
+            client: Optional custom Anthropic client instance.
+            retry_timeout_secs: Request timeout in seconds for retry logic.
+            retry_on_timeout: Whether to retry the request once if it times out.
+            **kwargs: Additional arguments passed to parent LLMService.
+        """
         super().__init__(**kwargs)
         params = params or AnthropicLLMService.InputParams()
         self._client = client or AsyncAnthropic(
             api_key=api_key
         )  # if the client is provided, use it and remove it, otherwise create a new one
         self.set_model_name(model)
+        self._retry_timeout_secs = retry_timeout_secs
+        self._retry_on_timeout = retry_on_timeout
         self._settings = {
             "max_tokens": params.max_tokens,
             "enable_prompt_caching_beta": params.enable_prompt_caching_beta or False,
@@ -164,6 +172,31 @@ class AnthropicLLMService(LLMService):
         """
         return True
 
+    async def _create_message_stream(self, api_call, params):
+        """Create message stream with optional timeout and retry.
+
+        Args:
+            api_call: The Anthropic API method to call.
+            params: Parameters for the API call.
+
+        Returns:
+            Async stream of message events.
+        """
+        if self._retry_on_timeout:
+            try:
+                response = await asyncio.wait_for(
+                    api_call(**params), timeout=self._retry_timeout_secs
+                )
+                return response
+            except (APITimeoutError, asyncio.TimeoutError):
+                # Retry, this time without a timeout so we get a response
+                logger.debug(f"{self}: Retrying message creation due to timeout")
+                response = await api_call(**params)
+                return response
+        else:
+            response = await api_call(**params)
+            return response
+
     @property
     def enable_prompt_caching_beta(self) -> bool:
         """Check if prompt caching beta feature is enabled.
@@ -247,7 +280,7 @@ class AnthropicLLMService(LLMService):
 
             params.update(self._settings["extra"])
 
-            response = await api_call(**params)
+            response = await self._create_message_stream(api_call, params)
 
             await self.stop_ttfb_metrics()
 
@@ -256,7 +289,7 @@ class AnthropicLLMService(LLMService):
             json_accumulator = ""
 
             function_calls = []
-            async for event in WatchdogAsyncIterator(response, manager=self.task_manager):
+            async for event in response:
                 # Aggregate streaming content, create frames, trigger events
 
                 if event.type == "content_block_delta":
@@ -344,6 +377,7 @@ class AnthropicLLMService(LLMService):
             await self._call_event_handler("on_completion_timeout")
         except Exception as e:
             self.logger.exception(f"{self} exception: {e}")
+            await self.push_error(ErrorFrame(f"{e}"))
         finally:
             await self.stop_processing_metrics()
             await self.push_frame(LLMFullResponseEndFrame())
@@ -425,12 +459,6 @@ class AnthropicLLMContext(OpenAILLMContext):
     Extends OpenAILLMContext to handle Anthropic-specific features like
     system messages, prompt caching, and message format conversions.
     Manages conversation state and message history formatting.
-
-    Args:
-        messages: Initial list of conversation messages.
-        tools: Available function calling tools.
-        tool_choice: Tool selection preference.
-        system: System message content.
     """
 
     def __init__(
@@ -441,15 +469,25 @@ class AnthropicLLMContext(OpenAILLMContext):
         *,
         system: Union[str, NotGiven] = NOT_GIVEN,
     ):
+        """Initialize the Anthropic LLM context.
+
+        Args:
+            messages: Initial list of conversation messages.
+            tools: Available function calling tools.
+            tool_choice: Tool selection preference.
+            system: System message content.
+        """
         super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
+        self.__setup_local()
+        self.system = system
 
+    def __setup_local(self):
         # For beta prompt caching. This is a counter that tracks the number of turns
         # we've seen above the cache threshold. We reset this when we reset the
         # messages list. We only care about this number being 0, 1, or 2. But
         # it's easiest just to treat it as a counter.
         self.turns_above_cache_threshold = 0
-
-        self.system = system
+        return
 
     @staticmethod
     def upgrade_to_anthropic(obj: OpenAILLMContext) -> "AnthropicLLMContext":
@@ -466,6 +504,7 @@ class AnthropicLLMContext(OpenAILLMContext):
         logger.debug(f"Upgrading to Anthropic: {obj}")
         if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AnthropicLLMContext):
             obj.__class__ = AnthropicLLMContext
+            obj.__setup_local()
             obj._restructure_from_openai_messages()
         return obj
 
@@ -534,20 +573,37 @@ class AnthropicLLMContext(OpenAILLMContext):
         Handles text content and function calls for both user and assistant messages.
 
         Args:
-            obj: Message in Anthropic format:
-                {
-                    "role": "user/assistant",
-                    "content": str | [{"type": "text/tool_use/tool_result", ...}]
-                }
+            obj: Message in Anthropic format.
 
         Returns:
-            List of messages in standard format:
-            [
+            List of messages in standard format.
+
+        Examples:
+            Input Anthropic format::
+
                 {
-                    "role": "user/assistant/tool",
-                    "content": [{"type": "text", "text": str}]
+                    "role": "assistant",
+                    "content": [
+                        {"type": "text", "text": "Hello"},
+                        {"type": "tool_use", "id": "123", "name": "search", "input": {"q": "test"}}
+                    ]
                 }
-            ]
+
+            Output standard format::
+
+                [
+                    {"role": "assistant", "content": [{"type": "text", "text": "Hello"}]},
+                    {
+                        "role": "assistant",
+                        "tool_calls": [
+                            {
+                                "type": "function",
+                                "id": "123",
+                                "function": {"name": "search", "arguments": '{"q": "test"}'}
+                            }
+                        ]
+                    }
+                ]
         """
         # todo: image format (?)
         # tool_use
@@ -609,23 +665,37 @@ class AnthropicLLMContext(OpenAILLMContext):
         Empty text content is converted to "(empty)".
 
         Args:
-            message: Message in standard format:
+            message: Message in standard format.
+
+        Returns:
+            Message in Anthropic format.
+
+        Examples:
+            Input standard format::
+
                 {
-                    "role": "user/assistant/tool",
-                    "content": str | [{"type": "text", ...}],
-                    "tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}]
+                    "role": "assistant",
+                    "tool_calls": [
+                        {
+                            "id": "123",
+                            "function": {"name": "search", "arguments": '{"q": "test"}'}
+                        }
+                    ]
                 }
 
-        Returns:
-            Message in Anthropic format:
-            {
-                "role": "user/assistant",
-                "content": str | [
-                    {"type": "text", "text": str} |
-                    {"type": "tool_use", "id": str, "name": str, "input": dict} |
-                    {"type": "tool_result", "tool_use_id": str, "content": str}
-                ]
-            }
+            Output Anthropic format::
+
+                {
+                    "role": "assistant",
+                    "content": [
+                        {
+                            "type": "tool_use",
+                            "id": "123",
+                            "name": "search",
+                            "input": {"q": "test"}
+                        }
+                    ]
+                }
         """
         # todo: image messages (?)
         if message["role"] == "tool":

pipecat/services/assemblyai/models.py

@@ -1,10 +1,30 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""AssemblyAI WebSocket API message models and connection parameters.
+
+This module defines Pydantic models for handling AssemblyAI's real-time
+transcription WebSocket messages and connection configuration.
+"""
+
 from typing import List, Literal, Optional
 
 from pydantic import BaseModel, Field
 
 
 class Word(BaseModel):
-    """Represents a single word in a transcription with timing and confidence."""
+    """Represents a single word in a transcription with timing and confidence.
+
+    Parameters:
+        start: Start time of the word in milliseconds.
+        end: End time of the word in milliseconds.
+        text: The transcribed word text.
+        confidence: Confidence score for the word (0.0 to 1.0).
+        word_is_final: Whether this word is finalized and won't change.
+    """
 
     start: int
     end: int
@@ -14,13 +34,23 @@ class Word(BaseModel):
 
 
 class BaseMessage(BaseModel):
-    """Base class for all AssemblyAI WebSocket messages."""
+    """Base class for all AssemblyAI WebSocket messages.
+
+    Parameters:
+        type: The message type identifier.
+    """
 
     type: str
 
 
 class BeginMessage(BaseMessage):
-    """Message sent when a new session begins."""
+    """Message sent when a new session begins.
+
+    Parameters:
+        type: Always "Begin" for this message type.
+        id: Unique session identifier.
+        expires_at: Unix timestamp when the session expires.
+    """
 
     type: Literal["Begin"] = "Begin"
     id: str
@@ -28,7 +58,17 @@ class BeginMessage(BaseMessage):
 
 
 class TurnMessage(BaseMessage):
-    """Message containing transcription data for a turn of speech."""
+    """Message containing transcription data for a turn of speech.
+
+    Parameters:
+        type: Always "Turn" for this message type.
+        turn_order: Sequential number of this turn in the session.
+        turn_is_formatted: Whether the transcript has been formatted.
+        end_of_turn: Whether this marks the end of a speaking turn.
+        transcript: The transcribed text for this turn.
+        end_of_turn_confidence: Confidence score for end-of-turn detection.
+        words: List of individual words with timing and confidence data.
+    """
 
     type: Literal["Turn"] = "Turn"
     turn_order: int
@@ -40,7 +80,13 @@ class TurnMessage(BaseMessage):
 
 
 class TerminationMessage(BaseMessage):
-    """Message sent when the session is terminated."""
+    """Message sent when the session is terminated.
+
+    Parameters:
+        type: Always "Termination" for this message type.
+        audio_duration_seconds: Total duration of audio processed.
+        session_duration_seconds: Total duration of the session.
+    """
 
     type: Literal["Termination"] = "Termination"
     audio_duration_seconds: float
@@ -52,6 +98,18 @@ AnyMessage = BeginMessage | TurnMessage | TerminationMessage
 
 
 class AssemblyAIConnectionParams(BaseModel):
+    """Configuration parameters for AssemblyAI WebSocket connection.
+
+    Parameters:
+        sample_rate: Audio sample rate in Hz. Defaults to 16000.
+        encoding: Audio encoding format. Defaults to "pcm_s16le".
+        formatted_finals: Whether to enable transcript formatting. Defaults to True.
+        word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds.
+        end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection.
+        min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn.
+        max_turn_silence: Maximum silence duration before forcing end-of-turn.
+    """
+
     sample_rate: int = 16000
     encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le"
     formatted_finals: bool = True

pipecat/services/assemblyai/stt.py

@@ -4,6 +4,12 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""AssemblyAI speech-to-text service implementation.
+
+This module provides integration with AssemblyAI's real-time speech-to-text
+WebSocket API for streaming audio transcription.
+"""
+
 import asyncio
 import json
 from typing import Any, AsyncGenerator, Dict
@@ -38,6 +44,7 @@ from .models import (
 
 try:
     import websockets
+    from websockets.asyncio.client import connect as websocket_connect
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error('In order to use AssemblyAI, you need to `pip install "pipecat-ai[assemblyai]"`.')
@@ -45,6 +52,13 @@ except ModuleNotFoundError as e:
 
 
 class AssemblyAISTTService(STTService):
+    """AssemblyAI real-time speech-to-text service.
+
+    Provides real-time speech transcription using AssemblyAI's WebSocket API.
+    Supports both interim and final transcriptions with configurable parameters
+    for audio processing and connection management.
+    """
+
     def __init__(
         self,
         *,
@@ -55,6 +69,16 @@ class AssemblyAISTTService(STTService):
         vad_force_turn_endpoint: bool = True,
         **kwargs,
     ):
+        """Initialize the AssemblyAI STT service.
+
+        Args:
+            api_key: AssemblyAI API key for authentication.
+            language: Language code for transcription. Defaults to English (Language.EN).
+            api_endpoint_base_url: WebSocket endpoint URL. Defaults to AssemblyAI's streaming endpoint.
+            connection_params: Connection configuration parameters. Defaults to AssemblyAIConnectionParams().
+            vad_force_turn_endpoint: Whether to force turn endpoint on VAD stop. Defaults to True.
+            **kwargs: Additional arguments passed to parent STTService class.
+        """
         self._api_key = api_key
         self._language = language
         self._api_endpoint_base_url = api_endpoint_base_url
@@ -75,22 +99,50 @@ class AssemblyAISTTService(STTService):
         self._chunk_size_bytes = 0
 
     def can_generate_metrics(self) -> bool:
+        """Check if the service can generate metrics.
+
+        Returns:
+            True if metrics generation is supported.
+        """
         return True
 
     async def start(self, frame: StartFrame):
+        """Start the speech-to-text service.
+
+        Args:
+            frame: Start frame to begin processing.
+        """
         await super().start(frame)
         self._chunk_size_bytes = int(self._chunk_size_ms * self._sample_rate * 2 / 1000)
         await self._connect()
 
     async def stop(self, frame: EndFrame):
+        """Stop the speech-to-text service.
+
+        Args:
+            frame: End frame to stop processing.
+        """
         await super().stop(frame)
         await self._disconnect()
 
     async def cancel(self, frame: CancelFrame):
+        """Cancel the speech-to-text service.
+
+        Args:
+            frame: Cancel frame to abort processing.
+        """
         await super().cancel(frame)
         await self._disconnect()
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        """Process audio data for speech-to-text conversion.
+
+        Args:
+            audio: Raw audio bytes to process.
+
+        Yields:
+            None (processing handled via WebSocket messages).
+        """
         self._audio_buffer.extend(audio)
 
         while len(self._audio_buffer) >= self._chunk_size_bytes:
@@ -101,6 +153,12 @@ class AssemblyAISTTService(STTService):
         yield None
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process frames for VAD and metrics handling.
+
+        Args:
+            frame: Frame to process.
+            direction: Direction of frame processing.
+        """
         await super().process_frame(frame, direction)
         if isinstance(frame, UserStartedSpeakingFrame):
             await self.start_ttfb_metrics()
@@ -133,9 +191,9 @@ class AssemblyAISTTService(STTService):
                 "Authorization": self._api_key,
                 "User-Agent": f"AssemblyAI/1.0 (integration=Pipecat/{pipecat_version})",
             }
-            self._websocket = await websockets.connect(
+            self._websocket = await websocket_connect(
                 ws_url,
-                extra_headers=headers,
+                additional_headers=headers,
             )
             self._connected = True
             self._receive_task = self.create_task(self._receive_task_handler())
@@ -161,10 +219,7 @@ class AssemblyAISTTService(STTService):
                 await self._websocket.send(json.dumps({"type": "Terminate"}))
 
                 try:
-                    await asyncio.wait_for(
-                        self._termination_event.wait(),
-                        timeout=5.0,
-                    )
+                    await asyncio.wait_for(self._termination_event.wait(), timeout=5.0)
                 except asyncio.TimeoutError:
                     logger.warning("Timed out waiting for termination message from server")
 
@@ -189,11 +244,9 @@ class AssemblyAISTTService(STTService):
         try:
             while self._connected:
                 try:
-                    message = await asyncio.wait_for(self._websocket.recv(), timeout=1.0)
+                    message = await self._websocket.recv()
                     data = json.loads(message)
                     await self._handle_message(data)
-                except asyncio.TimeoutError:
-                    self.reset_watchdog()
                 except websockets.exceptions.ConnectionClosedOK:
                     break
                 except Exception as e:
@@ -254,7 +307,7 @@ class AssemblyAISTTService(STTService):
             await self.push_frame(
                 TranscriptionFrame(
                     message.transcript,
-                    "",  # participant
+                    self._user_id,
                     time_now_iso8601(),
                     self._language,
                     message,
@@ -266,7 +319,7 @@ class AssemblyAISTTService(STTService):
             await self.push_frame(
                 InterimTranscriptionFrame(
                     message.transcript,
-                    "",  # participant
+                    self._user_id,
                     time_now_iso8601(),
                     self._language,
                     message,