dv-pipecat-ai 0.0.74.dev770__py3-none-any.whl → 0.0.82.dev776__py3-none-any.whl

pipecat/services/neuphonic/tts.py

@@ -4,11 +4,18 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Neuphonic text-to-speech service implementations.
+
+This module provides WebSocket and HTTP-based integrations with Neuphonic's
+text-to-speech API for real-time audio synthesis.
+"""
+
 import asyncio
 import base64
 import json
 from typing import Any, AsyncGenerator, Mapping, Optional
 
+import aiohttp
 from loguru import logger
 from pydantic import BaseModel
 
@@ -29,12 +36,11 @@ from pipecat.frames.frames import (
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.tts_service import InterruptibleTTSService, TTSService
 from pipecat.transcriptions.language import Language
-from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator
 from pipecat.utils.tracing.service_decorators import traced_tts
 
 try:
-    import websockets
-    from pyneuphonic import Neuphonic, TTSConfig
+    from websockets.asyncio.client import connect as websocket_connect
+    from websockets.protocol import State
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error("In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`.")
@@ -42,6 +48,14 @@ except ModuleNotFoundError as e:
 
 
 def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
+    """Convert a Language enum to Neuphonic language code.
+
+    Args:
+        language: The Language enum value to convert.
+
+    Returns:
+        The corresponding Neuphonic language code, or None if not supported.
+    """
     BASE_LANGUAGES = {
         Language.DE: "de",
         Language.EN: "en",
@@ -69,7 +83,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
 
 
 class NeuphonicTTSService(InterruptibleTTSService):
+    """Neuphonic real-time text-to-speech service using WebSocket streaming.
+
+    Provides real-time text-to-speech synthesis using Neuphonic's WebSocket API.
+    Supports interruption handling, keepalive connections, and configurable voice
+    parameters for high-quality speech generation.
+    """
+
     class InputParams(BaseModel):
+        """Input parameters for Neuphonic TTS configuration.
+
+        Parameters:
+            language: Language for synthesis. Defaults to English.
+            speed: Speech speed multiplier. Defaults to 1.0.
+        """
+
         language: Optional[Language] = Language.EN
         speed: Optional[float] = 1.0
 
@@ -82,10 +110,23 @@ class NeuphonicTTSService(InterruptibleTTSService):
         sample_rate: Optional[int] = 22050,
         encoding: str = "pcm_linear",
         params: Optional[InputParams] = None,
+        aggregate_sentences: Optional[bool] = True,
         **kwargs,
     ):
+        """Initialize the Neuphonic TTS service.
+
+        Args:
+            api_key: Neuphonic API key for authentication.
+            voice_id: ID of the voice to use for synthesis.
+            url: WebSocket URL for the Neuphonic API.
+            sample_rate: Audio sample rate in Hz. Defaults to 22050.
+            encoding: Audio encoding format. Defaults to "pcm_linear".
+            params: Additional input parameters for TTS configuration.
+            aggregate_sentences: Whether to aggregate sentences within the TTSService.
+            **kwargs: Additional arguments passed to parent InterruptibleTTSService.
+        """
         super().__init__(
-            aggregate_sentences=True,
+            aggregate_sentences=aggregate_sentences,
             push_text_frames=False,
             push_stop_frames=True,
             stop_frame_timeout_s=2.0,
@@ -114,12 +155,26 @@ class NeuphonicTTSService(InterruptibleTTSService):
         self._keepalive_task = None
 
     def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Neuphonic service supports metrics generation.
+        """
         return True
 
     def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert a Language enum to Neuphonic service language format.
+
+        Args:
+            language: The language to convert.
+
+        Returns:
+            The Neuphonic-specific language code, or None if not supported.
+        """
         return language_to_neuphonic_lang_code(language)
 
     async def _update_settings(self, settings: Mapping[str, Any]):
+        """Update service settings and reconnect with new configuration."""
         if "voice_id" in settings:
             self.set_voice(settings["voice_id"])
 
@@ -129,28 +184,56 @@ class NeuphonicTTSService(InterruptibleTTSService):
         logger.info(f"Switching TTS to settings: [{self._settings}]")
 
     async def start(self, frame: StartFrame):
+        """Start the Neuphonic TTS service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
         await super().start(frame)
         await self._connect()
 
     async def stop(self, frame: EndFrame):
+        """Stop the Neuphonic TTS service.
+
+        Args:
+            frame: The end frame.
+        """
         await super().stop(frame)
         await self._disconnect()
 
     async def cancel(self, frame: CancelFrame):
+        """Cancel the Neuphonic TTS service.
+
+        Args:
+            frame: The cancel frame.
+        """
         await super().cancel(frame)
         await self._disconnect()
 
     async def flush_audio(self):
+        """Flush any pending audio synthesis by sending stop command."""
         if self._websocket:
             msg = {"text": "<STOP>"}
             await self._websocket.send(json.dumps(msg))
 
     async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+        """Push a frame downstream with special handling for stop conditions.
+
+        Args:
+            frame: The frame to push.
+            direction: The direction to push the frame.
+        """
         await super().push_frame(frame, direction)
         if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
             self._started = False
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process frames with special handling for speech control.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame processing.
+        """
         await super().process_frame(frame, direction)
 
         # If we received a TTSSpeakFrame and the LLM response included text (it
@@ -164,6 +247,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
             await self.resume_processing_frames()
 
     async def _connect(self):
+        """Connect to Neuphonic WebSocket and start background tasks."""
         await self._connect_websocket()
 
         if self._websocket and not self._receive_task:
@@ -173,6 +257,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
             self._keepalive_task = self.create_task(self._keepalive_task_handler())
 
     async def _disconnect(self):
+        """Disconnect from Neuphonic WebSocket and clean up tasks."""
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
@@ -184,8 +269,9 @@ class NeuphonicTTSService(InterruptibleTTSService):
         await self._disconnect_websocket()
 
     async def _connect_websocket(self):
+        """Establish WebSocket connection to Neuphonic API."""
         try:
-            if self._websocket and self._websocket.open:
+            if self._websocket and self._websocket.state is State.OPEN:
                 return
 
             logger.debug("Connecting to Neuphonic")
@@ -195,20 +281,25 @@ class NeuphonicTTSService(InterruptibleTTSService):
                 "voice_id": self._voice_id,
             }
 
-            query_params = [f"api_key={self._api_key}"]
+            query_params = []
             for key, value in tts_config.items():
                 if value is not None:
                     query_params.append(f"{key}={value}")
 
-            url = f"{self._url}/speak/{self._settings['lang_code']}?{'&'.join(query_params)}"
+            url = f"{self._url}/speak/{self._settings['lang_code']}"
+            if query_params:
+                url += f"?{'&'.join(query_params)}"
 
-            self._websocket = await websockets.connect(url)
+            headers = {"x-api-key": self._api_key}
+
+            self._websocket = await websocket_connect(url, additional_headers=headers)
         except Exception as e:
             logger.error(f"{self} initialization error: {e}")
             self._websocket = None
             await self._call_event_handler("on_connection_error", f"{e}")
 
     async def _disconnect_websocket(self):
+        """Close WebSocket connection and clean up state."""
         try:
             await self.stop_all_metrics()
 
@@ -222,10 +313,11 @@ class NeuphonicTTSService(InterruptibleTTSService):
             self._websocket = None
 
     async def _receive_messages(self):
-        async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager):
+        """Receive and process messages from Neuphonic WebSocket."""
+        async for message in self._websocket:
             if isinstance(message, str):
                 msg = json.loads(message)
-                if msg.get("data", {}).get("audio") is not None:
+                if msg.get("data") and msg["data"].get("audio"):
                     await self.stop_ttfb_metrics()
 
                     audio = base64.b64decode(msg["data"]["audio"])
@@ -233,24 +325,40 @@ class NeuphonicTTSService(InterruptibleTTSService):
                     await self.push_frame(frame)
 
     async def _keepalive_task_handler(self):
-        KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3
+        """Handle keepalive messages to maintain WebSocket connection."""
+        KEEPALIVE_SLEEP = 10
         while True:
-            self.reset_watchdog()
             await asyncio.sleep(KEEPALIVE_SLEEP)
-            await self._send_text("")
+            await self._send_keepalive()
+
+    async def _send_keepalive(self):
+        """Send keepalive message to maintain connection."""
+        if self._websocket:
+            # Send empty text for keepalive
+            msg = {"text": ""}
+            await self._websocket.send(json.dumps(msg))
 
     async def _send_text(self, text: str):
+        """Send text to Neuphonic WebSocket for synthesis."""
         if self._websocket:
-            msg = {"text": text}
+            msg = {"text": f"{text} <STOP>"}
             logger.debug(f"Sending text to websocket: {msg}")
             await self._websocket.send(json.dumps(msg))
 
     @traced_tts
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        """Generate speech from text using Neuphonic's streaming API.
+
+        Args:
+            text: The text to synthesize into speech.
+
+        Yields:
+            Frame: Audio frames containing the synthesized speech.
+        """
         logger.debug(f"Generating TTS: [{text}]")
 
         try:
-            if not self._websocket or self._websocket.closed:
+            if not self._websocket or self._websocket.state is State.CLOSED:
                 await self._connect()
 
             try:
@@ -274,19 +382,21 @@ class NeuphonicTTSService(InterruptibleTTSService):
 
 
 class NeuphonicHttpTTSService(TTSService):
-    """Neuphonic Text-to-Speech service using HTTP streaming.
+    """Neuphonic text-to-speech service using HTTP streaming.
 
-    Args:
-        api_key: Neuphonic API key
-        voice_id: ID of the voice to use
-        url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com")
-        sample_rate: Sample rate for audio output (default: 22050Hz)
-        encoding: Audio encoding format (default: "pcm_linear")
-        params: Additional parameters for TTS generation including language and speed
-        **kwargs: Additional keyword arguments passed to the parent class
+    Provides text-to-speech synthesis using Neuphonic's HTTP API with server-sent
+    events for streaming audio delivery. Suitable for applications that prefer
+    HTTP-based communication over WebSocket connections.
     """
 
     class InputParams(BaseModel):
+        """Input parameters for Neuphonic HTTP TTS configuration.
+
+        Parameters:
+            language: Language for synthesis. Defaults to English.
+            speed: Speech speed multiplier. Defaults to 1.0.
+        """
+
         language: Optional[Language] = Language.EN
         speed: Optional[float] = 1.0
 
@@ -295,66 +405,183 @@ class NeuphonicHttpTTSService(TTSService):
         *,
         api_key: str,
         voice_id: Optional[str] = None,
+        aiohttp_session: aiohttp.ClientSession,
         url: str = "https://api.neuphonic.com",
         sample_rate: Optional[int] = 22050,
-        encoding: str = "pcm_linear",
+        encoding: Optional[str] = "pcm_linear",
         params: Optional[InputParams] = None,
         **kwargs,
     ):
+        """Initialize the Neuphonic HTTP TTS service.
+
+        Args:
+            api_key: Neuphonic API key for authentication.
+            voice_id: ID of the voice to use for synthesis.
+            aiohttp_session: Shared aiohttp session for HTTP requests.
+            url: Base URL for the Neuphonic HTTP API.
+            sample_rate: Audio sample rate in Hz. Defaults to 22050.
+            encoding: Audio encoding format. Defaults to "pcm_linear".
+            params: Additional input parameters for TTS configuration.
+            **kwargs: Additional arguments passed to parent TTSService.
+        """
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         params = params or NeuphonicHttpTTSService.InputParams()
 
         self._api_key = api_key
-        self._url = url
-        self._settings = {
-            "lang_code": self.language_to_service_language(params.language),
-            "speed": params.speed,
-            "encoding": encoding,
-            "sampling_rate": sample_rate,
-        }
+        self._session = aiohttp_session
+        self._base_url = url.rstrip("/")
+        self._lang_code = self.language_to_service_language(params.language) or "en"
+        self._speed = params.speed
+        self._encoding = encoding
         self.set_voice(voice_id)
 
     def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Neuphonic HTTP service supports metrics generation.
+        """
         return True
 
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert a Language enum to Neuphonic service language format.
+
+        Args:
+            language: The language to convert.
+
+        Returns:
+            The Neuphonic-specific language code, or None if not supported.
+        """
+        return language_to_neuphonic_lang_code(language)
+
     async def start(self, frame: StartFrame):
+        """Start the Neuphonic HTTP TTS service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
         await super().start(frame)
 
     async def flush_audio(self):
+        """Flush any pending audio synthesis.
+
+        Note:
+            HTTP-based service doesn't require explicit flushing.
+        """
         pass
 
+    def _parse_sse_message(self, message: str) -> dict | None:
+        """Parse a Server-Sent Event message.
+
+        Args:
+            message: The SSE message to parse.
+
+        Returns:
+            Parsed message dictionary or None if not a data message.
+        """
+        message = message.strip()
+
+        if not message or "data" not in message:
+            return None
+
+        try:
+            # Split on ": " and take the part after "data: "
+            _, data_content = message.split(": ", 1)
+
+            if not data_content or data_content == "[DONE]":
+                return None
+
+            message_dict = json.loads(data_content)
+
+            # Check for errors in the response
+            if message_dict.get("errors") is not None:
+                raise Exception(
+                    f"Neuphonic API error {message_dict.get('status_code', 'unknown')}: {message_dict['errors']}"
+                )
+
+            return message_dict
+        except (ValueError, json.JSONDecodeError) as e:
+            logger.warning(f"Failed to parse SSE message: {e}")
+            return None
+
     @traced_tts
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Neuphonic streaming API.
 
         Args:
-            text: The text to convert to speech
+            text: The text to convert to speech.
+
         Yields:
-            Frames containing audio data and status information
+            Frame: Audio frames containing the synthesized speech and status information.
         """
         logger.debug(f"Generating TTS: [{text}]")
 
-        client = Neuphonic(api_key=self._api_key, base_url=self._url.replace("https://", ""))
+        url = f"{self._base_url}/sse/speak/{self._lang_code}"
 
-        sse = client.tts.AsyncSSEClient()
+        headers = {
+            "X-API-KEY": self._api_key,
+            "Content-Type": "application/json",
+        }
+
+        payload = {
+            "text": text,
+            "lang_code": self._lang_code,
+            "encoding": self._encoding,
+            "sampling_rate": self.sample_rate,
+            "speed": self._speed,
+        }
+
+        if self._voice_id:
+            payload["voice_id"] = self._voice_id
 
         try:
             await self.start_ttfb_metrics()
-            response = sse.send(text, TTSConfig(**self._settings, voice_id=self._voice_id))
 
-            await self.start_tts_usage_metrics(text)
-            yield TTSStartedFrame()
+            async with self._session.post(url, json=payload, headers=headers) as response:
+                if response.status != 200:
+                    error_text = await response.text()
+                    error_message = f"Neuphonic API error: HTTP {response.status} - {error_text}"
+                    logger.error(error_message)
+                    yield ErrorFrame(error=error_message)
+                    return
 
-            async for message in response:
-                if message.status_code != 200:
-                    logger.error(f"{self} error: {message.errors}")
-                    yield ErrorFrame(error=f"Neuphonic API error: {message.errors}")
-
-                await self.stop_ttfb_metrics()
-                yield TTSAudioRawFrame(message.data.audio, self.sample_rate, 1)
+                await self.start_tts_usage_metrics(text)
+                yield TTSStartedFrame()
+
+                # Process SSE stream line by line
+                async for line in response.content:
+                    if not line:
+                        continue
+
+                    message = line.decode("utf-8", errors="ignore")
+                    if not message.strip():
+                        continue
+
+                    try:
+                        parsed_message = self._parse_sse_message(message)
+
+                        if (
+                            parsed_message is not None
+                            and parsed_message.get("data", {}).get("audio") is not None
+                        ):
+                            audio_b64 = parsed_message["data"]["audio"]
+                            audio_bytes = base64.b64decode(audio_b64)
+
+                            await self.stop_ttfb_metrics()
+                            yield TTSAudioRawFrame(audio_bytes, self.sample_rate, 1)
+
+                    except Exception as e:
+                        logger.error(f"Error processing SSE message: {e}")
+                        # Don't yield error frame for individual message failures
+                        continue
+
+        except asyncio.CancelledError:
+            logger.debug("TTS generation cancelled")
+            raise
         except Exception as e:
-            logger.error(f"Error in run_tts: {e}")
-            yield ErrorFrame(error=str(e))
+            logger.exception(f"Error in run_tts: {e}")
+            yield ErrorFrame(error=f"Neuphonic TTS error: {str(e)}")
         finally:
+            await self.stop_ttfb_metrics()
             yield TTSStoppedFrame()

pipecat/services/nim/llm.py

@@ -21,12 +21,6 @@ class NimLLMService(OpenAILLMService):
     This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining
     compatibility with the OpenAI-style interface. It specifically handles the difference
     in token usage reporting between NIM (incremental) and OpenAI (final summary).
-
-    Args:
-        api_key: The API key for accessing NVIDIA's NIM API.
-        base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1".
-        model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct".
-        **kwargs: Additional keyword arguments passed to OpenAILLMService.
     """
 
     def __init__(
@@ -37,6 +31,14 @@ class NimLLMService(OpenAILLMService):
         model: str = "nvidia/llama-3.1-nemotron-70b-instruct",
         **kwargs,
     ):
+        """Initialize the NimLLMService.
+
+        Args:
+            api_key: The API key for accessing NVIDIA's NIM API.
+            base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1".
+            model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct".
+            **kwargs: Additional keyword arguments passed to OpenAILLMService.
+        """
         super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
         # Counters for accumulating token usage metrics
         self._prompt_tokens = 0

pipecat/services/ollama/llm.py

@@ -6,6 +6,8 @@
 
 """OLLama LLM service implementation for Pipecat AI framework."""
 
+from loguru import logger
+
 from pipecat.services.openai.llm import OpenAILLMService
 
 
@@ -14,12 +16,30 @@ class OLLamaLLMService(OpenAILLMService):
 
     This service extends OpenAILLMService to work with locally hosted OLLama models,
     providing a compatible interface for running large language models locally.
-
-    Args:
-        model: The OLLama model to use. Defaults to "llama2".
-        base_url: The base URL for the OLLama API endpoint.
-                 Defaults to "http://localhost:11434/v1".
     """
 
-    def __init__(self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1"):
-        super().__init__(model=model, base_url=base_url, api_key="ollama")
+    def __init__(
+        self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1", **kwargs
+    ):
+        """Initialize OLLama LLM service.
+
+        Args:
+            model: The OLLama model to use. Defaults to "llama2".
+            base_url: The base URL for the OLLama API endpoint.
+                    Defaults to "http://localhost:11434/v1".
+            **kwargs: Additional keyword arguments passed to OpenAILLMService.
+        """
+        super().__init__(model=model, base_url=base_url, api_key="ollama", **kwargs)
+
+    def create_client(self, base_url=None, **kwargs):
+        """Create OpenAI-compatible client for Ollama.
+
+        Args:
+            base_url: The base URL for the API. If None, uses instance base_url.
+            **kwargs: Additional keyword arguments passed to the parent create_client method.
+
+        Returns:
+            An OpenAI-compatible client configured for Ollama.
+        """
+        logger.debug(f"Creating Ollama client with api {base_url}")
+        return super().create_client(base_url=base_url, **kwargs)