dv-pipecat-ai 0.0.85.dev818__py3-none-any.whl → 0.0.85.dev858__py3-none-any.whl

pipecat/services/soniox/stt.py

@@ -49,6 +49,33 @@ END_TOKEN = "<end>"
 FINALIZED_TOKEN = "<fin>"
 
 
+class SonioxContextGeneralItem(BaseModel):
+    """Represents a key-value pair for structured general context information."""
+
+    key: str
+    value: str
+
+
+class SonioxContextTranslationTerm(BaseModel):
+    """Represents a custom translation mapping for ambiguous or domain-specific terms."""
+
+    source: str
+    target: str
+
+
+class SonioxContextObject(BaseModel):
+    """Context object for models with context_version 2, for Soniox stt-rt-v3-preview and higher.
+
+    Learn more about context in the documentation:
+    https://soniox.com/docs/stt/concepts/context
+    """
+
+    general: Optional[List[SonioxContextGeneralItem]] = None
+    text: Optional[str] = None
+    terms: Optional[List[str]] = None
+    translation_terms: Optional[List[SonioxContextTranslationTerm]] = None
+
+
 class SonioxInputParams(BaseModel):
     """Real-time transcription settings.
 
@@ -60,9 +87,9 @@ class SonioxInputParams(BaseModel):
         audio_format: Audio format to use for transcription.
         num_channels: Number of channels to use for transcription.
         language_hints: List of language hints to use for transcription.
-        context: Customization for transcription.
-        enable_non_final_tokens: Whether to enable non-final tokens. If false, only final tokens will be returned.
-        max_non_final_tokens_duration_ms: Maximum duration of non-final tokens.
+        context: Customization for transcription. String for models with context_version 1 and ContextObject for models with context_version 2.
+        enable_speaker_diarization: Whether to enable speaker diarization. Tokens are annotated with speaker IDs.
+        enable_language_identification: Whether to enable language identification. Tokens are annotated with language IDs.
         client_reference_id: Client reference ID to use for transcription.
     """
 
@@ -72,10 +99,10 @@ class SonioxInputParams(BaseModel):
     num_channels: Optional[int] = 1
 
     language_hints: Optional[List[Language]] = None
-    context: Optional[str] = None
+    context: Optional[SonioxContextObject | str] = None
 
-    enable_non_final_tokens: Optional[bool] = True
-    max_non_final_tokens_duration_ms: Optional[int] = None
+    enable_speaker_diarization: Optional[bool] = False
+    enable_language_identification: Optional[bool] = False
 
     client_reference_id: Optional[str] = None
 
@@ -173,6 +200,10 @@ class SonioxSTTService(STTService):
         # Either one or the other is required.
         enable_endpoint_detection = not self._vad_force_turn_endpoint
 
+        context = self._params.context
+        if isinstance(context, SonioxContextObject):
+            context = context.model_dump()
+
         # Send the initial configuration message.
         config = {
             "api_key": self._api_key,
@@ -182,9 +213,9 @@ class SonioxSTTService(STTService):
             "enable_endpoint_detection": enable_endpoint_detection,
             "sample_rate": self.sample_rate,
             "language_hints": _prepare_language_hints(self._params.language_hints),
-            "context": self._params.context,
-            "enable_non_final_tokens": self._params.enable_non_final_tokens,
-            "max_non_final_tokens_duration_ms": self._params.max_non_final_tokens_duration_ms,
+            "context": context,
+            "enable_speaker_diarization": self._params.enable_speaker_diarization,
+            "enable_language_identification": self._params.enable_language_identification,
             "client_reference_id": self._params.client_reference_id,
         }
 
@@ -210,6 +241,7 @@ class SonioxSTTService(STTService):
             if self._receive_task != asyncio.current_task():
                 await self._receive_task
             self._receive_task = None
+        self.logger.debug("Disconnected from Soniox STT")
 
     async def stop(self, frame: EndFrame):
         """Stop the Soniox STT websocket connection.
@@ -351,7 +383,10 @@ class SonioxSTTService(STTService):
 
                 if self._final_transcription_buffer or non_final_transcription:
                     final_text = "".join(
-                        map(lambda token: token["text"], self._final_transcription_buffer)
+                        map(
+                            lambda token: token["text"],
+                            self._final_transcription_buffer,
+                        )
                     )
                     non_final_text = "".join(
                         map(lambda token: token["text"], non_final_transcription)

pipecat/services/vistaar/llm.py

@@ -10,19 +10,27 @@ from typing import Any, AsyncGenerator, Dict, List, Optional
 from urllib.parse import urlencode
 
 import httpx
+import jwt
 from loguru import logger
 from pydantic import BaseModel, Field
 
+try:
+    import redis.asyncio as redis
+    REDIS_AVAILABLE = True
+except ImportError:
+    REDIS_AVAILABLE = False
+    redis = None
+
 from pipecat.frames.frames import (
-    EndFrame,
     CancelFrame,
+    EndFrame,
     Frame,
+    InterruptionFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesFrame,
     LLMTextFrame,
     LLMUpdateSettingsFrame,
-    StartInterruptionFrame,
 )
 from pipecat.processors.aggregators.llm_response import (
     LLMAssistantAggregatorParams,
@@ -55,7 +63,9 @@ class VistaarLLMService(LLMService):
         Parameters:
             source_lang: Source language code (e.g., 'mr' for Marathi, 'hi' for Hindi).
             target_lang: Target language code for responses.
-            session_id: Session ID for maintaining conversation context.
+            session_id: Session ID for maintaining conversation context (also used for JWT caching).
+            pre_query_response_phrases: List of phrases to say while waiting for response.
+            phone_number: Phone number for JWT subject claim.
             extra: Additional model-specific parameters
         """
 
@@ -63,6 +73,7 @@ class VistaarLLMService(LLMService):
         target_lang: Optional[str] = Field(default="mr")
         session_id: Optional[str] = Field(default=None)
         pre_query_response_phrases: Optional[List[str]] = Field(default_factory=list)
+        phone_number: Optional[str] = Field(default="UNKNOWN")
         extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
 
     def __init__(
@@ -72,6 +83,9 @@ class VistaarLLMService(LLMService):
         params: Optional[InputParams] = None,
         timeout: float = 30.0,
         interim_timeout: float = 5.0,
+        redis_client: Optional[Any] = None,  # redis.Redis type
+        jwt_private_key: Optional[str] = None,
+        jwt_token_expiry: int = 3600,
         **kwargs,
     ):
         """Initialize Vistaar LLM service.
@@ -81,6 +95,9 @@ class VistaarLLMService(LLMService):
             params: Input parameters for model configuration and behavior.
             timeout: Request timeout in seconds. Defaults to 30.0 seconds.
             interim_timeout: Time in seconds before sending interim message. Defaults to 5.0 seconds.
+            redis_client: Optional Redis client for JWT token caching.
+            jwt_private_key: Optional RSA private key in PEM format for JWT signing.
+            jwt_token_expiry: JWT token expiry time in seconds. Defaults to 3600 (1 hour).
             **kwargs: Additional arguments passed to the parent LLMService.
         """
         super().__init__(**kwargs)
@@ -95,6 +112,16 @@ class VistaarLLMService(LLMService):
         self._extra = params.extra if isinstance(params.extra, dict) else {}
         self._timeout = timeout
         self._interim_timeout = interim_timeout
+        self._phone_number = params.phone_number
+
+        # JWT authentication setup
+        self._redis_client = redis_client
+        self._jwt_private_key = jwt_private_key
+        self._jwt_token_expiry = jwt_token_expiry
+        self._jwt_issuer = "voice-provider"
+
+        if self._jwt_private_key and not self._redis_client:
+            logger.warning("JWT private key provided but no Redis client for caching. JWT auth will regenerate tokens on each request.")
 
         # Create an async HTTP client
         self._client = httpx.AsyncClient(timeout=httpx.Timeout(self._timeout), verify=False)
@@ -112,6 +139,53 @@ class VistaarLLMService(LLMService):
             f"Vistaar LLM initialized - Base URL: {self._base_url}, Session ID: {self._session_id}, Source Lang: {self._source_lang}, Target Lang: {self._target_lang}, Timeout: {self._timeout}s"
         )
 
+    async def _get_jwt_token(self) -> Optional[str]:
+        """Generate or retrieve a cached JWT token.
+
+        Returns:
+            JWT token string or None if JWT auth is not configured.
+        """
+        if not self._jwt_private_key:
+            return None
+
+        # Try to get from Redis cache if available
+        if self._redis_client and self._session_id:
+            redis_key = f"vistaar_jwt:{self._session_id}"
+            try:
+                cached_token = await self._redis_client.get(redis_key)
+                if cached_token:
+                    logger.debug(f"Retrieved JWT token from Redis cache for session_id: {self._session_id}")
+                    return cached_token.decode('utf-8') if isinstance(cached_token, bytes) else cached_token
+            except Exception as e:
+                logger.warning(f"Redis cache retrieval failed: {e}. Generating new token.")
+
+        # Generate new token
+        current_time = int(time.time())
+        payload = {
+            "sub": self._phone_number,  # Subject identifier (phone number)
+            "iss": self._jwt_issuer,    # Issuer
+            "iat": current_time,         # Issued at timestamp
+            "exp": current_time + self._jwt_token_expiry  # Expiration timestamp
+        }
+
+        token = jwt.encode(payload, self._jwt_private_key, algorithm="RS256")
+        logger.info(f"Generated new JWT token for {self._phone_number}, expires in {self._jwt_token_expiry}s")
+
+        # Cache in Redis if available
+        if self._redis_client and self._session_id:
+            redis_key = f"vistaar_jwt:{self._session_id}"
+            try:
+                await self._redis_client.setex(
+                    redis_key,
+                    self._jwt_token_expiry,
+                    token
+                )
+                logger.debug(f"Cached JWT token in Redis for session_id: {self._session_id} with {self._jwt_token_expiry}s TTL")
+            except Exception as e:
+                logger.warning(f"Redis cache storage failed: {e}. Continuing without cache.")
+
+        return token
+
     async def _extract_messages_to_query(self, context: OpenAILLMContext) -> str:
         """Extract only the last user message from context.
 
@@ -259,9 +333,23 @@ class VistaarLLMService(LLMService):
         self._interim_in_progress = False
         self._interim_completion_event.clear()  # Reset the event for new request
 
+        # Prepare headers with JWT authentication if configured
+        headers = {}
+        try:
+            jwt_token = await self._get_jwt_token()
+            if jwt_token:
+                headers["Authorization"] = f"Bearer {jwt_token}"
+                logger.debug(f"Added JWT authentication header for session_id: {self._session_id}")
+        except Exception as e:
+            logger.error(f"Failed to generate JWT token: {e}")
+            raise
+
+        await self.start_connection_metrics()
+        
         try:
             # Use httpx to handle SSE streaming
-            async with self._client.stream("GET", url) as response:
+            async with self._client.stream("GET", url, headers=headers) as response:
+                await self.stop_connection_metrics(success=True, connection_type="http")
                 self._current_response = response  # Store for potential cancellation
                 response.raise_for_status()
 
@@ -279,14 +367,17 @@ class VistaarLLMService(LLMService):
                     yield line
 
         except httpx.HTTPStatusError as e:
+            await self.stop_connection_metrics(success=False, error=f"HTTP {e.response.status_code}", connection_type="http")
             logger.error(
                 f"Vistaar HTTP error - Status: {e.response.status_code}, URL: {url}, Response: {e.response.text if hasattr(e.response, 'text') else 'N/A'}"
             )
             raise
         except httpx.TimeoutException as e:
+            await self.stop_connection_metrics(success=False, error="Timeout", connection_type="http")
             logger.error(f"Vistaar timeout error - URL: {url}, Timeout: {self._timeout}s")
             raise
         except Exception as e:
+            await self.stop_connection_metrics(success=False, error=str(e), connection_type="http")
             logger.error(
                 f"Vistaar unexpected error - Type: {type(e).__name__}, Message: {str(e)}, URL: {url}"
             )
@@ -391,7 +482,7 @@ class VistaarLLMService(LLMService):
             )
             await self.push_frame(frame, direction)
             return
-        elif isinstance(frame, StartInterruptionFrame):
+        elif isinstance(frame, InterruptionFrame):
             await self._handle_interruption()
             await self.push_frame(frame, direction)
             return
@@ -467,4 +558,4 @@ class VistaarLLMService(LLMService):
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics."""
-        return True
+        return True

pipecat/transcriptions/language.py

@@ -569,3 +569,53 @@ class Language(StrEnum):
     # Zulu
     ZU = "zu"
     ZU_ZA = "zu-ZA"
+
+
+def resolve_language(
+    language: Language, language_map: dict[Language, str], use_base_code: bool = True
+) -> str:
+    """Resolve a Language enum to a service-specific language code.
+
+    Checks the language map first, then falls back to extracting the appropriate
+    code format with a warning if not found in the verified list.
+
+    Args:
+        language: The Language enum value to convert.
+        language_map: Dictionary mapping Language enums to service language codes.
+        use_base_code: If True, extracts base code (e.g., 'en' from 'en-US').
+                      If False, uses full language code as-is.
+
+    Returns:
+        The resolved language code for the service.
+
+    Examples::
+
+        # Service expecting base codes (e.g., Cartesia)
+        >>> LANGUAGE_MAP = {Language.EN: "en", Language.ES: "es"}
+        >>> resolve_language(Language.EN_US, LANGUAGE_MAP, use_base_code=True)
+        # Logs: "Language en-US not verified. Using base code 'en'."
+        "en"
+
+        # Service expecting full codes (e.g., AWS)
+        >>> LANGUAGE_MAP = {Language.EN_US: "en-US", Language.ES_ES: "es-ES"}
+        >>> resolve_language(Language.EN_GB, LANGUAGE_MAP, use_base_code=False)
+        # Logs: "Language en-GB not verified. Using 'en-GB'."
+        "en-GB"
+    """
+    # Check if language is in the verified map
+    result = language_map.get(language)
+
+    if result is not None:
+        return result
+
+    # Not in map - fall back with warning
+    lang_str = str(language.value)
+
+    if use_base_code:
+        # Extract base code (e.g., "en" from "en-US")
+        base_code = lang_str.split("-")[0].lower()
+        # logger.warning(f"Language {language.value} not verified. Using base code '{base_code}'.")
+        return base_code
+    else:
+        # logger.warning(f"Language {language.value} not verified. Using '{lang_str}'.")
+        return lang_str

pipecat/transports/base_input.py

@@ -297,6 +297,17 @@ class BaseInputTransport(FrameProcessor):
         elif isinstance(frame, EmulateUserStoppedSpeakingFrame):
             self.logger.debug("Emulating user stopped speaking")
             await self._handle_user_interruption(VADState.QUIET, emulated=True)
+        elif isinstance(frame, VADParamsUpdateFrame):
+            if self.vad_analyzer:
+                self.vad_analyzer.set_params(frame.params, self.logger)
+                speech_frame = SpeechControlParamsFrame(
+                    vad_params=frame.params,
+                    turn_params=self._params.turn_analyzer.params
+                    if self._params.turn_analyzer
+                    else None,
+                )
+                await self.push_frame(speech_frame)
+            await self.push_frame(frame, direction)
         # All other system frames
         elif isinstance(frame, SystemFrame):
             await self.push_frame(frame, direction)
@@ -309,16 +320,6 @@ class BaseInputTransport(FrameProcessor):
         elif isinstance(frame, StopFrame):
             await self.push_frame(frame, direction)
             await self.pause(frame)
-        elif isinstance(frame, VADParamsUpdateFrame):
-            if self.vad_analyzer:
-                self.vad_analyzer.set_params(frame.params)
-                speech_frame = SpeechControlParamsFrame(
-                    vad_params=frame.params,
-                    turn_params=self._params.turn_analyzer.params
-                    if self._params.turn_analyzer
-                    else None,
-                )
-                await self.push_frame(speech_frame)
         elif isinstance(frame, FilterUpdateSettingsFrame) and self._params.audio_in_filter:
             await self._params.audio_in_filter.process_frame(frame)
         # Other frames
@@ -444,7 +445,10 @@ class BaseInputTransport(FrameProcessor):
             await self._handle_user_interruption(VADState.QUIET)
 
     async def _run_turn_analyzer(
-        self, frame: InputAudioRawFrame, vad_state: VADState, previous_vad_state: VADState
+        self,
+        frame: InputAudioRawFrame,
+        vad_state: VADState,
+        previous_vad_state: VADState,
     ):
         """Run turn analysis on audio frame and handle results."""
         is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING

pipecat/transports/base_output.py

@@ -50,6 +50,11 @@ from pipecat.utils.time import nanoseconds_to_seconds
 
 # TODO: When we use GeminiMultimodalLiveLLMService, we need to change this to 0.35 but that creates issue for faster TTS.
 BOT_VAD_STOP_SECS = 0.30
+# For the very first bot utterance (e.g., intro), we can safely
+# detect end-of-speech sooner to improve responsiveness for the
+# user’s first short reply. Keep conservative to avoid mid-utterance
+# false stops when TTS streams quickly.
+FIRST_BOT_VAD_STOP_SECS = 0.12
 
 
 class BaseOutputTransport(FrameProcessor):
@@ -84,6 +89,7 @@ class BaseOutputTransport(FrameProcessor):
         # us to send multiple streams at the same time if the transport allows
         # it.
         self._media_senders: Dict[Any, "BaseOutputTransport.MediaSender"] = {}
+        self._register_event_handler("on_output_terminated")
 
     @property
     def sample_rate(self) -> int:
@@ -301,10 +307,12 @@ class BaseOutputTransport(FrameProcessor):
             await self.start(frame)
         elif isinstance(frame, EndFrame):
             await self.stop(frame)
+            await self._call_event_handler("on_output_terminated", frame)
             # Keep pushing EndFrame down so all the pipeline stops nicely.
             await self.push_frame(frame, direction)
         elif isinstance(frame, CancelFrame):
             await self.cancel(frame)
+            await self._call_event_handler("on_output_terminated", frame)
             await self.push_frame(frame, direction)
         elif isinstance(frame, InterruptionFrame):
             await self.push_frame(frame, direction)
@@ -403,6 +411,9 @@ class BaseOutputTransport(FrameProcessor):
             self._bot_speaking_frame_period = 0.2
             # Last time the bot actually spoke.
             self._bot_speech_last_time = 0
+            # Before the first stop event, we use a shorter silence
+            # threshold to make the first turn more responsive.
+            self._first_stop_pending = True
 
             self._audio_task: Optional[asyncio.Task] = None
             self._video_task: Optional[asyncio.Task] = None
@@ -628,6 +639,10 @@ class BaseOutputTransport(FrameProcessor):
 
             self._bot_speaking = False
 
+            # Mark that the first stop has been completed so subsequent
+            # stops use the regular (longer) VAD stop threshold.
+            self._first_stop_pending = False
+
             # Clean audio buffer (there could be tiny left overs if not multiple
             # to our output chunk size).
             self._audio_buffer = bytearray()
@@ -687,9 +702,14 @@ class BaseOutputTransport(FrameProcessor):
             async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
                 while True:
                     try:
-                        frame = await asyncio.wait_for(
-                            self._audio_queue.get(), timeout=vad_stop_secs
+                        # Use a shorter timeout only for the first bot stop to
+                        # accelerate the initial turn handoff right after the intro.
+                        timeout = (
+                            FIRST_BOT_VAD_STOP_SECS
+                            if getattr(self, "_first_stop_pending", True)
+                            else BOT_VAD_STOP_SECS
                         )
+                        frame = await asyncio.wait_for(self._audio_queue.get(), timeout=timeout)
                         yield frame
                         self._audio_queue.task_done()
                     except asyncio.TimeoutError:
@@ -710,7 +730,13 @@ class BaseOutputTransport(FrameProcessor):
                     except asyncio.QueueEmpty:
                         # Notify the bot stopped speaking upstream if necessary.
                         diff_time = time.time() - last_frame_time
-                        if diff_time > vad_stop_secs:
+                        # Use a shorter threshold for the first stop only.
+                        current_stop_secs = (
+                            FIRST_BOT_VAD_STOP_SECS
+                            if getattr(self, "_first_stop_pending", True)
+                            else BOT_VAD_STOP_SECS
+                        )
+                        if diff_time > current_stop_secs:
                             await self._bot_stopped_speaking()
                         # Generate an audio frame with only the mixer's part.
                         frame = OutputAudioRawFrame(

pipecat/utils/redis.py

@@ -0,0 +1,58 @@
+"""Async Redis helper utilities."""
+
+from __future__ import annotations
+
+from typing import Any, Optional, TYPE_CHECKING
+
+from urllib.parse import urlparse
+
+try:
+    import redis.asyncio as redis
+except ImportError:  # pragma: no cover - Redis is optional
+    redis = None
+
+
+if TYPE_CHECKING:  # pragma: no cover - typing aid
+    from redis.asyncio import Redis
+
+
+def create_async_redis_client(
+    url: Optional[str],
+    *,
+    decode_responses: bool = True,
+    encoding: str = "utf-8",
+    logger: Optional[Any] = None,
+    **kwargs,
+) -> Optional["Redis"]:
+    """Return a configured async Redis client or None if unavailable.
+
+    Args:
+        url: Redis connection URL.
+        decode_responses: Whether to decode responses to str.
+        encoding: Character encoding to use with decoded responses.
+        logger: Optional logger supporting .warning() for diagnostics.
+        **kwargs: Additional keyword arguments forwarded to Redis.from_url.
+    """
+    if redis is None:
+        return None
+
+    if not url or url in {"redis_url", "REDIS_URL"}:
+        return None
+
+    parsed = urlparse(url)
+    connection_kwargs = {
+        "decode_responses": decode_responses,
+        "encoding": encoding,
+    }
+    connection_kwargs.update(kwargs)
+
+    if parsed.scheme == "rediss":
+        connection_kwargs.setdefault("ssl_cert_reqs", "none")
+        connection_kwargs.setdefault("ssl_check_hostname", False)
+
+    try:
+        return redis.Redis.from_url(url, **connection_kwargs)
+    except Exception as exc:  # pragma: no cover - best effort logging
+        if logger is not None:
+            logger.warning(f"Failed to create Redis client: {exc}")
+        return None