videosdk-plugins-deepgram 0.0.38__py3-none-any.whl → 0.0.55__py3-none-any.whl

videosdk/plugins/deepgram/__init__.py

@@ -1,3 +1,4 @@
 from .stt import DeepgramSTT
+from .stt_v2 import DeepgramSTTV2
 from .tts import DeepgramTTS
-__all__ = ["DeepgramSTT", "DeepgramTTS"]
+__all__ = ["DeepgramSTT","DeepgramSTTV2" "DeepgramTTS"]

videosdk/plugins/deepgram/stt_v2.py

@@ -0,0 +1,291 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import numpy as np
+from typing import Any, Optional
+import os
+from urllib.parse import urlencode
+import aiohttp
+from videosdk.agents import STT as BaseSTT, STTResponse, SpeechEventType, SpeechData, global_event_emitter
+import logging
+
+try:
+    from scipy import signal
+    SCIPY_AVAILABLE = True
+except ImportError:
+    SCIPY_AVAILABLE = False
+logger = logging.getLogger(__name__)
+
+
+class DeepgramSTTV2(BaseSTT):
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        model: str = "flux-general-en",
+        input_sample_rate: int = 48000,
+        target_sample_rate: int = 16000,
+        eager_eot_threshold:float=0.6,
+        eot_threshold:float=0.8,
+        eot_timeout_ms:int=7000,
+        base_url: str = "wss://api.deepgram.com/v2/listen",
+        enable_preemptive_generation: bool = False,
+    ) -> None:
+        """Initialize the Deepgram STT plugin
+
+        Args:
+            api_key (str | None, optional): Deepgram API key. Uses DEEPGRAM_API_KEY environment variable if not provided. Defaults to None.
+            model (str): The model to use for the STT plugin. Defaults to "flux-general-en".
+            input_sample_rate (int): The input sample rate to use for the STT plugin. Defaults to 48000.
+            target_sample_rate (int): The target sample rate to use for the STT plugin. Defaults to 16000.
+            eager_eot_threshold (float): Eager end-of-turn threshold. Defaults to 0.6.
+            eot_threshold (float): End-of-turn threshold. Defaults to 0.8.
+            eot_timeout_ms (int): End-of-turn timeout in milliseconds. Defaults to 7000.
+            base_url (str): The base URL to use for the STT plugin. Defaults to "wss://api.deepgram.com/v2/listen".
+            enable_preemptive_generation (bool): Enable preemptive generation based on EagerEndOfTurn events. Defaults to False.
+        """
+        super().__init__()
+
+        self.api_key = api_key or os.getenv("DEEPGRAM_API_KEY")
+        if not self.api_key:
+            raise ValueError(
+                "Deepgram API key must be provided either through api_key parameter or DEEPGRAM_API_KEY environment variable")
+
+        self.model = model
+        self.input_sample_rate = input_sample_rate
+        self.target_sample_rate = target_sample_rate
+        self.eager_eot_threshold = eager_eot_threshold
+        self.eot_threshold=eot_threshold
+        self.eot_timeout_ms = eot_timeout_ms
+        self.base_url = base_url
+        self.enable_preemptive_generation = enable_preemptive_generation
+
+        self._stream_buffer = bytearray()
+        self._target_chunk_size = int(0.1 * self.target_sample_rate * 2)
+        self._min_chunk_size = int(0.05 * self.target_sample_rate * 2)
+
+        self._session: Optional[aiohttp.ClientSession] = None
+        self._ws: Optional[aiohttp.ClientWebSocketResponse] = None
+        self._ws_task: Optional[asyncio.Task] = None
+        self._last_transcript: str = ""
+        self._ws_task = None
+    
+
+    async def process_audio(
+        self,
+        audio_frames: bytes,
+        **kwargs: Any
+    ) -> None:
+        """Process audio frames and send to Deeepgram's Flux API"""
+        
+        if not self._ws:
+            await self._connect_ws()
+            self._ws_task = asyncio.create_task(self._listen_for_responses())
+            
+        try:
+            resampled_audio = self._resample_audio(audio_frames)
+            if not resampled_audio:
+                return
+                
+            self._stream_buffer.extend(resampled_audio)
+             # chunk size 100ms
+            while len(self._stream_buffer) >= self._target_chunk_size:
+                chunk_to_send = bytes(self._stream_buffer[:self._target_chunk_size])
+                self._stream_buffer = self._stream_buffer[self._target_chunk_size:]
+                
+                await self._ws.send_bytes(bytes(chunk_to_send))
+                
+        except Exception as e:
+            logger.error(f"Error in process_audio: {str(e)}")
+            self.emit("error", str(e))
+            if self._ws:
+                await self._ws.close()
+                self._ws = None
+                if self._ws_task:
+                    self._ws_task.cancel()
+                    self._ws_task = None
+
+    async def _listen_for_responses(self) -> None:
+        """Background task to listen for WebSocket responses"""
+        if not self._ws:
+            return
+
+        try:
+            async for msg in self._ws:
+                if msg.type == aiohttp.WSMsgType.TEXT:
+                    data = msg.json()
+                    responses = self._handle_ws_message(data)
+                    for response in responses:
+                        if self._transcript_callback:
+                            await self._transcript_callback(response)
+                elif msg.type == aiohttp.WSMsgType.ERROR:
+                    logger.error(f"WebSocket error: {self._ws.exception()}")
+                    self.emit(
+                        "error", f"WebSocket error: {self._ws.exception()}")
+                    break
+        except Exception as e:
+            logger.error(f"Error in WebSocket listener: {str(e)}")
+            self.emit("error", f"Error in WebSocket listener: {str(e)}")
+        finally:
+            if self._ws:
+                await self._ws.close()
+                self._ws = None
+
+    async def _connect_ws(self) -> None:
+        """Establish WebSocket connection with Deepgram's Streaming API"""
+        if not self._session:
+            self._session = aiohttp.ClientSession()
+
+        query_params = {
+            "model": self.model,
+            "encoding": "linear16",
+            "sample_rate": self.target_sample_rate,
+            "eot_threshold": self.eot_threshold,
+            "eot_timeout_ms": self.eot_timeout_ms,
+            "eager_eot_threshold": self.eager_eot_threshold,
+        }
+        headers = {"Authorization": f"Token {self.api_key}"}
+        ws_url = f"{self.base_url}?{urlencode(query_params)}"
+
+        try:
+            self._ws = await self._session.ws_connect(ws_url, headers=headers)
+            logger.info("Connected to Deepgram V2 WebSocket.")
+        except Exception as e:
+            logger.error(f"Error connecting to WebSocket: {str(e)}")
+            raise
+
+    def _handle_ws_message(self, msg: dict) -> list[STTResponse]:
+        """Handle incoming WebSocket messages and generate STT responses"""
+        responses = []
+
+        try:
+            if msg.get("type") != "TurnInfo":
+                return responses
+
+            event = msg.get("event")
+            transcript = msg.get("transcript", "")
+            # logger.info(f"{event} and {transcript}")
+            start_time = msg.get("audio_window_start", 0.0)
+            end_time = msg.get("audio_window_end", 0.0)
+            confidence = msg.get("end_of_turn_confidence", 0.0)
+
+            self._last_transcript = transcript
+            # Emit turn-related events
+            if event == "StartOfTurn":
+                global_event_emitter.emit("speech_started")
+            elif event == "EagerEndOfTurn":
+                # Handle EagerEndOfTurn for preemptive generation
+                if self.enable_preemptive_generation and transcript and self._transcript_callback:
+                    responses.append(
+                        STTResponse(
+                            event_type=SpeechEventType.PREFLIGHT,
+                            data=SpeechData(
+                                text=transcript,
+                                confidence=confidence,
+                                start_time=start_time,
+                                end_time=end_time,
+                            ),
+                            metadata={"model": self.model},
+                        )
+                    )
+            elif event == "EndOfTurn":
+                logger.info(f"EndOfTurn (FINAL) Transcript: {transcript} and Confidence: {confidence}")
+                global_event_emitter.emit("speech_stopped")
+                if transcript and self._transcript_callback:
+                    responses.append(
+                        STTResponse(
+                            event_type=SpeechEventType.FINAL,
+                            data=SpeechData(
+                                text=transcript,
+                                confidence=confidence,
+                                start_time=start_time,
+                                end_time=end_time,
+                            ),
+                            metadata={"model": self.model},
+                        )
+                    )
+            elif event == "TurnResumed":
+                # Send interim to signal user continued speaking
+                if self.enable_preemptive_generation and transcript:
+                    responses.append(
+                            STTResponse(
+                                event_type=SpeechEventType.INTERIM,
+                                data=SpeechData(
+                                    text=transcript,
+                                    confidence=confidence,
+                                    start_time=start_time,
+                                    end_time=end_time,
+                                ),
+                                metadata={"model": self.model, "turn_resumed": True},
+                            )
+                    )
+
+        except Exception as e:
+            logger.error(f"Error handling WebSocket message: {str(e)}")
+
+        return responses
+    
+    def _resample_audio(self, audio_bytes: bytes) -> bytes:
+        """Resample audio from input sample rate to target sample rate and convert to mono."""
+        try:
+            if not audio_bytes:
+                return b''
+
+            raw_audio = np.frombuffer(audio_bytes, dtype=np.int16)
+            if raw_audio.size == 0:
+                return b''
+
+            if raw_audio.size % 2 == 0: 
+                stereo_audio = raw_audio.reshape(-1, 2)
+                mono_audio = stereo_audio.astype(np.float32).mean(axis=1)
+            else:
+                mono_audio = raw_audio.astype(np.float32)
+
+            if self.input_sample_rate != self.target_sample_rate:
+                target_length = int(len(mono_audio) * self.target_sample_rate / self.input_sample_rate)
+                resampled_data = signal.resample(mono_audio, target_length)
+            else:
+                resampled_data = mono_audio
+
+            resampled_data = np.clip(resampled_data, -32767, 32767)
+            return resampled_data.astype(np.int16).tobytes()
+
+        except Exception as e:
+            logger.error(f"Error resampling audio: {e}")
+            return b''
+
+
+    async def aclose(self) -> None:
+        """Cleanup resources"""
+        
+        if len(self._stream_buffer) >= self._min_chunk_size and self._ws:
+            try:
+                final_chunk = bytes(self._stream_buffer)
+                await self._ws.send_bytes(final_chunk)
+            except Exception as e:
+                logger.error(f"Error sending final audio: {e}")
+        
+        if self._ws:
+            try:
+                await self._ws.send_str(json.dumps({"type": "Terminate"}))
+                await asyncio.sleep(0.5)  
+            except Exception as e:
+                logger.error(f"Error sending termination: {e}")
+
+        if self._ws_task:
+            self._ws_task.cancel()
+            try:
+                await self._ws_task
+            except asyncio.CancelledError:
+                pass
+            self._ws_task = None
+            
+        if self._ws:
+            await self._ws.close()
+            self._ws = None
+            
+        if self._session:
+            await self._session.close()
+            self._session = None
+        await super().aclose()

videosdk/plugins/deepgram/tts.py

@@ -81,12 +81,16 @@ class DeepgramTTS(TTS):
                 self.emit("error", "Audio track or event loop not set")
                 return
 
-            await self.interrupt()
+            self._should_stop = True
+            if self._send_task and not self._send_task.done():
+                self._send_task.cancel()
+                
             self._should_stop = False
             await self._stream_synthesis(text)
 
         except Exception as e:
             self.emit("error", f"TTS synthesis failed: {str(e)}")
+            raise
 
     async def _stream_synthesis(self, text: Union[AsyncIterator[str], str]) -> None:
         try:

videosdk/plugins/deepgram/version.py

@@ -1 +1 @@
-__version__ = "0.0.38"
+__version__ = "0.0.55"

{videosdk_plugins_deepgram-0.0.38.dist-info → videosdk_plugins_deepgram-0.0.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: videosdk-plugins-deepgram
-Version: 0.0.38
+Version: 0.0.55
 Summary: VideoSDK Agent Framework plugin for Deepgram
 Author: videosdk
 License-Expression: Apache-2.0
@@ -12,7 +12,7 @@ Classifier: Topic :: Multimedia :: Sound/Audio
 Classifier: Topic :: Multimedia :: Video
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.11
-Requires-Dist: videosdk-agents>=0.0.38
+Requires-Dist: videosdk-agents>=0.0.55
 Description-Content-Type: text/markdown
 
 # VideoSDK Deepgram Plugin

videosdk_plugins_deepgram-0.0.55.dist-info/RECORD

@@ -0,0 +1,8 @@
+videosdk/plugins/deepgram/__init__.py,sha256=3Bz_6gDJ7C2SHXHqZEmjpiwspGtZx7JT8ig9S8TSNDY,147
+videosdk/plugins/deepgram/stt.py,sha256=FrhLg-57kqySa07Zo8yFJmMjcDcmdkihWcBMfFcxBRY,8309
+videosdk/plugins/deepgram/stt_v2.py,sha256=ybC4dCOMPzc6iq8d0eyybvQaY0e_U_IkivR5kkrGRB4,11761
+videosdk/plugins/deepgram/tts.py,sha256=ZcQoF50wmNnWsozo4iwOcZIxavGelLMQLqCRk3GvAvs,7025
+videosdk/plugins/deepgram/version.py,sha256=jXOOiiBG8YrcSzw2nujXNZ67Lc4_BDh5cva6CjfFnJo,23
+videosdk_plugins_deepgram-0.0.55.dist-info/METADATA,sha256=VZ34CjoCYKWNVRJFOJBYYm8j7SiiptfL507jqdh1llk,767
+videosdk_plugins_deepgram-0.0.55.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+videosdk_plugins_deepgram-0.0.55.dist-info/RECORD,,

{videosdk_plugins_deepgram-0.0.38.dist-info → videosdk_plugins_deepgram-0.0.55.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

videosdk_plugins_deepgram-0.0.38.dist-info/RECORD

@@ -1,7 +0,0 @@
-videosdk/plugins/deepgram/__init__.py,sha256=gzJRVqNliLxQHAbCSiki_YiFqicxRCSKgTk1cCLRkFg,98
-videosdk/plugins/deepgram/stt.py,sha256=FrhLg-57kqySa07Zo8yFJmMjcDcmdkihWcBMfFcxBRY,8309
-videosdk/plugins/deepgram/tts.py,sha256=hd4oifQ3lRV-Ry57EGf-8VrcBm2bM9Fj3VUKSAfmgh8,6884
-videosdk/plugins/deepgram/version.py,sha256=R5QxTjVaID7odO0eBWpOnyCjNQxBZ7cpyruM_NMOoDc,23
-videosdk_plugins_deepgram-0.0.38.dist-info/METADATA,sha256=9mXxKWbfmNQ3BrruolL09sPjKPfeuCtZoKiven2VZfU,767
-videosdk_plugins_deepgram-0.0.38.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-videosdk_plugins_deepgram-0.0.38.dist-info/RECORD,,