rasa-pro 3.11.5__py3-none-any.whl → 3.12.0__py3-none-any.whl

rasa/core/channels/voice_stream/asr/deepgram.py

@@ -1,9 +1,12 @@
-from dataclasses import dataclass
-from typing import Any, Dict, Optional
 import json
 import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
 
+import structlog
 import websockets
+import websockets.exceptions
 from websockets.legacy.client import WebSocketClientProtocol
 
 from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine, ASREngineConfig
@@ -15,15 +18,20 @@ from rasa.core.channels.voice_stream.asr.asr_event import (
 from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
 from rasa.shared.constants import DEEPGRAM_API_KEY_ENV_VAR
 
+logger = structlog.get_logger(__name__)
+
 
 @dataclass
 class DeepgramASRConfig(ASREngineConfig):
     endpoint: Optional[str] = None
-    # number of miliseconds of silence to determine end of speech
+    # number of milliseconds of silence to determine end of speech
     endpointing: Optional[int] = None
     language: Optional[str] = None
     model: Optional[str] = None
     smart_format: Optional[bool] = None
+    # number of milliseconds of no new transcript to determine end of speech
+    # should be at least 1000 according to docs
+    utterance_end_ms: Optional[int] = None
 
 
 class DeepgramASR(ASREngine[DeepgramASRConfig]):
@@ -37,22 +45,47 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
         """Connect to the ASR system."""
         deepgram_api_key = os.environ[DEEPGRAM_API_KEY_ENV_VAR]
         extra_headers = {"Authorization": f"Token {deepgram_api_key}"}
-        api_url = self._get_api_url()
-        query_params = self._get_query_params()
-        return await websockets.connect(  # type: ignore
-            api_url + query_params,
-            extra_headers=extra_headers,
-        )
+        try:
+            return await websockets.connect(  # type: ignore
+                self._get_api_url_with_query_params(),
+                extra_headers=extra_headers,
+            )
+        except websockets.exceptions.InvalidStatusCode as e:
+            if e.status_code == 401:
+                error_msg = "Please make sure your Deepgram API key is correct."
+            else:
+                error_msg = "Connection to Deepgram failed."
+            logger.error(
+                "deepgram.connection.failed",
+                status_code=e.status_code,
+                error=error_msg,
+            )
+            raise
+
+    def _get_api_url_with_query_params(self) -> str:
+        """Combine api url and query params."""
+        return self._get_api_url() + self._get_query_params()
 
     def _get_api_url(self) -> str:
+        """Get the api url with the configured endpoint."""
         return f"wss://{self.config.endpoint}/v1/listen?"
 
     def _get_query_params(self) -> str:
-        return (
-            f"encoding=mulaw&sample_rate={HERTZ}&endpointing={self.config.endpointing}"
-            f"&vad_events=true&language={self.config.language}&interim_results=true"
-            f"&model={self.config.model}&smart_format={str(self.config.smart_format).lower()}"
-        )
+        """Get the configured query parameters for the api."""
+        query_params = {
+            "encoding": "mulaw",
+            "sample_rate": HERTZ,
+            "endpointing": self.config.endpointing,
+            "vad_events": "true",
+            "language": self.config.language,
+            "interim_results": "true",
+            "model": self.config.model,
+            "smart_format": str(self.config.smart_format).lower(),
+        }
+        if self.config.utterance_end_ms and self.config.utterance_end_ms > 0:
+            query_params["utterance_end_ms"] = self.config.utterance_end_ms
+
+        return urlencode(query_params)
 
     async def signal_audio_done(self) -> None:
         """Signal to the ASR Api that you are done sending data."""
@@ -67,24 +100,48 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
     def engine_event_to_asr_event(self, e: Any) -> Optional[ASREvent]:
         """Translate an engine event to a common ASREvent."""
         data = json.loads(e)
-        if "is_final" in data:
-            transcript = data["channel"]["alternatives"][0]["transcript"]
+        data_type = data["type"]
+        if data_type == "Results":
+            transcript_data = data["channel"]["alternatives"][0]
+            transcript = transcript_data["transcript"]
             if data["is_final"]:
                 if data.get("speech_final"):
-                    full_transcript = self.accumulated_transcript + transcript
+                    full_transcript = self.concatenate_transcripts(
+                        self.accumulated_transcript, transcript
+                    )
                     self.accumulated_transcript = ""
                     if full_transcript:
                         return NewTranscript(full_transcript)
                 else:
-                    self.accumulated_transcript += transcript
+                    self.accumulated_transcript = self.concatenate_transcripts(
+                        self.accumulated_transcript, transcript
+                    )
             elif transcript:
                 return UserIsSpeaking()
+        # event that comes after utterance_end_ms of no new transcript
+        elif data_type == "UtteranceEnd":
+            if self.accumulated_transcript:
+                transcript = self.accumulated_transcript
+                self.accumulated_transcript = ""
+                return NewTranscript(transcript)
         return None
 
     @staticmethod
     def get_default_config() -> DeepgramASRConfig:
-        return DeepgramASRConfig("api.deepgram.com", 400, "en", "nova-2-general", True)
+        return DeepgramASRConfig(
+            endpoint="api.deepgram.com",
+            endpointing=400,
+            language="en",
+            model="nova-2-general",
+            smart_format=True,
+            utterance_end_ms=1000,
+        )
 
     @classmethod
     def from_config_dict(cls, config: Dict) -> "DeepgramASR":
         return DeepgramASR(DeepgramASRConfig.from_dict(config))
+
+    @staticmethod
+    def concatenate_transcripts(t1: str, t2: str) -> str:
+        """Concatenate two transcripts making sure there is a space between them."""
+        return (t1.strip() + " " + t2.strip()).strip()

rasa/core/channels/voice_stream/audiocodes.py

@@ -0,0 +1,292 @@
+import asyncio
+import base64
+import json
+from typing import Any, Awaitable, Callable, Dict, Optional, Text
+
+import structlog
+from sanic import (  # type: ignore[attr-defined]
+    Blueprint,
+    HTTPResponse,
+    Request,
+    Websocket,
+    response,
+)
+
+from rasa.core.channels import UserMessage
+from rasa.core.channels.voice_ready.audiocodes import map_call_params
+from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.call_state import (
+    call_state,
+)
+from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
+from rasa.core.channels.voice_stream.voice_channel import (
+    ContinueConversationAction,
+    EndConversationAction,
+    NewAudioAction,
+    VoiceChannelAction,
+    VoiceInputChannel,
+    VoiceOutputChannel,
+)
+from rasa.shared.utils.common import mark_as_beta_feature
+
+logger = structlog.get_logger(__name__)
+PREFERRED_AUDIO_FORMAT = "raw/mulaw"
+
+
+class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "audiocodes_stream"
+
+    def _ensure_stream_id(self) -> None:
+        """Audiocodes requires a stream ID with playStream messages."""
+        if "stream_id" not in call_state.channel_data:
+            call_state.channel_data["stream_id"] = 0
+
+    def _increment_stream_id(self) -> None:
+        self._ensure_stream_id()
+        call_state.channel_data["stream_id"] += 1
+
+    def _get_stream_id(self) -> str:
+        self._ensure_stream_id()
+        return str(call_state.channel_data["stream_id"])
+
+    def rasa_audio_bytes_to_channel_bytes(
+        self, rasa_audio_bytes: RasaAudioBytes
+    ) -> bytes:
+        return base64.b64encode(rasa_audio_bytes)
+
+    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
+        media_message = json.dumps(
+            {
+                "type": "playStream.chunk",
+                "streamId": self._get_stream_id(),
+                "audioChunk": channel_bytes.decode("utf-8"),
+            }
+        )
+        return media_message
+
+    async def send_start_marker(self, recipient_id: str) -> None:
+        """Send playStream.start before first audio chunk."""
+        self._increment_stream_id()
+        media_message = json.dumps(
+            {
+                "type": "playStream.start",
+                "streamId": self._get_stream_id(),
+                "mediaFormat": PREFERRED_AUDIO_FORMAT,
+            }
+        )
+        logger.debug("Sending start marker", stream_id=self._get_stream_id())
+        await self.voice_websocket.send(media_message)
+
+    async def send_intermediate_marker(self, recipient_id: str) -> None:
+        """Audiocodes doesn't need intermediate markers, so do nothing."""
+        pass
+
+    async def send_end_marker(self, recipient_id: str) -> None:
+        """Send playStream.stop after last audio chunk."""
+        media_message = json.dumps(
+            {
+                "type": "playStream.stop",
+                "streamId": self._get_stream_id(),
+            }
+        )
+        logger.debug("Sending end marker", stream_id=self._get_stream_id())
+        await self.voice_websocket.send(media_message)
+
+
+class AudiocodesVoiceInputChannel(VoiceInputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "audiocodes_stream"
+
+    def __init__(
+        self,
+        server_url: str,
+        asr_config: Dict,
+        tts_config: Dict,
+        monitor_silence: bool = False,
+    ):
+        mark_as_beta_feature("Audiocodes (audiocodes_stream) Channel")
+        super().__init__(server_url, asr_config, tts_config, monitor_silence)
+
+    def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
+        return RasaAudioBytes(base64.b64decode(input_bytes))
+
+    async def collect_call_parameters(
+        self, channel_websocket: Websocket
+    ) -> Optional[CallParameters]:
+        async for message in channel_websocket:
+            data = json.loads(message)
+            if data["type"] == "session.initiate":
+                # contains info about mediaformats
+                logger.info(
+                    "audiocodes_stream.collect_call_parameters.session.initiate",
+                    data=data,
+                )
+                self._send_accepted(channel_websocket, data)
+            elif data["type"] == "activities":
+                activities = data["activities"]
+                for activity in activities:
+                    logger.debug(
+                        "audiocodes_stream.collect_call_parameters.activity",
+                        data=activity,
+                    )
+                    if activity["name"] == "start":
+                        return map_call_params(activity["parameters"])
+            else:
+                logger.warning("audiocodes_stream.unknown_message", data=data)
+        return None
+
+    def map_input_message(
+        self,
+        message: Any,
+        ws: Websocket,
+    ) -> VoiceChannelAction:
+        data = json.loads(message)
+        if data["type"] == "activities":
+            activities = data["activities"]
+            for activity in activities:
+                logger.debug("audiocodes_stream.activity", data=activity)
+                if activity["name"] == "start":
+                    # already handled in collect_call_parameters
+                    pass
+                elif activity["name"] == "dtmf":
+                    # TODO: handle DTMF input
+                    pass
+                elif activity["name"] == "playFinished":
+                    logger.debug("audiocodes_stream.playFinished", data=activity)
+                    if call_state.should_hangup:
+                        logger.info("audiocodes.hangup")
+                        self._send_hangup(ws, data)
+                        # the conversation should continue until
+                        # we receive a end message from audiocodes
+                    pass
+                else:
+                    logger.warning("audiocodes_stream.unknown_activity", data=activity)
+        elif data["type"] == "userStream.start":
+            logger.debug("audiocodes_stream.userStream.start", data=data)
+            self._send_recognition_started(ws, data)
+        elif data["type"] == "userStream.chunk":
+            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(data["audioChunk"])
+            return NewAudioAction(audio_bytes)
+        elif data["type"] == "userStream.stop":
+            logger.debug("audiocodes_stream.stop_recognition", data=data)
+            self._send_recognition_ended(ws, data)
+        elif data["type"] == "session.resume":
+            logger.debug("audiocodes_stream.resume", data=data)
+            self._send_accepted(ws, data)
+        elif data["type"] == "session.end":
+            logger.debug("audiocodes_stream.end", data=data)
+            return EndConversationAction()
+        elif data["type"] == "connection.validate":
+            # not part of call flow; only sent when integration is created
+            self._send_validated(ws, data)
+        else:
+            logger.warning("audiocodes_stream.unknown_message", data=data)
+
+        return ContinueConversationAction()
+
+    def _send_accepted(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        supported_formats = data.get("supportedMediaFormats", [])
+        preferred_format = PREFERRED_AUDIO_FORMAT
+
+        if preferred_format not in supported_formats:
+            logger.warning(
+                "audiocodes_stream.format_not_supported",
+                supported_formats=supported_formats,
+                preferred_format=preferred_format,
+            )
+            raise
+
+        payload = {
+            "type": "session.accepted",
+            "mediaFormat": PREFERRED_AUDIO_FORMAT,
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_recognition_started(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {"type": "userStream.started"}
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_recognition_ended(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {"type": "userStream.stopped"}
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_hypothesis(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        """
+        TODO: The hypothesis message is sent by the bot to provide partial
+        recognition results. Using this message is recommended,
+        as VAIC relies on it for performing barge-in.
+        """
+        pass
+
+    def _send_recognition(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        """
+        TODO: The recognition message is sent by the bot to provide
+        the final recognition result. Using this message is recommended
+        mainly for logging purposes.
+        """
+        pass
+
+    def _send_hangup(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {
+            "conversationId": data["conversationId"],
+            "type": "activities",
+            "activities": [{"type": "event", "name": "hangup"}],
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_validated(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {
+            "type": "connection.validated",
+            "success": True,
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def create_output_channel(
+        self, voice_websocket: Websocket, tts_engine: TTSEngine
+    ) -> VoiceOutputChannel:
+        return AudiocodesVoiceOutputChannel(
+            voice_websocket,
+            tts_engine,
+            self.tts_cache,
+        )
+
+    def blueprint(
+        self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
+    ) -> Blueprint:
+        """Defines a Sanic bluelogger.debug."""
+        blueprint = Blueprint("audiocodes_stream", __name__)
+
+        @blueprint.route("/", methods=["GET"])
+        async def health(_: Request) -> HTTPResponse:
+            return response.json({"status": "ok"})
+
+        @blueprint.websocket("/websocket")  # type: ignore
+        async def receive(request: Request, ws: Websocket) -> None:
+            # TODO: validate API key header
+            logger.info("audiocodes.receive", message="Starting audio streaming")
+            try:
+                await self.run_audio_streaming(on_new_message, ws)
+            except Exception as e:
+                logger.exception(
+                    "audiocodes.receive",
+                    message="Error during audio streaming",
+                    error=e,
+                )
+                # return 500 error
+                raise
+
+        return blueprint
+
+
+def _schedule_async_task(coro: Awaitable[Any]) -> None:
+    """Helper function to schedule a coroutine in the event loop.
+
+    Args:
+        coro: The coroutine to schedule
+    """
+    loop = asyncio.get_running_loop()
+    loop.call_soon_threadsafe(lambda: loop.create_task(coro))

rasa/core/channels/voice_stream/browser_audio.py

@@ -1,20 +1,23 @@
 import audioop
 import base64
 import json
-
-import structlog
 import uuid
 from typing import Any, Awaitable, Callable, Optional, Tuple
 
-from sanic import Blueprint, HTTPResponse, Request, response
-from sanic import Websocket  # type: ignore
-
+import structlog
+from sanic import (  # type: ignore[attr-defined]
+    Blueprint,
+    HTTPResponse,
+    Request,
+    Websocket,
+    response,
+)
 
 from rasa.core.channels import UserMessage
 from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.call_state import call_state
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
-from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.voice_channel import (
     ContinueConversationAction,
     EndConversationAction,
@@ -62,6 +65,7 @@ class BrowserAudioInputChannel(VoiceInputChannel):
     def map_input_message(
         self,
         message: Any,
+        ws: Websocket,
     ) -> VoiceChannelAction:
         data = json.loads(message)
         if "audio" in data:
@@ -102,6 +106,9 @@ class BrowserAudioInputChannel(VoiceInputChannel):
 
         @blueprint.websocket("/websocket")  # type: ignore
         async def handle_message(request: Request, ws: Websocket) -> None:
-            await self.run_audio_streaming(on_new_message, ws)
+            try:
+                await self.run_audio_streaming(on_new_message, ws)
+            except Exception as e:
+                logger.error("browser_audio.handle_message.error", error=e)
 
         return blueprint

rasa/core/channels/voice_stream/call_state.py

@@ -1,8 +1,9 @@
 import asyncio
 from contextvars import ContextVar
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
 from werkzeug.local import LocalProxy
-from dataclasses import dataclass
-from typing import Optional
 
 
 # Per voice session data
@@ -18,6 +19,9 @@ class CallState:
     should_hangup: bool = False
     connection_failed: bool = False
 
+    # Generic field for channel-specific state data
+    channel_data: Dict[str, Any] = field(default_factory=dict)
+
 
 _call_state: ContextVar[CallState] = ContextVar("call_state")
 call_state = LocalProxy(_call_state)