rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/core/channels/voice_stream/asr/deepgram.py

@@ -0,0 +1,90 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+import json
+import os
+
+import websockets
+from websockets.legacy.client import WebSocketClientProtocol
+
+from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine, ASREngineConfig
+from rasa.core.channels.voice_stream.asr.asr_event import (
+    ASREvent,
+    NewTranscript,
+    UserIsSpeaking,
+)
+from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.shared.constants import DEEPGRAM_API_KEY_ENV_VAR
+
+
+@dataclass
+class DeepgramASRConfig(ASREngineConfig):
+    endpoint: Optional[str] = None
+    # number of miliseconds of silence to determine end of speech
+    endpointing: Optional[int] = None
+    language: Optional[str] = None
+    model: Optional[str] = None
+    smart_format: Optional[bool] = None
+
+
+class DeepgramASR(ASREngine[DeepgramASRConfig]):
+    required_env_vars = (DEEPGRAM_API_KEY_ENV_VAR,)
+
+    def __init__(self, config: Optional[DeepgramASRConfig] = None):
+        super().__init__(config)
+        self.accumulated_transcript = ""
+
+    async def open_websocket_connection(self) -> WebSocketClientProtocol:
+        """Connect to the ASR system."""
+        deepgram_api_key = os.environ[DEEPGRAM_API_KEY_ENV_VAR]
+        extra_headers = {"Authorization": f"Token {deepgram_api_key}"}
+        api_url = self._get_api_url()
+        query_params = self._get_query_params()
+        return await websockets.connect(  # type: ignore
+            api_url + query_params,
+            extra_headers=extra_headers,
+        )
+
+    def _get_api_url(self) -> str:
+        return f"wss://{self.config.endpoint}/v1/listen?"
+
+    def _get_query_params(self) -> str:
+        return (
+            f"encoding=mulaw&sample_rate={HERTZ}&endpointing={self.config.endpointing}"
+            f"&vad_events=true&language={self.config.language}&interim_results=true"
+            f"&model={self.config.model}&smart_format={str(self.config.smart_format).lower()}"
+        )
+
+    async def signal_audio_done(self) -> None:
+        """Signal to the ASR Api that you are done sending data."""
+        if self.asr_socket is None:
+            raise AttributeError("Websocket not connected.")
+        await self.asr_socket.send(json.dumps({"type": "CloseStream"}))
+
+    def rasa_audio_bytes_to_engine_bytes(self, chunk: RasaAudioBytes) -> bytes:
+        """Convert RasaAudioBytes to bytes usable by this engine."""
+        return chunk
+
+    def engine_event_to_asr_event(self, e: Any) -> Optional[ASREvent]:
+        """Translate an engine event to a common ASREvent."""
+        data = json.loads(e)
+        if "is_final" in data:
+            transcript = data["channel"]["alternatives"][0]["transcript"]
+            if data["is_final"]:
+                if data.get("speech_final"):
+                    full_transcript = self.accumulated_transcript + transcript
+                    self.accumulated_transcript = ""
+                    if full_transcript:
+                        return NewTranscript(full_transcript)
+                else:
+                    self.accumulated_transcript += transcript
+            elif transcript:
+                return UserIsSpeaking()
+        return None
+
+    @staticmethod
+    def get_default_config() -> DeepgramASRConfig:
+        return DeepgramASRConfig("api.deepgram.com", 400, "en", "nova-2-general", True)
+
+    @classmethod
+    def from_config_dict(cls, config: Dict) -> "DeepgramASR":
+        return DeepgramASR(DeepgramASRConfig.from_dict(config))

rasa/core/channels/voice_stream/audio_bytes.py

@@ -0,0 +1,8 @@
+from typing import NewType
+
+# a common intermediate audio byte format that acts as a common data format,
+# to prevent quadratic complexity between formats of channels, asr engines,
+# and tts engines
+# currently corresponds to raw wave, 8khz, 8bit, mono channel, mulaw encoding
+RasaAudioBytes = NewType("RasaAudioBytes", bytes)
+HERTZ = 8000

rasa/core/channels/voice_stream/browser_audio.py

@@ -0,0 +1,107 @@
+import audioop
+import base64
+import json
+
+import structlog
+import uuid
+from typing import Any, Awaitable, Callable, Optional, Tuple
+
+from sanic import Blueprint, HTTPResponse, Request, response
+from sanic import Websocket  # type: ignore
+
+
+from rasa.core.channels import UserMessage
+from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.call_state import call_state
+from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.voice_channel import (
+    ContinueConversationAction,
+    EndConversationAction,
+    NewAudioAction,
+    VoiceChannelAction,
+    VoiceInputChannel,
+    VoiceOutputChannel,
+)
+
+logger = structlog.get_logger()
+
+
+class BrowserAudioOutputChannel(VoiceOutputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "browser_audio"
+
+    def rasa_audio_bytes_to_channel_bytes(
+        self, rasa_audio_bytes: RasaAudioBytes
+    ) -> bytes:
+        return audioop.ulaw2lin(rasa_audio_bytes, 4)
+
+    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
+        return json.dumps({"audio": base64.b64encode(channel_bytes).decode("utf-8")})
+
+    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
+        message_id = uuid.uuid4().hex
+        return json.dumps({"marker": message_id}), message_id
+
+
+class BrowserAudioInputChannel(VoiceInputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "browser_audio"
+
+    def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
+        return RasaAudioBytes(audioop.lin2ulaw(input_bytes, 4))
+
+    async def collect_call_parameters(
+        self, channel_websocket: Websocket
+    ) -> Optional[CallParameters]:
+        call_id = f"inspect-{uuid.uuid4()}"
+        return CallParameters(call_id, "local", "local", stream_id=call_id)
+
+    def map_input_message(
+        self,
+        message: Any,
+    ) -> VoiceChannelAction:
+        data = json.loads(message)
+        if "audio" in data:
+            channel_bytes = base64.b64decode(data["audio"])
+            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(channel_bytes)
+            return NewAudioAction(audio_bytes)
+        elif "marker" in data:
+            if data["marker"] == call_state.latest_bot_audio_id:
+                # Just finished streaming last audio bytes
+                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
+                if call_state.should_hangup:
+                    logger.debug(
+                        "browser_audio.hangup", marker=call_state.latest_bot_audio_id
+                    )
+                    return EndConversationAction()
+            else:
+                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+        return ContinueConversationAction()
+
+    def create_output_channel(
+        self, voice_websocket: Websocket, tts_engine: TTSEngine
+    ) -> VoiceOutputChannel:
+        return BrowserAudioOutputChannel(
+            voice_websocket,
+            tts_engine,
+            self.tts_cache,
+        )
+
+    def blueprint(
+        self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
+    ) -> Blueprint:
+        """Defines a Sanic bluelogger.debug."""
+        blueprint = Blueprint("browser_audio", __name__)
+
+        @blueprint.route("/", methods=["GET"])
+        async def health(_: Request) -> HTTPResponse:
+            return response.json({"status": "ok"})
+
+        @blueprint.websocket("/websocket")  # type: ignore
+        async def handle_message(request: Request, ws: Websocket) -> None:
+            await self.run_audio_streaming(on_new_message, ws)
+
+        return blueprint

rasa/core/channels/voice_stream/call_state.py

@@ -0,0 +1,23 @@
+import asyncio
+from contextvars import ContextVar
+from werkzeug.local import LocalProxy
+from dataclasses import dataclass
+from typing import Optional
+
+
+# Per voice session data
+# This is similar to how flask makes the "request" object available as a global variable
+# It's a "global" variable that is local to an async task (i.e. websocket session)
+@dataclass
+class CallState:
+    is_user_speaking: bool = False
+    is_bot_speaking: bool = False
+    silence_timeout_watcher: Optional[asyncio.Task] = None
+    silence_timeout: Optional[float] = None
+    latest_bot_audio_id: Optional[str] = None
+    should_hangup: bool = False
+    connection_failed: bool = False
+
+
+_call_state: ContextVar[CallState] = ContextVar("call_state")
+call_state = LocalProxy(_call_state)

rasa/core/channels/voice_stream/tts/azure.py

@@ -0,0 +1,106 @@
+import os
+from typing import AsyncIterator, Dict, Optional
+from dataclasses import dataclass
+
+import aiohttp
+import structlog
+from aiohttp import ClientConnectorError, ClientTimeout
+
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.tts.tts_engine import (
+    TTSEngine,
+    TTSEngineConfig,
+    TTSError,
+)
+from rasa.shared.constants import AZURE_SPEECH_API_KEY_ENV_VAR
+from rasa.shared.exceptions import ConnectionException
+
+
+structlogger = structlog.get_logger()
+
+
+@dataclass
+class AzureTTSConfig(TTSEngineConfig):
+    speech_region: Optional[str] = None
+
+
+class AzureTTS(TTSEngine[AzureTTSConfig]):
+    session: Optional[aiohttp.ClientSession] = None
+    required_env_vars = (AZURE_SPEECH_API_KEY_ENV_VAR,)
+
+    def __init__(self, config: Optional[AzureTTSConfig] = None):
+        super().__init__(config)
+        timeout = ClientTimeout(total=self.config.timeout)
+        # Have to create this class-shared session lazily at run time otherwise
+        # the async event loop doesn't work
+        if self.__class__.session is None or self.__class__.session.closed:
+            self.__class__.session = aiohttp.ClientSession(timeout=timeout)
+
+    async def synthesize(
+        self, text: str, config: Optional[AzureTTSConfig] = None
+    ) -> AsyncIterator[RasaAudioBytes]:
+        """Generate speech from text using a remote TTS system."""
+        config = self.config.merge(config)
+        azure_speech_url = self.get_tts_endpoint(config)
+        headers = self.get_request_headers()
+        body = self.create_request_body(text, config)
+        if self.session is None:
+            raise ConnectionException("Client session is not initialized")
+        try:
+            async with self.session.post(
+                azure_speech_url, headers=headers, data=body, chunked=True
+            ) as response:
+                if 200 <= response.status < 300:
+                    async for data in response.content.iter_chunked(1024):
+                        yield self.engine_bytes_to_rasa_audio_bytes(data)
+                    return
+                else:
+                    structlogger.error(
+                        "azure.synthesize.rest.failed",
+                        status_code=response.status,
+                        msg=response.text(),
+                    )
+                    raise TTSError(f"TTS failed: {response.text()}")
+        except ClientConnectorError as e:
+            raise TTSError(e)
+        except TimeoutError as e:
+            raise TTSError(e)
+
+    @staticmethod
+    def get_request_headers() -> dict[str, str]:
+        azure_speech_api_key = os.environ[AZURE_SPEECH_API_KEY_ENV_VAR]
+        return {
+            "Ocp-Apim-Subscription-Key": azure_speech_api_key,
+            "Content-Type": "application/ssml+xml",
+            "X-Microsoft-OutputFormat": "raw-8khz-8bit-mono-mulaw",
+        }
+
+    @staticmethod
+    def get_tts_endpoint(config: AzureTTSConfig) -> str:
+        return f"https://{config.speech_region}.tts.speech.microsoft.com/cognitiveservices/v1"
+
+    @staticmethod
+    def create_request_body(text: str, conf: AzureTTSConfig) -> str:
+        return f"""
+        <speak version='1.0' xml:lang='{conf.language}'>
+            <voice xml:lang='{conf.language}' name='{conf.voice}'>
+                {text}
+            </voice>
+        </speak>"""
+
+    def engine_bytes_to_rasa_audio_bytes(self, chunk: bytes) -> RasaAudioBytes:
+        """Convert the generated tts audio bytes into rasa audio bytes."""
+        return RasaAudioBytes(chunk)
+
+    @staticmethod
+    def get_default_config() -> AzureTTSConfig:
+        return AzureTTSConfig(
+            language="en-US",
+            voice="en-US-JennyNeural",
+            timeout=10,
+            speech_region="germanywestcentral",
+        )
+
+    @classmethod
+    def from_config_dict(cls, config: Dict) -> "AzureTTS":
+        return cls(AzureTTSConfig.from_dict(config))

rasa/core/channels/voice_stream/tts/cartesia.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass
+from typing import AsyncIterator, Dict, Optional
+import os
+import aiohttp
+import structlog
+from aiohttp import ClientConnectorError, ClientTimeout
+
+from rasa.core.channels.voice_stream.tts.tts_engine import (
+    TTSEngineConfig,
+)
+
+from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine, TTSError
+from rasa.shared.constants import CARTESIA_API_KEY_ENV_VAR
+from rasa.shared.exceptions import ConnectionException
+
+structlogger = structlog.get_logger()
+
+
+@dataclass
+class CartesiaTTSConfig(TTSEngineConfig):
+    model_id: Optional[str] = None
+    version: Optional[str] = None
+
+
+class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
+    session: Optional[aiohttp.ClientSession] = None
+    required_env_vars = (CARTESIA_API_KEY_ENV_VAR,)
+
+    def __init__(self, config: Optional[CartesiaTTSConfig] = None):
+        super().__init__(config)
+        timeout = ClientTimeout(total=self.config.timeout)
+        # Have to create this class-shared session lazily at run time otherwise
+        # the async event loop doesn't work
+        if self.__class__.session is None or self.__class__.session.closed:
+            self.__class__.session = aiohttp.ClientSession(timeout=timeout)
+
+    @staticmethod
+    def get_tts_endpoint() -> str:
+        """Create the endpoint string for cartesia."""
+        return "https://api.cartesia.ai/tts/bytes"
+
+    @staticmethod
+    def get_request_body(text: str, config: CartesiaTTSConfig) -> Dict:
+        """Create the request body for cartesia."""
+        # more info on payload:
+        # https://docs.cartesia.ai/reference/api-reference/rest/stream-speech-bytes
+        return {
+            "model_id": config.model_id,
+            "transcript": text,
+            "language": config.language,
+            "voice": {
+                "mode": "id",
+                "id": config.voice,
+            },
+            "output_format": {
+                "container": "raw",
+                "encoding": "pcm_mulaw",
+                "sample_rate": HERTZ,
+            },
+        }
+
+    @staticmethod
+    def get_request_headers(config: CartesiaTTSConfig) -> dict[str, str]:
+        cartesia_api_key = os.environ[CARTESIA_API_KEY_ENV_VAR]
+        return {
+            "Cartesia-Version": str(config.version),
+            "Content-Type": "application/json",
+            "X-API-Key": str(cartesia_api_key),
+        }
+
+    async def synthesize(
+        self, text: str, config: Optional[CartesiaTTSConfig] = None
+    ) -> AsyncIterator[RasaAudioBytes]:
+        """Generate speech from text using a remote TTS system."""
+        config = self.config.merge(config)
+        payload = self.get_request_body(text, config)
+        headers = self.get_request_headers(config)
+        url = self.get_tts_endpoint()
+        if self.session is None:
+            raise ConnectionException("Client session is not initialized")
+        try:
+            async with self.session.post(
+                url, headers=headers, json=payload, chunked=True
+            ) as response:
+                if 200 <= response.status < 300:
+                    async for data in response.content.iter_chunked(1024):
+                        yield self.engine_bytes_to_rasa_audio_bytes(data)
+                    return
+                else:
+                    structlogger.error(
+                        "cartesia.synthesize.rest.failed",
+                        status_code=response.status,
+                        msg=response.text(),
+                    )
+                    raise TTSError(f"TTS failed: {response.text()}")
+        except ClientConnectorError as e:
+            raise TTSError(e)
+        except TimeoutError as e:
+            raise TTSError(e)
+
+    def engine_bytes_to_rasa_audio_bytes(self, chunk: bytes) -> RasaAudioBytes:
+        """Convert the generated tts audio bytes into rasa audio bytes."""
+        return RasaAudioBytes(chunk)
+
+    @staticmethod
+    def get_default_config() -> CartesiaTTSConfig:
+        return CartesiaTTSConfig(
+            language="en",
+            voice="248be419-c632-4f23-adf1-5324ed7dbf1d",
+            timeout=10,
+            model_id="sonic-english",
+            version="2024-06-10",
+        )
+
+    @classmethod
+    def from_config_dict(cls, config: Dict) -> "CartesiaTTS":
+        return cls(CartesiaTTSConfig.from_dict(config))

rasa/core/channels/voice_stream/tts/tts_cache.py

@@ -0,0 +1,27 @@
+from typing import Optional
+from collections import OrderedDict
+import logging
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+
+logger = logging.getLogger(__name__)
+
+
+class TTSCache:
+    """An LRU Cache for TTS based on pythons OrderedDict."""
+
+    def __init__(self, max_size: int):
+        self.cache: OrderedDict[str, RasaAudioBytes] = OrderedDict()
+        self.max_size = max_size
+
+    def get(self, text: str) -> Optional[RasaAudioBytes]:
+        if text not in self.cache:
+            return None
+        else:
+            self.cache.move_to_end(text)
+            return self.cache[text]
+
+    def put(self, text: str, audio_bytes: RasaAudioBytes) -> None:
+        self.cache[text] = audio_bytes
+        self.cache.move_to_end(text)
+        if len(self.cache) > self.max_size:
+            self.cache.popitem(last=False)

rasa/core/channels/voice_stream/tts/tts_engine.py

@@ -0,0 +1,58 @@
+from typing import AsyncIterator, Dict, Generic, Optional, Tuple, Type, TypeVar
+from dataclasses import dataclass
+
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.util import MergeableConfig
+from rasa.shared.exceptions import RasaException
+from rasa.shared.utils.common import validate_environment
+
+
+class TTSError(RasaException):
+    pass
+
+
+T = TypeVar("T", bound="TTSEngineConfig")
+E = TypeVar("E", bound="TTSEngine")
+
+
+@dataclass
+class TTSEngineConfig(MergeableConfig):
+    language: Optional[str] = None
+    voice: Optional[str] = None
+    timeout: Optional[int] = None
+
+
+class TTSEngine(Generic[T]):
+    required_env_vars: Tuple[str, ...] = ()
+    required_packages: Tuple[str, ...] = ()
+
+    def __init__(self, config: Optional[T] = None):
+        self.config = self.get_default_config().merge(config)
+        validate_environment(
+            self.required_env_vars,
+            self.required_packages,
+            f"TTS Engine {self.__class__.__name__}",
+        )
+
+    async def close_connection(self) -> None:
+        """Cleanup the connection if necessary."""
+        return
+
+    async def synthesize(
+        self, text: str, config: Optional[T] = None
+    ) -> AsyncIterator[RasaAudioBytes]:
+        """Generate speech from text using a remote TTS system."""
+        yield RasaAudioBytes(b"")
+
+    def engine_bytes_to_rasa_audio_bytes(self, chunk: bytes) -> RasaAudioBytes:
+        """Convert the generated tts audio bytes into rasa audio bytes."""
+        raise NotImplementedError
+
+    @staticmethod
+    def get_default_config() -> T:
+        """Get the default config for this component."""
+        raise NotImplementedError
+
+    @classmethod
+    def from_config_dict(cls: Type[E], config: Dict) -> E:
+        raise NotImplementedError