rasa-pro 3.10.15__py3-none-any.whl → 3.11.0__py3-none-any.whl

rasa/core/channels/voice_stream/call_state.py

@@ -0,0 +1,23 @@
+import asyncio
+from contextvars import ContextVar
+from werkzeug.local import LocalProxy
+from dataclasses import dataclass
+from typing import Optional
+
+
+# Per voice session data
+# This is similar to how flask makes the "request" object available as a global variable
+# It's a "global" variable that is local to an async task (i.e. websocket session)
+@dataclass
+class CallState:
+    is_user_speaking: bool = False
+    is_bot_speaking: bool = False
+    silence_timeout_watcher: Optional[asyncio.Task] = None
+    silence_timeout: Optional[float] = None
+    latest_bot_audio_id: Optional[str] = None
+    should_hangup: bool = False
+    connection_failed: bool = False
+
+
+_call_state: ContextVar[CallState] = ContextVar("call_state")
+call_state = LocalProxy(_call_state)

rasa/core/channels/voice_stream/tts/azure.py

@@ -0,0 +1,106 @@
+import os
+from typing import AsyncIterator, Dict, Optional
+from dataclasses import dataclass
+
+import aiohttp
+import structlog
+from aiohttp import ClientConnectorError, ClientTimeout
+
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.tts.tts_engine import (
+    TTSEngine,
+    TTSEngineConfig,
+    TTSError,
+)
+from rasa.shared.constants import AZURE_SPEECH_API_KEY_ENV_VAR
+from rasa.shared.exceptions import ConnectionException
+
+
+structlogger = structlog.get_logger()
+
+
+@dataclass
+class AzureTTSConfig(TTSEngineConfig):
+    speech_region: Optional[str] = None
+
+
+class AzureTTS(TTSEngine[AzureTTSConfig]):
+    session: Optional[aiohttp.ClientSession] = None
+    required_env_vars = (AZURE_SPEECH_API_KEY_ENV_VAR,)
+
+    def __init__(self, config: Optional[AzureTTSConfig] = None):
+        super().__init__(config)
+        timeout = ClientTimeout(total=self.config.timeout)
+        # Have to create this class-shared session lazily at run time otherwise
+        # the async event loop doesn't work
+        if self.__class__.session is None or self.__class__.session.closed:
+            self.__class__.session = aiohttp.ClientSession(timeout=timeout)
+
+    async def synthesize(
+        self, text: str, config: Optional[AzureTTSConfig] = None
+    ) -> AsyncIterator[RasaAudioBytes]:
+        """Generate speech from text using a remote TTS system."""
+        config = self.config.merge(config)
+        azure_speech_url = self.get_tts_endpoint(config)
+        headers = self.get_request_headers()
+        body = self.create_request_body(text, config)
+        if self.session is None:
+            raise ConnectionException("Client session is not initialized")
+        try:
+            async with self.session.post(
+                azure_speech_url, headers=headers, data=body, chunked=True
+            ) as response:
+                if 200 <= response.status < 300:
+                    async for data in response.content.iter_chunked(1024):
+                        yield self.engine_bytes_to_rasa_audio_bytes(data)
+                    return
+                else:
+                    structlogger.error(
+                        "azure.synthesize.rest.failed",
+                        status_code=response.status,
+                        msg=response.text(),
+                    )
+                    raise TTSError(f"TTS failed: {response.text()}")
+        except ClientConnectorError as e:
+            raise TTSError(e)
+        except TimeoutError as e:
+            raise TTSError(e)
+
+    @staticmethod
+    def get_request_headers() -> dict[str, str]:
+        azure_speech_api_key = os.environ[AZURE_SPEECH_API_KEY_ENV_VAR]
+        return {
+            "Ocp-Apim-Subscription-Key": azure_speech_api_key,
+            "Content-Type": "application/ssml+xml",
+            "X-Microsoft-OutputFormat": "raw-8khz-8bit-mono-mulaw",
+        }
+
+    @staticmethod
+    def get_tts_endpoint(config: AzureTTSConfig) -> str:
+        return f"https://{config.speech_region}.tts.speech.microsoft.com/cognitiveservices/v1"
+
+    @staticmethod
+    def create_request_body(text: str, conf: AzureTTSConfig) -> str:
+        return f"""
+        <speak version='1.0' xml:lang='{conf.language}'>
+            <voice xml:lang='{conf.language}' name='{conf.voice}'>
+                {text}
+            </voice>
+        </speak>"""
+
+    def engine_bytes_to_rasa_audio_bytes(self, chunk: bytes) -> RasaAudioBytes:
+        """Convert the generated tts audio bytes into rasa audio bytes."""
+        return RasaAudioBytes(chunk)
+
+    @staticmethod
+    def get_default_config() -> AzureTTSConfig:
+        return AzureTTSConfig(
+            language="en-US",
+            voice="en-US-JennyNeural",
+            timeout=10,
+            speech_region="germanywestcentral",
+        )
+
+    @classmethod
+    def from_config_dict(cls, config: Dict) -> "AzureTTS":
+        return cls(AzureTTSConfig.from_dict(config))

rasa/core/channels/voice_stream/tts/cartesia.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass
+from typing import AsyncIterator, Dict, Optional
+import os
+import aiohttp
+import structlog
+from aiohttp import ClientConnectorError, ClientTimeout
+
+from rasa.core.channels.voice_stream.tts.tts_engine import (
+    TTSEngineConfig,
+)
+
+from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine, TTSError
+from rasa.shared.constants import CARTESIA_API_KEY_ENV_VAR
+from rasa.shared.exceptions import ConnectionException
+
+structlogger = structlog.get_logger()
+
+
+@dataclass
+class CartesiaTTSConfig(TTSEngineConfig):
+    model_id: Optional[str] = None
+    version: Optional[str] = None
+
+
+class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
+    session: Optional[aiohttp.ClientSession] = None
+    required_env_vars = (CARTESIA_API_KEY_ENV_VAR,)
+
+    def __init__(self, config: Optional[CartesiaTTSConfig] = None):
+        super().__init__(config)
+        timeout = ClientTimeout(total=self.config.timeout)
+        # Have to create this class-shared session lazily at run time otherwise
+        # the async event loop doesn't work
+        if self.__class__.session is None or self.__class__.session.closed:
+            self.__class__.session = aiohttp.ClientSession(timeout=timeout)
+
+    @staticmethod
+    def get_tts_endpoint() -> str:
+        """Create the endpoint string for cartesia."""
+        return "https://api.cartesia.ai/tts/bytes"
+
+    @staticmethod
+    def get_request_body(text: str, config: CartesiaTTSConfig) -> Dict:
+        """Create the request body for cartesia."""
+        # more info on payload:
+        # https://docs.cartesia.ai/reference/api-reference/rest/stream-speech-bytes
+        return {
+            "model_id": config.model_id,
+            "transcript": text,
+            "language": config.language,
+            "voice": {
+                "mode": "id",
+                "id": config.voice,
+            },
+            "output_format": {
+                "container": "raw",
+                "encoding": "pcm_mulaw",
+                "sample_rate": HERTZ,
+            },
+        }
+
+    @staticmethod
+    def get_request_headers(config: CartesiaTTSConfig) -> dict[str, str]:
+        cartesia_api_key = os.environ[CARTESIA_API_KEY_ENV_VAR]
+        return {
+            "Cartesia-Version": str(config.version),
+            "Content-Type": "application/json",
+            "X-API-Key": str(cartesia_api_key),
+        }
+
+    async def synthesize(
+        self, text: str, config: Optional[CartesiaTTSConfig] = None
+    ) -> AsyncIterator[RasaAudioBytes]:
+        """Generate speech from text using a remote TTS system."""
+        config = self.config.merge(config)
+        payload = self.get_request_body(text, config)
+        headers = self.get_request_headers(config)
+        url = self.get_tts_endpoint()
+        if self.session is None:
+            raise ConnectionException("Client session is not initialized")
+        try:
+            async with self.session.post(
+                url, headers=headers, json=payload, chunked=True
+            ) as response:
+                if 200 <= response.status < 300:
+                    async for data in response.content.iter_chunked(1024):
+                        yield self.engine_bytes_to_rasa_audio_bytes(data)
+                    return
+                else:
+                    structlogger.error(
+                        "cartesia.synthesize.rest.failed",
+                        status_code=response.status,
+                        msg=response.text(),
+                    )
+                    raise TTSError(f"TTS failed: {response.text()}")
+        except ClientConnectorError as e:
+            raise TTSError(e)
+        except TimeoutError as e:
+            raise TTSError(e)
+
+    def engine_bytes_to_rasa_audio_bytes(self, chunk: bytes) -> RasaAudioBytes:
+        """Convert the generated tts audio bytes into rasa audio bytes."""
+        return RasaAudioBytes(chunk)
+
+    @staticmethod
+    def get_default_config() -> CartesiaTTSConfig:
+        return CartesiaTTSConfig(
+            language="en",
+            voice="248be419-c632-4f23-adf1-5324ed7dbf1d",
+            timeout=10,
+            model_id="sonic-english",
+            version="2024-06-10",
+        )
+
+    @classmethod
+    def from_config_dict(cls, config: Dict) -> "CartesiaTTS":
+        return cls(CartesiaTTSConfig.from_dict(config))

rasa/core/channels/voice_stream/tts/tts_cache.py

@@ -0,0 +1,27 @@
+from typing import Optional
+from collections import OrderedDict
+import logging
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+
+logger = logging.getLogger(__name__)
+
+
+class TTSCache:
+    """An LRU Cache for TTS based on pythons OrderedDict."""
+
+    def __init__(self, max_size: int):
+        self.cache: OrderedDict[str, RasaAudioBytes] = OrderedDict()
+        self.max_size = max_size
+
+    def get(self, text: str) -> Optional[RasaAudioBytes]:
+        if text not in self.cache:
+            return None
+        else:
+            self.cache.move_to_end(text)
+            return self.cache[text]
+
+    def put(self, text: str, audio_bytes: RasaAudioBytes) -> None:
+        self.cache[text] = audio_bytes
+        self.cache.move_to_end(text)
+        if len(self.cache) > self.max_size:
+            self.cache.popitem(last=False)

rasa/core/channels/voice_stream/tts/tts_engine.py

@@ -0,0 +1,58 @@
+from typing import AsyncIterator, Dict, Generic, Optional, Tuple, Type, TypeVar
+from dataclasses import dataclass
+
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.util import MergeableConfig
+from rasa.shared.exceptions import RasaException
+from rasa.shared.utils.common import validate_environment
+
+
+class TTSError(RasaException):
+    pass
+
+
+T = TypeVar("T", bound="TTSEngineConfig")
+E = TypeVar("E", bound="TTSEngine")
+
+
+@dataclass
+class TTSEngineConfig(MergeableConfig):
+    language: Optional[str] = None
+    voice: Optional[str] = None
+    timeout: Optional[int] = None
+
+
+class TTSEngine(Generic[T]):
+    required_env_vars: Tuple[str, ...] = ()
+    required_packages: Tuple[str, ...] = ()
+
+    def __init__(self, config: Optional[T] = None):
+        self.config = self.get_default_config().merge(config)
+        validate_environment(
+            self.required_env_vars,
+            self.required_packages,
+            f"TTS Engine {self.__class__.__name__}",
+        )
+
+    async def close_connection(self) -> None:
+        """Cleanup the connection if necessary."""
+        return
+
+    async def synthesize(
+        self, text: str, config: Optional[T] = None
+    ) -> AsyncIterator[RasaAudioBytes]:
+        """Generate speech from text using a remote TTS system."""
+        yield RasaAudioBytes(b"")
+
+    def engine_bytes_to_rasa_audio_bytes(self, chunk: bytes) -> RasaAudioBytes:
+        """Convert the generated tts audio bytes into rasa audio bytes."""
+        raise NotImplementedError
+
+    @staticmethod
+    def get_default_config() -> T:
+        """Get the default config for this component."""
+        raise NotImplementedError
+
+    @classmethod
+    def from_config_dict(cls: Type[E], config: Dict) -> E:
+        raise NotImplementedError

rasa/core/channels/voice_stream/twilio_media_streams.py

@@ -0,0 +1,173 @@
+import base64
+import json
+import uuid
+
+import structlog
+from typing import Any, Awaitable, Callable, Dict, Optional, Text, Tuple
+
+from sanic import Blueprint, HTTPResponse, Request, response
+from sanic import Websocket  # type: ignore
+
+
+from rasa.core.channels import UserMessage
+from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.call_state import call_state
+from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.voice_channel import (
+    EndConversationAction,
+    NewAudioAction,
+    VoiceChannelAction,
+    ContinueConversationAction,
+    VoiceInputChannel,
+    VoiceOutputChannel,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+def map_call_params(data: Dict[Text, Any]) -> CallParameters:
+    """Map the twilio stream parameters to the CallParameters dataclass."""
+    stream_sid = data["streamSid"]
+    parameters = data["start"]["customParameters"]
+    return CallParameters(
+        call_id=parameters.get("call_id", ""),
+        user_phone=parameters.get("user_phone", ""),
+        bot_phone=parameters.get("bot_phone", ""),
+        direction=parameters.get("direction"),
+        stream_id=stream_sid,
+    )
+
+
+class TwilioMediaStreamsOutputChannel(VoiceOutputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "twilio_media_streams"
+
+    def rasa_audio_bytes_to_channel_bytes(
+        self, rasa_audio_bytes: RasaAudioBytes
+    ) -> bytes:
+        return base64.b64encode(rasa_audio_bytes)
+
+    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
+        message_id = uuid.uuid4().hex
+        mark_message = json.dumps(
+            {
+                "event": "mark",
+                "streamSid": recipient_id,
+                "mark": {"name": message_id},
+            }
+        )
+        return mark_message, message_id
+
+    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
+        media_message = json.dumps(
+            {
+                "event": "media",
+                "streamSid": recipient_id,
+                "media": {
+                    "payload": channel_bytes.decode("utf-8"),
+                },
+            }
+        )
+        return media_message
+
+
+class TwilioMediaStreamsInputChannel(VoiceInputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "twilio_media_streams"
+
+    def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
+        return RasaAudioBytes(base64.b64decode(input_bytes))
+
+    async def collect_call_parameters(
+        self, channel_websocket: Websocket
+    ) -> Optional[CallParameters]:
+        async for message in channel_websocket:
+            data = json.loads(message)
+            if data["event"] == "start":
+                # retrieve parameters set in the webhook - contains info about the
+                # caller
+                return map_call_params(data)
+        return None
+
+    def map_input_message(
+        self,
+        message: Any,
+    ) -> VoiceChannelAction:
+        data = json.loads(message)
+        if data["event"] == "media":
+            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(
+                data["media"]["payload"]
+            )
+            return NewAudioAction(audio_bytes)
+        elif data["event"] == "stop":
+            return EndConversationAction()
+        elif data["event"] == "mark":
+            if data["mark"]["name"] == call_state.latest_bot_audio_id:
+                # Just finished streaming last audio bytes
+                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
+                if call_state.should_hangup:
+                    logger.debug(
+                        "twilio_streams.hangup", marker=call_state.latest_bot_audio_id
+                    )
+                    return EndConversationAction()
+            else:
+                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+        return ContinueConversationAction()
+
+    def create_output_channel(
+        self, voice_websocket: Websocket, tts_engine: TTSEngine
+    ) -> VoiceOutputChannel:
+        return TwilioMediaStreamsOutputChannel(
+            voice_websocket,
+            tts_engine,
+            self.tts_cache,
+        )
+
+    def websocket_stream_url(self) -> str:
+        """Returns the websocket stream URL."""
+        # depending on the config value, the url might contain http as a
+        # protocol or not - we'll make sure both work
+        if self.server_url.startswith("http"):
+            base_url = self.server_url.replace("http", "ws")
+        else:
+            base_url = f"wss://{self.server_url}"
+        return f"{base_url}/webhooks/twilio_media_streams/websocket"
+
+    def blueprint(
+        self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
+    ) -> Blueprint:
+        """Defines a Sanic bluelogger.debug."""
+        blueprint = Blueprint("twilio_media_streams", __name__)
+
+        @blueprint.route("/", methods=["GET"])
+        async def health(_: Request) -> HTTPResponse:
+            return response.json({"status": "ok"})
+
+        @blueprint.route("/webhook", methods=["POST"])
+        async def receive(request: Request) -> HTTPResponse:
+            from twilio.twiml.voice_response import Connect, VoiceResponse
+
+            voice_response = VoiceResponse()
+            start = Connect()
+            stream = start.stream(url=self.websocket_stream_url())
+            # pass information about the call to the webhook - so we can
+            # store it in the input channel
+            stream.parameter(name="call_id", value=request.form.get("CallSid", None))
+            stream.parameter(name="user_phone", value=request.form.get("From", None))
+            stream.parameter(name="bot_phone", value=request.form.get("To", None))
+            stream.parameter(
+                name="direction", value=request.form.get("Direction", None)
+            )
+
+            voice_response.append(start)
+
+            return response.text(str(voice_response), content_type="text/xml")
+
+        @blueprint.websocket("/websocket")  # type: ignore
+        async def handle_message(request: Request, ws: Websocket) -> None:
+            await self.run_audio_streaming(on_new_message, ws)
+
+        return blueprint

rasa/core/channels/voice_stream/util.py

@@ -0,0 +1,57 @@
+import wave
+import audioop
+from dataclasses import asdict, dataclass
+from typing import Optional, Type, TypeVar
+
+import structlog
+
+from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.shared.exceptions import RasaException
+
+structlogger = structlog.get_logger()
+
+
+def read_wav_to_rasa_audio_bytes(file_name: str) -> Optional[RasaAudioBytes]:
+    """Reads rasa audio bytes from a file."""
+    if not file_name.endswith(".wav"):
+        raise RasaException("Should only read .wav files with this method.")
+    wave_object = wave.open(file_name, "rb")
+    wave_data = wave_object.readframes(wave_object.getnframes())
+    if wave_object.getnchannels() != 1:
+        wave_data = audioop.tomono(wave_data, wave_object.getsampwidth(), 1, 1)
+    if wave_object.getsampwidth() != 1:
+        wave_data = audioop.lin2lin(wave_data, wave_object.getsampwidth(), 1)
+        # 8 bit is unsigned
+        # wave_data = audioop.bias(wave_data, 1, 128)
+    if wave_object.getframerate() != HERTZ:
+        wave_data, _ = audioop.ratecv(
+            wave_data, 1, 1, wave_object.getframerate(), HERTZ, None
+        )
+    wave_data = audioop.lin2ulaw(wave_data, 1)
+    return RasaAudioBytes(wave_data)
+
+
+def generate_silence(length_in_seconds: float = 1.0) -> RasaAudioBytes:
+    return RasaAudioBytes(b"\00" * int(length_in_seconds * HERTZ))
+
+
+T = TypeVar("T", bound="MergeableConfig")
+
+
+@dataclass
+class MergeableConfig:
+    def __init__(self) -> None:
+        pass
+
+    def merge(self: T, other: Optional[T]) -> T:
+        """Merges two configs while dropping None values of the second config."""
+        if other is None:
+            return self
+        other_dict = asdict(other)
+        other_dict_clean = {k: v for k, v in other_dict.items() if v is not None}
+        merged = {**asdict(self), **other_dict_clean}
+        return self.from_dict(merged)
+
+    @classmethod
+    def from_dict(cls: Type[T], data: dict[str, Optional[str]]) -> T:
+        return cls(**data)