rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/core/channels/voice_ready/twilio_voice.py

@@ -0,0 +1,403 @@
+from sanic import Blueprint, response
+from sanic.request import Request, RequestParameters
+from sanic.response import HTTPResponse
+from twilio.twiml.voice_response import VoiceResponse, Gather
+from typing import Text, Callable, Awaitable, List, Any, Dict, Optional
+from dataclasses import asdict
+
+import structlog
+import rasa.utils.io
+import rasa.shared.utils.io
+from rasa.shared.core.events import BotUttered
+from rasa.shared.exceptions import InvalidConfigException
+from rasa.core.channels.channel import (
+    InputChannel,
+    CollectingOutputChannel,
+    UserMessage,
+)
+from rasa.core.channels.voice_ready.utils import CallParameters
+
+logger = structlog.get_logger(__name__)
+
+
+def map_call_params(form: RequestParameters) -> CallParameters:
+    """Map the Audiocodes parameters to the CallParameters dataclass."""
+    return CallParameters(
+        call_id=form.get("CallSid"),
+        user_phone=form.get("Caller"),
+        bot_phone=form.get("Called"),
+        direction=form.get("Direction"),
+    )
+
+
+class TwilioVoiceInput(InputChannel):
+    """Input channel for Twilio Voice."""
+
+    SUPPORTED_VOICES = [  # noqa: RUF012
+        "man",
+        "woman",
+        "alice",
+        "Polly.Mads",
+        "Polly.Naja",
+        "Polly.Lotte",
+        "Polly.Reuben",
+        "Polly.Nicole",
+        "Polly.Russell",
+        "Polly.Amy",
+        "Polly.Brian",
+        "Polly.Emma",
+        "Polly.Amy-Neural",
+        "Polly.Emma-Neural",
+        "Polly.Brian-Neural",
+        "Polly.Raveena",
+        "Polly.Ivy",
+        "Polly.Joanna",
+        "Polly.Joey",
+        "Polly.Justin",
+        "Polly.Kendra",
+        "Polly.Kimberly",
+        "Polly.Matthew",
+        "Polly.Salli",
+        "Polly.Ivy-Neural",
+        "Polly.Joanna-Neural",
+        "Polly.Kendra-Neural",
+        "Polly.Kimberly-Neural",
+        "Polly.Sally-Neural",
+        "Polly.Joey-Neural",
+        "Polly.Justin-Neural",
+        "Polly.Matthew-Neural",
+        "Polly.Geraint",
+        "Polly.Celine",
+        "Polly.Mathieu",
+        "Polly.Chantal",
+        "Polly.Hans",
+        "Polly.Marlene",
+        "Polly.Vicki",
+        "Polly.Dora",
+        "Polly.Karl",
+        "Polly.Carla",
+        "Polly.Giorgio",
+        "Polly.Mizuki",
+        "Polly.Takumi",
+        "Polly.Liv",
+        "Polly.Jacek",
+        "Polly.Jan",
+        "Polly.Ewa",
+        "Polly.Maja",
+        "Polly.Ricardo",
+        "Polly.Vitoria",
+        "Polly.Camila-Neural",
+        "Polly.Cristiano",
+        "Polly.Ines",
+        "Polly.Carmen",
+        "Polly.Maxim",
+        "Polly.Tatyana",
+        "Polly.Conchita",
+        "Polly.Enrique",
+        "Polly.Miguel",
+        "Polly.Penelope",
+        "Polly.Lupe-Neural",
+        "Polly.Astrid",
+        "Polly.Filiz",
+        "Polly.Gwyneth",
+        "Polly.Aditi",
+    ]
+
+    SUPPORTED_SPEECH_MODELS = [  # noqa: RUF012
+        "default",
+        "numbers_and_commands",
+        "phone_call",
+    ]
+
+    @classmethod
+    def name(cls) -> Text:
+        """Name of channel."""
+        return "twilio_voice"
+
+    @classmethod
+    def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> InputChannel:
+        """Load custom configurations."""
+        credentials = credentials or {}
+
+        return cls(
+            credentials.get(
+                "reprompt_fallback_phrase",
+                "I'm sorry I didn't get that could you rephrase.",
+            ),
+            credentials.get("assistant_voice", "woman"),
+            credentials.get("speech_timeout", "5"),
+            credentials.get("speech_model", "default"),
+            credentials.get("enhanced", "false"),
+        )
+
+    def __init__(
+        self,
+        reprompt_fallback_phrase: Optional[Text],
+        assistant_voice: Optional[Text],
+        speech_timeout: Text = "5",
+        speech_model: Text = "default",
+        enhanced: Text = "false",
+    ) -> None:
+        """Creates a connection to Twilio voice.
+
+        Args:
+            reprompt_fallback_phrase: phrase to use if no user response.
+            assistant_voice: name of the assistant voice to use.
+            speech_timeout: how long to pause when user finished speaking.
+            speech_model: type of transcription model to use from Twilio.
+            enhanced: toggle to use Twilio's premium speech transcription model.
+        """
+        self.reprompt_fallback_phrase = reprompt_fallback_phrase
+        self.assistant_voice = assistant_voice
+        self.speech_timeout = speech_timeout
+        self.speech_model = speech_model
+        self.enhanced = enhanced
+
+        self._validate_configuration()
+
+    def _validate_configuration(self) -> None:
+        """Checks that the user configurations are valid."""
+        if self.assistant_voice not in self.SUPPORTED_VOICES:
+            self._raise_invalid_voice_exception()
+
+        try:
+            int(self.speech_timeout)
+        except ValueError:
+            if self.speech_timeout.lower() != "auto":
+                self._raise_invalid_speech_timeout_exception()
+
+        if self.speech_model not in self.SUPPORTED_SPEECH_MODELS:
+            self._raise_invalid_speech_model_exception()
+
+        if self.enhanced.lower() not in [
+            "true",
+            "false",
+        ]:
+            self._raise_invalid_enhanced_option_exception()
+
+        if (
+            self.enhanced.lower() == "true"
+            and self.speech_model.lower() != "phone_call"
+        ):
+            self._raise_invalid_enhanced_speech_model_exception()
+
+        if (
+            self.speech_model.lower() != "numbers_and_commands"
+            and self.speech_timeout.lower() == "auto"
+        ):
+            self._raise_invalid_speech_model_timeout_exception()
+
+    def _raise_invalid_speech_model_timeout_exception(self) -> None:
+        """Raises an error if incompatible speech_timeout and speech_model used."""
+        raise InvalidConfigException(
+            "If speech_timeout is 'auto' the speech_model must be "
+            "'numbers_and_commands'. Please update your speech_model "
+            "to be 'numbers_and_commands' if you would like to continue "
+            "using the 'auto' speech_model."
+        )
+
+    def _raise_invalid_enhanced_option_exception(self) -> None:
+        """Raises an error if an invalid value is passed to the enhanced parameter."""
+        raise InvalidConfigException(
+            f"The value {self.enhanced} is invalid for the enhanced parameter. "
+            f"You must provide either `true` or `false` for this value."
+        )
+
+    def _raise_invalid_speech_model_exception(self) -> None:
+        """Raises an error if an invalid speech_model is provided."""
+        raise InvalidConfigException(
+            f"The value {self.speech_model} for speech_model is invalid. "
+            f"You must choose one of 'default', 'numbers_and_commands', "
+            f"or 'phone_call'. Refer to the documentation for details "
+            f"about the selections."
+        )
+
+    def _raise_invalid_speech_timeout_exception(self) -> None:
+        """Raises an error if an invalid speech_timeout is provided."""
+        raise InvalidConfigException(
+            f"The vale {self.speech_timeout} is an invalid value for speech_timeout. "
+            f"Only integers and 'auto' are valid entries."
+        )
+
+    def _raise_invalid_voice_exception(self) -> None:
+        """Raises an error if an invalid voice is provided."""
+        raise InvalidConfigException(
+            f"The value {self.assistant_voice} is an invalid for assistant_voice. "
+            f"Please refer to the documentation for a list of valid voices "
+            f"you can use for your voice assistant."
+        )
+
+    def _raise_invalid_enhanced_speech_model_exception(self) -> None:
+        """Raises error if enhanced is used with an incompatible speech_model."""
+        raise InvalidConfigException(
+            f"If you set enhanced to 'true' then speech_model must be 'phone_call'. "
+            f"Current speech_model is: {self.speech_model}."
+        )
+
+    def blueprint(
+        self, on_new_message: Callable[[UserMessage], Awaitable[None]]
+    ) -> Blueprint:
+        """Defines endpoints for Twilio voice channel."""
+        twilio_voice_webhook = Blueprint("Twilio_voice_webhook", __name__)
+
+        @twilio_voice_webhook.route("/", methods=["GET"])
+        async def health(request: Request) -> HTTPResponse:
+            return response.json({"status": "ok"})
+
+        @twilio_voice_webhook.route("/webhook", methods=["POST"])
+        async def receive(request: Request) -> HTTPResponse:
+            sender_id = request.form.get("From")
+            text = request.form.get("SpeechResult")
+            input_channel = self.name()
+            call_status = request.form.get("CallStatus")
+            metadata = {}
+
+            collector = TwilioVoiceCollectingOutputChannel()
+
+            logger.debug(
+                "twilio_voice.webhook",
+                sender_id=sender_id,
+                text=text,
+                call_status=call_status,
+            )
+            # Provide an initial greeting to answer the user's call.
+            if (text is None) and (call_status == "ringing"):
+                text = "/session_start"
+                metadata = asdict(map_call_params(request.form))
+
+            # when call is disconnected
+            if call_status == "completed":
+                text = "/session_end"
+                metadata = {"reason": "user disconnected"}
+
+            # determine the response.
+            if text is not None:
+                logger.info("twilio_voice.webhook.text_not_none", sender_id=sender_id)
+                await on_new_message(
+                    UserMessage(
+                        text,
+                        collector,
+                        sender_id,
+                        input_channel=input_channel,
+                        metadata=metadata,
+                    )
+                )
+
+                twilio_response = self._build_twilio_voice_response(collector.messages)
+            # If the user doesn't respond resend the last message.
+            else:
+                logger.info("twilio_voice.webhook.text_none", sender_id=sender_id)
+                # Get last user utterance from tracker.
+                tracker = await request.app.ctx.agent.tracker_store.retrieve(sender_id)
+                last_response = None
+                if tracker:
+                    last_response = next(
+                        (
+                            e
+                            for e in reversed(tracker.events)
+                            if isinstance(e, BotUttered)
+                        ),
+                        None,
+                    )
+
+                # If no previous utterance found use the reprompt_fallback phrase.
+                if last_response is None:
+                    last_response_text = self.reprompt_fallback_phrase
+                else:
+                    last_response_text = last_response.text
+
+                twilio_response = self._build_twilio_voice_response(
+                    [{"text": last_response_text}]
+                )
+            return response.text(str(twilio_response), content_type="text/xml")
+
+        return twilio_voice_webhook
+
+    def _build_twilio_voice_response(
+        self, messages: List[Dict[Text, Any]]
+    ) -> VoiceResponse:
+        """Builds the Twilio Voice Response object."""
+        logger.debug("twilio_voice.build_twilio_voice_response", messages=messages)
+        voice_response = VoiceResponse()
+        gather = Gather(
+            input="speech",
+            action=f"/webhooks/{self.name()}/webhook",
+            actionOnEmptyResult=True,
+            speechTimeout=self.speech_timeout,
+            speechModel=self.speech_model,
+            enhanced=self.enhanced,
+        )
+
+        # Add pauses between messages.
+        # Add a listener to the last message to listen for user response.
+        for i, message in enumerate(messages):
+            msg_text = message["text"]
+            # Check if the message is a hangup message.
+            if message.get("custom", {}).get("hangup"):
+                voice_response.hangup()
+                break
+
+            if i + 1 == len(messages):
+                gather.say(msg_text, voice=self.assistant_voice)
+                voice_response.append(gather)
+            else:
+                voice_response.say(msg_text, voice=self.assistant_voice)
+                voice_response.pause(length=1)
+
+        return voice_response
+
+
+class TwilioVoiceCollectingOutputChannel(CollectingOutputChannel):
+    """Output channel that collects send messages in a list.
+
+    (doesn't send them anywhere, just collects them).
+    """
+
+    @classmethod
+    def name(cls) -> Text:
+        """Name of the output channel."""
+        return "twilio_voice"
+
+    async def send_text_message(
+        self, recipient_id: Text, text: Text, **kwargs: Any
+    ) -> None:
+        """Sends the text message after removing emojis."""
+        text = rasa.utils.io.remove_emojis(text)
+        for message_part in text.strip().split("\n\n"):
+            await self._persist_message(self._message(recipient_id, text=message_part))
+
+    async def send_text_with_buttons(
+        self,
+        recipient_id: str,
+        text: str,
+        buttons: List[Dict[str, Any]],
+        **kwargs: Any,
+    ) -> None:
+        """Uses the concise button output format for voice channels."""
+        await self.send_text_with_buttons_concise(recipient_id, text, buttons, **kwargs)
+
+    async def send_image_url(
+        self, recipient_id: Text, image: Text, **kwargs: Any
+    ) -> None:
+        """For voice channel do not send images."""
+        rasa.shared.utils.io.raise_warning(
+            "An image was removed from the voice message and "
+            "only the text of message was sent. "
+            "It's recommended that you define voice-friendly "
+            "alternatives for all responses "
+            "with a visual elements such as images and emojis "
+            "that are used in your voice channel."
+        )
+
+    async def hangup(self, recipient_id: Text, **kwargs: Any) -> None:
+        """
+        Indicate that the conversation should be ended.
+
+        Parent class is a collecting output channel, so we don't actually hang up
+        but we add a custom message to the list of messages to be sent.
+        This message will be picked up by _build_twilio_voice_response
+        which will hang up the call.
+        """
+        await self._persist_message(
+            self._message(recipient_id, custom={"hangup": True})
+        )

rasa/core/channels/voice_ready/utils.py

@@ -0,0 +1,37 @@
+import structlog
+from dataclasses import dataclass
+from typing import Optional
+
+
+structlogger = structlog.get_logger()
+
+
+def validate_voice_license_scope() -> None:
+    from rasa.utils.licensing import (
+        PRODUCT_AREA,
+        VOICE_SCOPE,
+        validate_license_from_env,
+    )
+
+    """Validate that the correct license scope is present."""
+    structlogger.info(
+        f"Validating current Rasa Pro license scope which must include "
+        f"the '{VOICE_SCOPE}' scope to use the voice channel."
+    )
+
+    voice_product_scope = PRODUCT_AREA + " " + VOICE_SCOPE
+    validate_license_from_env(product_area=voice_product_scope)
+
+
+@dataclass
+class CallParameters:
+    """Standardized call parameters for voice channels."""
+
+    call_id: str
+    user_phone: str
+    bot_phone: str
+    user_name: Optional[str] = None
+    user_host: Optional[str] = None
+    bot_host: Optional[str] = None
+    direction: Optional[str] = None
+    stream_id: Optional[str] = None

rasa/core/channels/voice_stream/asr/asr_engine.py

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+from typing import (
+    Dict,
+    AsyncIterator,
+    Any,
+    Generic,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+)
+
+from websockets.legacy.client import WebSocketClientProtocol
+
+from rasa.core.channels.voice_stream.asr.asr_event import ASREvent
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.util import MergeableConfig
+from rasa.shared.exceptions import ConnectionException
+from rasa.shared.utils.common import validate_environment
+
+T = TypeVar("T", bound="ASREngineConfig")
+E = TypeVar("E", bound="ASREngine")
+
+
+@dataclass
+class ASREngineConfig(MergeableConfig):
+    pass
+
+
+class ASREngine(Generic[T]):
+    required_env_vars: Tuple[str, ...] = ()
+    required_packages: Tuple[str, ...] = ()
+
+    def __init__(self, config: Optional[T] = None):
+        self.config = self.get_default_config().merge(config)
+        self.asr_socket: Optional[WebSocketClientProtocol] = None
+        validate_environment(
+            self.required_env_vars,
+            self.required_packages,
+            f"ASR Engine {self.__class__.__name__}",
+        )
+
+    async def connect(self) -> None:
+        self.asr_socket = await self.open_websocket_connection()
+
+    async def open_websocket_connection(self) -> WebSocketClientProtocol:
+        """Connect to the ASR system."""
+        raise NotImplementedError
+
+    @classmethod
+    def from_config_dict(cls: Type[E], config: Dict) -> E:
+        raise NotImplementedError
+
+    async def close_connection(self) -> None:
+        if self.asr_socket:
+            await self.asr_socket.close()
+
+    async def signal_audio_done(self) -> None:
+        """Signal to the ASR Api that you are done sending data."""
+        raise NotImplementedError
+
+    async def send_audio_chunks(self, chunk: RasaAudioBytes) -> None:
+        """Send audio chunks to the ASR system via the websocket."""
+        if self.asr_socket is None:
+            raise ConnectionException("Websocket not connected.")
+        engine_bytes = self.rasa_audio_bytes_to_engine_bytes(chunk)
+        await self.asr_socket.send(engine_bytes)
+
+    def rasa_audio_bytes_to_engine_bytes(self, chunk: RasaAudioBytes) -> bytes:
+        """Convert RasaAudioBytes to bytes usable by this engine."""
+        raise NotImplementedError
+
+    async def stream_asr_events(self) -> AsyncIterator[ASREvent]:
+        """Stream the events returned by the ASR system as it is fed audio bytes."""
+        if self.asr_socket is None:
+            raise ConnectionException("Websocket not connected.")
+        async for message in self.asr_socket:
+            asr_event = self.engine_event_to_asr_event(message)
+            if asr_event:
+                yield asr_event
+
+    def engine_event_to_asr_event(self, e: Any) -> Optional[ASREvent]:
+        """Translate an engine event to a common ASREvent."""
+        raise NotImplementedError
+
+    @staticmethod
+    def get_default_config() -> T:
+        """Get the default config for this component."""
+        raise NotImplementedError

rasa/core/channels/voice_stream/asr/asr_event.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class ASREvent:
+    @classmethod
+    def name(cls) -> str:
+        return cls.__name__
+
+
+@dataclass
+class NewTranscript(ASREvent):
+    text: str
+
+
+@dataclass
+class UserIsSpeaking(ASREvent):
+    pass

rasa/core/channels/voice_stream/asr/azure.py

@@ -0,0 +1,130 @@
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, AsyncIterator
+import asyncio
+
+from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine, ASREngineConfig
+from rasa.core.channels.voice_stream.asr.asr_event import (
+    ASREvent,
+    NewTranscript,
+    UserIsSpeaking,
+)
+from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.shared.constants import AZURE_SPEECH_API_KEY_ENV_VAR
+from rasa.shared.exceptions import ConnectionException
+
+
+@dataclass
+class AzureASRConfig(ASREngineConfig):
+    language: Optional[str] = None
+    speech_region: Optional[str] = None
+
+
+class AzureASR(ASREngine[AzureASRConfig]):
+    required_env_vars = (AZURE_SPEECH_API_KEY_ENV_VAR,)
+    required_packages = ("azure.cognitiveservices.speech",)
+
+    def __init__(self, config: Optional[AzureASRConfig] = None):
+        super().__init__(config)
+
+        import azure.cognitiveservices.speech as speechsdk
+
+        self.speech_recognizer: Optional[speechsdk.SpeechRecognizer] = None
+        self.stream: Optional[speechsdk.audio.PushAudioInputStream] = None
+        self.is_recognizing = False
+        self.queue: asyncio.Queue[speechsdk.SpeechRecognitionEventArgs] = (
+            asyncio.Queue()
+        )
+        self.main_loop = asyncio.get_running_loop()
+
+    def signal_user_is_speaking(self, event: Any) -> None:
+        """Replace the azure event with a generic is speaking event."""
+        self.fill_queue(UserIsSpeaking())
+
+    def fill_queue(self, event: Any) -> None:
+        """Either puts the event or a dedicated ASR Event into the queue."""
+        # This function is used by call backs of the azure speech library
+        # which seems to run separate threads/processes
+        # To properly wake up the task waiting at queue.get, we need to
+        # put to the queue in the same event loop
+        self.main_loop.call_soon_threadsafe(self.queue.put_nowait, event)
+
+    async def connect(self) -> None:
+        import azure.cognitiveservices.speech as speechsdk
+
+        speech_config = speechsdk.SpeechConfig(
+            subscription=os.environ[AZURE_SPEECH_API_KEY_ENV_VAR],
+            region=self.config.speech_region,
+        )
+        audio_format = speechsdk.audio.AudioStreamFormat(
+            samples_per_second=HERTZ,
+            bits_per_sample=8,
+            channels=1,
+            wave_stream_format=speechsdk.AudioStreamWaveFormat.MULAW,
+        )
+        self.stream = speechsdk.audio.PushAudioInputStream(stream_format=audio_format)
+        audio_config = speechsdk.audio.AudioConfig(stream=self.stream)
+        self.speech_recognizer = speechsdk.SpeechRecognizer(
+            speech_config=speech_config,
+            language=self.config.language,
+            audio_config=audio_config,
+        )
+        self.speech_recognizer.recognized.connect(self.fill_queue)
+        self.speech_recognizer.recognizing.connect(self.signal_user_is_speaking)
+        self.speech_recognizer.start_continuous_recognition_async()
+        self.is_recognizing = True
+
+    async def close_connection(self) -> None:
+        if self.speech_recognizer is None:
+            raise ConnectionException("Websocket not connected.")
+        self.speech_recognizer.stop_continuous_recognition_async()
+
+    async def signal_audio_done(self) -> None:
+        """Signal to the ASR Api that you are done sending data."""
+        self.is_recognizing = False
+
+    def rasa_audio_bytes_to_engine_bytes(self, chunk: RasaAudioBytes) -> bytes:
+        """Convert RasaAudioBytes to bytes usable by this engine."""
+        return chunk
+
+    async def send_audio_chunks(self, chunk: RasaAudioBytes) -> None:
+        """Send audio chunks to the ASR system via the websocket."""
+        if self.speech_recognizer is None or self.stream is None:
+            raise ConnectionException("ASR not connected.")
+        engine_bytes = self.rasa_audio_bytes_to_engine_bytes(chunk)
+        self.stream.write(engine_bytes)
+
+    async def stream_asr_events(self) -> AsyncIterator[ASREvent]:
+        """Stream the events returned by the ASR system as it is fed audio bytes."""
+        if self.speech_recognizer is None:
+            raise ConnectionException("Websocket not connected.")
+        while self.is_recognizing or not self.queue.empty():
+            try:
+                message = await asyncio.wait_for(self.queue.get(), timeout=2)
+                asr_event = self.engine_event_to_asr_event(message)
+                if asr_event:
+                    yield asr_event
+            except asyncio.TimeoutError:
+                pass
+
+    def engine_event_to_asr_event(self, e: Any) -> Optional[ASREvent]:
+        """Translate an engine event to a common ASREvent."""
+        import azure.cognitiveservices.speech as speechsdk
+
+        if isinstance(e, speechsdk.SpeechRecognitionEventArgs) and isinstance(
+            e.result, speechsdk.SpeechRecognitionResult
+        ):
+            return NewTranscript(e.result.text)
+        if isinstance(e, ASREvent):
+            # transformation happened before
+            return e
+
+        return None
+
+    @staticmethod
+    def get_default_config() -> AzureASRConfig:
+        return AzureASRConfig("en-US", "germanywestcentral")
+
+    @classmethod
+    def from_config_dict(cls, config: Dict) -> "AzureASR":
+        return AzureASR(AzureASRConfig.from_dict(config))