rasa-pro 3.12.0.dev13__py3-none-any.whl → 3.12.0rc1__py3-none-any.whl

rasa/core/channels/telegram.py

@@ -1,21 +1,9 @@
 import json
 import logging
+import typing
 from copy import deepcopy
 from typing import Any, Awaitable, Callable, Dict, List, Optional, Text
 
-from aiogram import Bot
-from aiogram.exceptions import TelegramAPIError
-from aiogram.types import (
-    InlineKeyboardButton,
-    KeyboardButton,
-    Message,
-    Update,
-)
-from aiogram.utils.keyboard import (
-    InlineKeyboardBuilder,
-    KeyboardBuilder,
-    ReplyKeyboardBuilder,
-)
 from sanic import Blueprint, response
 from sanic.request import Request
 from sanic.response import HTTPResponse
@@ -27,8 +15,11 @@ from rasa.shared.exceptions import RasaException
 
 logger = logging.getLogger(__name__)
 
+if typing.TYPE_CHECKING:
+    from aiogram.types import Message, Update
 
-class TelegramOutput(Bot, OutputChannel):
+
+class TelegramOutput(OutputChannel):
     """Output channel for Telegram."""
 
     # skipcq: PYL-W0236
@@ -37,20 +28,28 @@ class TelegramOutput(Bot, OutputChannel):
         return "telegram"
 
     def __init__(self, access_token: Optional[Text]) -> None:
-        Bot.__init__(self, access_token)
+        try:
+            from aiogram import Bot
+
+            self.bot = Bot(access_token)
+        except ImportError:
+            raise ImportError(
+                "To use the Telegram channel, please install the aiogram package "
+                "with 'pip install aiogram'"
+            )
 
     async def send_text_message(
         self, recipient_id: Text, text: Text, **kwargs: Any
     ) -> None:
         """Sends text message."""
         for message_part in text.strip().split("\n\n"):
-            await self.send_message(recipient_id, message_part)
+            await self.bot.send_message(recipient_id, message_part)
 
     async def send_image_url(
         self, recipient_id: Text, image: Text, **kwargs: Any
     ) -> None:
         """Sends an image."""
-        await self.send_photo(recipient_id, image)
+        await self.bot.send_photo(recipient_id, image)
 
     async def send_text_with_buttons(
         self,
@@ -70,8 +69,15 @@ class TelegramOutput(Bot, OutputChannel):
 
         :button_type reply: reply keyboard
         """
+        from aiogram.types import InlineKeyboardButton, KeyboardButton
+        from aiogram.utils.keyboard import (
+            InlineKeyboardBuilder,
+            KeyboardBuilder,
+            ReplyKeyboardBuilder,
+        )
+
         if button_type == "inline":
-            reply_markup_builder: KeyboardBuilder = InlineKeyboardBuilder()
+            reply_markup_builder: "KeyboardBuilder" = InlineKeyboardBuilder()
             button_list = [
                 InlineKeyboardButton(text=s["title"], callback_data=s["payload"])
                 for s in buttons
@@ -110,7 +116,7 @@ class TelegramOutput(Bot, OutputChannel):
             )
             return
 
-        await self.send_message(recipient_id, text, reply_markup=reply_markup)
+        await self.bot.send_message(recipient_id, text, reply_markup=reply_markup)
 
     async def send_custom_json(
         self, recipient_id: Text, json_message: Dict[Text, Any], **kwargs: Any
@@ -150,9 +156,17 @@ class TelegramOutput(Bot, OutputChannel):
         for params in send_functions.keys():
             if all(json_message.get(p) is not None for p in params):
                 args = [json_message.pop(p) for p in params]
-                api_call = getattr(self, send_functions[params])
+                api_call = getattr(self.bot, send_functions[params])
                 await api_call(recipient_id, *args, **json_message)
 
+    async def get_me(self) -> Any:
+        """Get information about the bot itself."""
+        return await self.bot.get_me()
+
+    async def set_webhook(self, url: Text) -> None:
+        """Set the webhook URL for telegram."""
+        await self.bot.set_webhook(url=url)
+
 
 class TelegramInput(InputChannel):
     """Telegram input channel."""
@@ -185,19 +199,19 @@ class TelegramInput(InputChannel):
         self.debug_mode = debug_mode
 
     @staticmethod
-    def _is_location(message: Message) -> bool:
+    def _is_location(message: "Message") -> bool:
         return message.location is not None
 
     @staticmethod
-    def _is_user_message(message: Message) -> bool:
+    def _is_user_message(message: "Message") -> bool:
         return message.text is not None
 
     @staticmethod
-    def _is_edited_message(message: Update) -> bool:
+    def _is_edited_message(message: "Update") -> bool:
         return message.edited_message is not None
 
     @staticmethod
-    def _is_button(message: Update) -> bool:
+    def _is_button(message: "Update") -> bool:
         return message.callback_query is not None
 
     def blueprint(
@@ -223,6 +237,8 @@ class TelegramInput(InputChannel):
 
         @telegram_webhook.route("/webhook", methods=["GET", "POST"])
         async def message(request: Request) -> Any:
+            from aiogram.types import Update
+
             if request.method == "POST":
                 request_dict = request.json
                 if isinstance(request_dict, Text):
@@ -322,6 +338,8 @@ class TelegramInput(InputChannel):
         return TelegramOutput(self.access_token)
 
     async def set_webhook(self, channel: TelegramOutput) -> None:
+        from aiogram.exceptions import TelegramAPIError
+
         try:
             await channel.set_webhook(url=self.webhook_url)
         except TelegramAPIError as error:

rasa/core/channels/voice_ready/utils.py

@@ -29,7 +29,7 @@ class CallParameters:
 
     call_id: str
     user_phone: str
-    bot_phone: str
+    bot_phone: Optional[str] = None
     user_name: Optional[str] = None
     user_host: Optional[str] = None
     bot_host: Optional[str] = None

rasa/core/channels/voice_stream/asr/asr_engine.py

@@ -10,6 +10,7 @@ from typing import (
     TypeVar,
 )
 
+import structlog
 from websockets.legacy.client import WebSocketClientProtocol
 
 from rasa.core.channels.voice_stream.asr.asr_event import ASREvent
@@ -20,6 +21,7 @@ from rasa.shared.utils.common import validate_environment
 
 T = TypeVar("T", bound="ASREngineConfig")
 E = TypeVar("E", bound="ASREngine")
+logger = structlog.get_logger(__name__)
 
 
 @dataclass
@@ -74,10 +76,14 @@ class ASREngine(Generic[T]):
         """Stream the events returned by the ASR system as it is fed audio bytes."""
         if self.asr_socket is None:
             raise ConnectionException("Websocket not connected.")
-        async for message in self.asr_socket:
-            asr_event = self.engine_event_to_asr_event(message)
-            if asr_event:
-                yield asr_event
+
+        try:
+            async for message in self.asr_socket:
+                asr_event = self.engine_event_to_asr_event(message)
+                if asr_event:
+                    yield asr_event
+        except Exception as e:
+            logger.warning(f"Error while streaming ASR events: {e}")
 
     def engine_event_to_asr_event(self, e: Any) -> Optional[ASREvent]:
         """Translate an engine event to a common ASREvent."""

rasa/core/channels/voice_stream/asr/azure.py

@@ -18,6 +18,8 @@ from rasa.shared.exceptions import ConnectionException
 class AzureASRConfig(ASREngineConfig):
     language: Optional[str] = None
     speech_region: Optional[str] = None
+    speech_host: Optional[str] = None
+    speech_endpoint: Optional[str] = None
 
 
 class AzureASR(ASREngine[AzureASRConfig]):
@@ -52,9 +54,18 @@ class AzureASR(ASREngine[AzureASRConfig]):
     async def connect(self) -> None:
         import azure.cognitiveservices.speech as speechsdk
 
+        # connecting to eastus by default
+        if (
+            self.config.speech_region is None
+            and self.config.speech_host is None
+            and self.config.speech_endpoint is None
+        ):
+            self.config.speech_region = "eastus"
         speech_config = speechsdk.SpeechConfig(
             subscription=os.environ[AZURE_SPEECH_API_KEY_ENV_VAR],
             region=self.config.speech_region,
+            endpoint=self.config.speech_endpoint,
+            host=self.config.speech_host,
         )
         audio_format = speechsdk.audio.AudioStreamFormat(
             samples_per_second=HERTZ,
@@ -123,7 +134,9 @@ class AzureASR(ASREngine[AzureASRConfig]):
 
     @staticmethod
     def get_default_config() -> AzureASRConfig:
-        return AzureASRConfig("en-US", "eastus")
+        return AzureASRConfig(
+            language=None, speech_region=None, speech_host=None, speech_endpoint=None
+        )
 
     @classmethod
     def from_config_dict(cls, config: Dict) -> "AzureASR":

rasa/core/channels/voice_stream/asr/deepgram.py

@@ -4,7 +4,9 @@ from dataclasses import dataclass
 from typing import Any, Dict, Optional
 from urllib.parse import urlencode
 
+import structlog
 import websockets
+import websockets.exceptions
 from websockets.legacy.client import WebSocketClientProtocol
 
 from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine, ASREngineConfig
@@ -16,6 +18,8 @@ from rasa.core.channels.voice_stream.asr.asr_event import (
 from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
 from rasa.shared.constants import DEEPGRAM_API_KEY_ENV_VAR
 
+logger = structlog.get_logger(__name__)
+
 
 @dataclass
 class DeepgramASRConfig(ASREngineConfig):
@@ -41,10 +45,22 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
         """Connect to the ASR system."""
         deepgram_api_key = os.environ[DEEPGRAM_API_KEY_ENV_VAR]
         extra_headers = {"Authorization": f"Token {deepgram_api_key}"}
-        return await websockets.connect(  # type: ignore
-            self._get_api_url_with_query_params(),
-            extra_headers=extra_headers,
-        )
+        try:
+            return await websockets.connect(  # type: ignore
+                self._get_api_url_with_query_params(),
+                extra_headers=extra_headers,
+            )
+        except websockets.exceptions.InvalidStatusCode as e:
+            if e.status_code == 401:
+                error_msg = "Please make sure your Deepgram API key is correct."
+            else:
+                error_msg = "Connection to Deepgram failed."
+            logger.error(
+                "deepgram.connection.failed",
+                status_code=e.status_code,
+                error=error_msg,
+            )
+            raise
 
     def _get_api_url_with_query_params(self) -> str:
         """Combine api url and query params."""

rasa/core/channels/voice_stream/audiocodes.py

@@ -0,0 +1,264 @@
+import asyncio
+import base64
+import json
+from typing import Any, Awaitable, Callable, Dict, Optional, Text
+
+import structlog
+from sanic import (  # type: ignore[attr-defined]
+    Blueprint,
+    HTTPResponse,
+    Request,
+    Websocket,
+    response,
+)
+
+from rasa.core.channels import UserMessage
+from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.call_state import (
+    call_state,
+)
+from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
+from rasa.core.channels.voice_stream.voice_channel import (
+    ContinueConversationAction,
+    EndConversationAction,
+    NewAudioAction,
+    VoiceChannelAction,
+    VoiceInputChannel,
+    VoiceOutputChannel,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+def map_call_params(data: Dict[Text, Any]) -> CallParameters:
+    """Map the audiocodes stream parameters to the CallParameters dataclass."""
+    return CallParameters(
+        call_id=data["conversationId"],
+        user_phone=data["caller"],
+        # Bot phone is not available in the Audiocodes API
+        direction="inbound",  # AudioCodes calls are always inbound
+    )
+
+
+class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "ac_voice"
+
+    def rasa_audio_bytes_to_channel_bytes(
+        self, rasa_audio_bytes: RasaAudioBytes
+    ) -> bytes:
+        return base64.b64encode(rasa_audio_bytes)
+
+    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
+        media_message = json.dumps(
+            {
+                "type": "playStream.chunk",
+                "streamId": str(call_state.stream_id),
+                "audioChunk": channel_bytes.decode("utf-8"),
+            }
+        )
+        return media_message
+
+    async def send_start_marker(self, recipient_id: str) -> None:
+        """Send playStream.start before first audio chunk."""
+        call_state.stream_id += 1  # type: ignore[attr-defined]
+        media_message = json.dumps(
+            {
+                "type": "playStream.start",
+                "streamId": str(call_state.stream_id),
+            }
+        )
+        logger.debug("Sending start marker", stream_id=call_state.stream_id)
+        await self.voice_websocket.send(media_message)
+
+    async def send_intermediate_marker(self, recipient_id: str) -> None:
+        """Audiocodes doesn't need intermediate markers, so do nothing."""
+        pass
+
+    async def send_end_marker(self, recipient_id: str) -> None:
+        """Send playStream.stop after last audio chunk."""
+        media_message = json.dumps(
+            {
+                "type": "playStream.stop",
+                "streamId": str(call_state.stream_id),
+            }
+        )
+        logger.debug("Sending end marker", stream_id=call_state.stream_id)
+        await self.voice_websocket.send(media_message)
+
+
+class AudiocodesVoiceInputChannel(VoiceInputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "ac_voice"
+
+    def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
+        return RasaAudioBytes(base64.b64decode(input_bytes))
+
+    async def collect_call_parameters(
+        self, channel_websocket: Websocket
+    ) -> Optional[CallParameters]:
+        async for message in channel_websocket:
+            data = json.loads(message)
+            if data["type"] == "session.initiate":
+                # retrieve parameters set in the webhook - contains info about the
+                # caller
+                logger.info("received initiate message", data=data)
+                self._send_accepted(channel_websocket, data)
+                return map_call_params(data)
+            else:
+                logger.warning("ac_voice.unknown_message", data=data)
+        return None
+
+    def map_input_message(
+        self,
+        message: Any,
+        ws: Websocket,
+    ) -> VoiceChannelAction:
+        data = json.loads(message)
+        if data["type"] == "activities":
+            activities = data["activities"]
+            for activity in activities:
+                logger.debug("ac_voice.activity", data=activity)
+                if activity["name"] == "start":
+                    pass
+                elif activity["name"] == "dtmf":
+                    # TODO: handle DTMF input
+                    pass
+                elif activity["name"] == "playFinished":
+                    logger.debug("ac_voice.playFinished", data=activity)
+                    if call_state.should_hangup:
+                        logger.info("audiocodes.hangup")
+                        self._send_hangup(ws, data)
+                        # the conversation should continue until
+                        # we receive a end message from audiocodes
+                    pass
+                else:
+                    logger.warning("ac_voice.unknown_activity", data=activity)
+        elif data["type"] == "userStream.start":
+            logger.debug("ac_voice.userStream.start", data=data)
+            self._send_recognition_started(ws, data)
+        elif data["type"] == "userStream.chunk":
+            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(data["audioChunk"])
+            return NewAudioAction(audio_bytes)
+        elif data["type"] == "userStream.stop":
+            logger.debug("ac_voice.stop_recognition", data=data)
+            self._send_recognition_ended(ws, data)
+        elif data["type"] == "session.resume":
+            logger.debug("ac_voice.resume", data=data)
+            self._send_accepted(ws, data)
+        elif data["type"] == "session.end":
+            logger.debug("ac_voice.end", data=data)
+            return EndConversationAction()
+        elif data["type"] == "connection.validate":
+            # not part of call flow; only sent when integration is created
+            self._send_validated(ws, data)
+        else:
+            logger.warning("ac_voice.unknown_message", data=data)
+
+        return ContinueConversationAction()
+
+    def _send_accepted(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        supported_formats = data.get("supportedMediaFormats", [])
+        preferred_format = "raw/mulaw"
+
+        if preferred_format not in supported_formats:
+            logger.warning(
+                "ac_voice.format_not_supported",
+                supported_formats=supported_formats,
+                preferred_format=preferred_format,
+            )
+            raise
+
+        payload = {
+            "type": "session.accepted",
+            "mediaFormat": "raw/mulaw",
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_recognition_started(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {"type": "userStream.started"}
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_recognition_ended(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {"type": "userStream.stopped"}
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_hypothesis(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        """
+        TODO: The hypothesis message is sent by the bot to provide partial
+        recognition results. Using this message is recommended,
+        as VAIC relies on it for performing barge-in.
+        """
+        pass
+
+    def _send_recognition(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        """
+        TODO: The recognition message is sent by the bot to provide
+        the final recognition result. Using this message is recommended
+        mainly for logging purposes.
+        """
+        pass
+
+    def _send_hangup(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {
+            "conversationId": data["conversationId"],
+            "type": "activities",
+            "activities": [{"type": "event", "name": "hangup"}],
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_validated(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {
+            "type": "connection.validated",
+            "success": True,
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def create_output_channel(
+        self, voice_websocket: Websocket, tts_engine: TTSEngine
+    ) -> VoiceOutputChannel:
+        return AudiocodesVoiceOutputChannel(
+            voice_websocket,
+            tts_engine,
+            self.tts_cache,
+        )
+
+    def blueprint(
+        self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
+    ) -> Blueprint:
+        """Defines a Sanic bluelogger.debug."""
+        blueprint = Blueprint("ac_voice", __name__)
+
+        @blueprint.route("/", methods=["GET"])
+        async def health(_: Request) -> HTTPResponse:
+            return response.json({"status": "ok"})
+
+        @blueprint.websocket("/websocket")  # type: ignore
+        async def receive(request: Request, ws: Websocket) -> None:
+            # TODO: validate API key header
+            logger.info("audiocodes.receive", message="Starting audio streaming")
+            try:
+                await self.run_audio_streaming(on_new_message, ws)
+            except Exception as e:
+                logger.exception(
+                    "audiocodes.receive",
+                    message="Error during audio streaming",
+                    error=e,
+                )
+                # return 500 error
+                raise
+
+        return blueprint
+
+
+def _schedule_async_task(coro: Awaitable[Any]) -> None:
+    """Helper function to schedule a coroutine in the event loop.
+
+    Args:
+        coro: The coroutine to schedule
+    """
+    loop = asyncio.get_running_loop()
+    loop.call_soon_threadsafe(lambda: loop.create_task(coro))

rasa/core/channels/voice_stream/browser_audio.py

@@ -106,6 +106,9 @@ class BrowserAudioInputChannel(VoiceInputChannel):
 
         @blueprint.websocket("/websocket")  # type: ignore
         async def handle_message(request: Request, ws: Websocket) -> None:
-            await self.run_audio_streaming(on_new_message, ws)
+            try:
+                await self.run_audio_streaming(on_new_message, ws)
+            except Exception as e:
+                logger.error("browser_audio.handle_message.error", error=e)
 
         return blueprint

rasa/core/channels/voice_stream/call_state.py

@@ -25,6 +25,9 @@ class CallState:
     server_sequence_number: int = 0
     audio_buffer: bytearray = field(default_factory=bytearray)
 
+    # Audiocodes requires a stream ID at start and end of stream
+    stream_id: int = 0
+
 
 _call_state: ContextVar[CallState] = ContextVar("call_state")
 call_state = LocalProxy(_call_state)

rasa/core/channels/voice_stream/genesys.py

@@ -104,7 +104,10 @@ class GenesysOutputChannel(VoiceOutputChannel):
                 current_position = end_position
 
     async def send_marker_message(self, recipient_id: str) -> None:
-        """Send a message that marks positions in the audio stream."""
+        """
+        Send a message that marks positions in the audio stream.
+        Genesys does not support this feature, so we do nothing here.
+        """
         pass
 
 
@@ -190,6 +193,8 @@ class GenesysInputChannel(VoiceInputChannel):
                 if call_state.should_hangup:
                     logger.info("genesys.hangup")
                     self.disconnect(ws, data)
+                    # the conversation should continue until
+                    # we receive a close message from Genesys
             elif msg_type == "dtmf":
                 logger.info("genesys.handle_dtmf", message=data)
             elif msg_type == "error":
@@ -259,7 +264,6 @@ class GenesysInputChannel(VoiceInputChannel):
         logger.debug("genesys.handle_close.closed", response=response)
 
         _schedule_ws_task(ws.send(json.dumps(response)))
-        _schedule_ws_task(ws.close())
 
     def disconnect(self, ws: Websocket, data: dict) -> None:
         """

rasa/core/channels/voice_stream/tts/azure.py

@@ -21,6 +21,7 @@ structlogger = structlog.get_logger()
 @dataclass
 class AzureTTSConfig(TTSEngineConfig):
     speech_region: Optional[str] = None
+    endpoint: Optional[str] = None
 
 
 class AzureTTS(TTSEngine[AzureTTSConfig]):
@@ -76,7 +77,13 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
 
     @staticmethod
     def get_tts_endpoint(config: AzureTTSConfig) -> str:
-        return f"https://{config.speech_region}.tts.speech.microsoft.com/cognitiveservices/v1"
+        if config.endpoint is not None:
+            return config.endpoint
+        else:
+            return (
+                f"https://{config.speech_region}.tts.speech.microsoft.com/"
+                f"cognitiveservices/v1"
+            )
 
     @staticmethod
     def create_request_body(text: str, conf: AzureTTSConfig) -> str:
@@ -99,6 +106,7 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
             voice="en-US-JennyNeural",
             timeout=10,
             speech_region="eastus",
+            endpoint=None,
         )
 
     @classmethod