rasa-pro 3.11.0__py3-none-any.whl → 3.11.0a1__py3-none-any.whl

rasa/core/channels/voice_stream/twilio_media_streams.py

@@ -1,9 +1,8 @@
 import base64
 import json
-import uuid
-
 import structlog
-from typing import Any, Awaitable, Callable, Dict, Optional, Text, Tuple
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Text
+import uuid
 
 from sanic import Blueprint, HTTPResponse, Request, response
 from sanic import Websocket  # type: ignore
@@ -11,7 +10,6 @@ from sanic import Websocket  # type: ignore
 
 from rasa.core.channels import UserMessage
 from rasa.core.channels.voice_ready.utils import CallParameters
-from rasa.core.channels.voice_stream.call_state import call_state
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.voice_channel import (
@@ -23,7 +21,7 @@ from rasa.core.channels.voice_stream.voice_channel import (
     VoiceOutputChannel,
 )
 
-logger = structlog.get_logger(__name__)
+structlogger = structlog.get_logger()
 
 
 def map_call_params(data: Dict[Text, Any]) -> CallParameters:
@@ -49,18 +47,10 @@ class TwilioMediaStreamsOutputChannel(VoiceOutputChannel):
     ) -> bytes:
         return base64.b64encode(rasa_audio_bytes)
 
-    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
+    def channel_bytes_to_messages(
+        self, recipient_id: str, channel_bytes: bytes
+    ) -> List[Any]:
         message_id = uuid.uuid4().hex
-        mark_message = json.dumps(
-            {
-                "event": "mark",
-                "streamSid": recipient_id,
-                "mark": {"name": message_id},
-            }
-        )
-        return mark_message, message_id
-
-    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
         media_message = json.dumps(
             {
                 "event": "media",
@@ -70,7 +60,15 @@ class TwilioMediaStreamsOutputChannel(VoiceOutputChannel):
                 },
             }
         )
-        return media_message
+        mark_message = json.dumps(
+            {
+                "event": "mark",
+                "streamSid": recipient_id,
+                "mark": {"name": message_id},
+            }
+        )
+        self.latest_message_id = message_id
+        return [media_message, mark_message]
 
 
 class TwilioMediaStreamsInputChannel(VoiceInputChannel):
@@ -105,16 +103,9 @@ class TwilioMediaStreamsInputChannel(VoiceInputChannel):
         elif data["event"] == "stop":
             return EndConversationAction()
         elif data["event"] == "mark":
-            if data["mark"]["name"] == call_state.latest_bot_audio_id:
-                # Just finished streaming last audio bytes
-                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
-                if call_state.should_hangup:
-                    logger.debug(
-                        "twilio_streams.hangup", marker=call_state.latest_bot_audio_id
-                    )
-                    return EndConversationAction()
-            else:
-                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+            if data["mark"]["name"] == self.hangup_after:
+                structlogger.debug("twilio_streams.hangup", marker=self.hangup_after)
+                return EndConversationAction()
         return ContinueConversationAction()
 
     def create_output_channel(
@@ -140,7 +131,7 @@ class TwilioMediaStreamsInputChannel(VoiceInputChannel):
         self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
     ) -> Blueprint:
         """Defines a Sanic bluelogger.debug."""
-        blueprint = Blueprint("twilio_media_streams", __name__)
+        blueprint = Blueprint("socketio_webhook", __name__)
 
         @blueprint.route("/", methods=["GET"])
         async def health(_: Request) -> HTTPResponse:

rasa/core/channels/voice_stream/util.py

@@ -5,7 +5,7 @@ from typing import Optional, Type, TypeVar
 
 import structlog
 
-from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.shared.exceptions import RasaException
 
 structlogger = structlog.get_logger()
@@ -23,16 +23,16 @@ def read_wav_to_rasa_audio_bytes(file_name: str) -> Optional[RasaAudioBytes]:
         wave_data = audioop.lin2lin(wave_data, wave_object.getsampwidth(), 1)
         # 8 bit is unsigned
         # wave_data = audioop.bias(wave_data, 1, 128)
-    if wave_object.getframerate() != HERTZ:
+    if wave_object.getframerate() != 8000:
         wave_data, _ = audioop.ratecv(
-            wave_data, 1, 1, wave_object.getframerate(), HERTZ, None
+            wave_data, 1, 1, wave_object.getframerate(), 8000, None
         )
     wave_data = audioop.lin2ulaw(wave_data, 1)
     return RasaAudioBytes(wave_data)
 
 
 def generate_silence(length_in_seconds: float = 1.0) -> RasaAudioBytes:
-    return RasaAudioBytes(b"\00" * int(length_in_seconds * HERTZ))
+    return RasaAudioBytes(b"\00" * int(length_in_seconds * 8000))
 
 
 T = TypeVar("T", bound="MergeableConfig")

rasa/core/channels/voice_stream/voice_channel.py

@@ -1,45 +1,25 @@
 import asyncio
-import structlog
+import logging
 import copy
 from dataclasses import asdict, dataclass
-from typing import Any, AsyncIterator, Awaitable, Callable, Dict, List, Optional, Tuple
-
-from rasa.core.channels.voice_stream.util import generate_silence
-from rasa.shared.core.constants import SLOT_SILENCE_TIMEOUT
-from rasa.shared.utils.common import (
-    class_from_module_path,
-    mark_as_beta_feature,
-)
-from rasa.shared.utils.cli import print_error_and_exit
+from typing import Any, Awaitable, Callable, Dict, List, Optional
 
 from sanic.exceptions import ServerError, WebsocketClosed
 
 from rasa.core.channels import InputChannel, OutputChannel, UserMessage
 from rasa.core.channels.voice_ready.utils import CallParameters
-from rasa.core.channels.voice_ready.utils import validate_voice_license_scope
 from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine
-from rasa.core.channels.voice_stream.asr.asr_event import (
-    ASREvent,
-    NewTranscript,
-    UserIsSpeaking,
-)
+from rasa.core.channels.voice_stream.asr.asr_event import ASREvent, NewTranscript
 from sanic import Websocket  # type: ignore
 
 from rasa.core.channels.voice_stream.asr.deepgram import DeepgramASR
-from rasa.core.channels.voice_stream.asr.azure import AzureASR
-from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
-from rasa.core.channels.voice_stream.call_state import (
-    CallState,
-    _call_state,
-    call_state,
-)
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.tts.azure import AzureTTS
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine, TTSError
 from rasa.core.channels.voice_stream.tts.cartesia import CartesiaTTS
 from rasa.core.channels.voice_stream.tts.tts_cache import TTSCache
-from rasa.utils.io import remove_emojis
 
-logger = structlog.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 @dataclass
@@ -63,55 +43,25 @@ class ContinueConversationAction(VoiceChannelAction):
 
 
 def asr_engine_from_config(asr_config: Dict) -> ASREngine:
-    name = str(asr_config["name"])
+    name = str(asr_config["name"]).lower()
     asr_config = copy.copy(asr_config)
     asr_config.pop("name")
-    if name.lower() == "deepgram":
+    if name == "deepgram":
         return DeepgramASR.from_config_dict(asr_config)
-    if name == "azure":
-        return AzureASR.from_config_dict(asr_config)
     else:
-        mark_as_beta_feature("Custom ASR Engine")
-        try:
-            asr_engine_class = class_from_module_path(name)
-            return asr_engine_class.from_config_dict(asr_config)
-        except NameError:
-            print_error_and_exit(
-                f"Failed to initialize ASR Engine with type '{name}'. "
-                f"Please make sure the method `from_config_dict`is implemented."
-            )
-        except TypeError as e:
-            print_error_and_exit(
-                f"Failed to initialize ASR Engine with type '{name}'. "
-                f"Invalid configuration provided. "
-                f"Error: {e}"
-            )
+        raise NotImplementedError
 
 
 def tts_engine_from_config(tts_config: Dict) -> TTSEngine:
-    name = str(tts_config["name"])
+    name = str(tts_config["name"]).lower()
     tts_config = copy.copy(tts_config)
     tts_config.pop("name")
-    if name.lower() == "azure":
+    if name == "azure":
         return AzureTTS.from_config_dict(tts_config)
-    elif name.lower() == "cartesia":
+    elif name == "cartesia":
         return CartesiaTTS.from_config_dict(tts_config)
     else:
-        mark_as_beta_feature("Custom TTS Engine")
-        try:
-            tts_engine_class = class_from_module_path(name)
-            return tts_engine_class.from_config_dict(tts_config)
-        except NameError:
-            print_error_and_exit(
-                f"Failed to initialize TTS Engine with type '{name}'. "
-                f"Please make sure the method `from_config_dict`is implemented."
-            )
-        except TypeError as e:
-            print_error_and_exit(
-                f"Failed to initialize ASR Engine with type '{name}'. "
-                f"Invalid configuration provided. "
-                f"Error: {e}"
-            )
+        raise NotImplementedError(f"TTS engine {name} is not implemented")
 
 
 class VoiceOutputChannel(OutputChannel):
@@ -121,171 +71,75 @@ class VoiceOutputChannel(OutputChannel):
         tts_engine: TTSEngine,
         tts_cache: TTSCache,
     ):
-        super().__init__()
         self.voice_websocket = voice_websocket
         self.tts_engine = tts_engine
         self.tts_cache = tts_cache
 
+        self.should_hangup = False
         self.latest_message_id: Optional[str] = None
 
     def rasa_audio_bytes_to_channel_bytes(
         self, rasa_audio_bytes: RasaAudioBytes
     ) -> bytes:
-        """Turn rasa's audio byte format into the format for the channel."""
         raise NotImplementedError
 
-    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
-        """Wrap the bytes for the channel in the proper format."""
+    def channel_bytes_to_messages(
+        self, recipient_id: str, channel_bytes: bytes
+    ) -> List[Any]:
         raise NotImplementedError
 
-    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
-        """Create a marker message for a specific channel."""
-        raise NotImplementedError
-
-    async def send_marker_message(self, recipient_id: str) -> None:
-        """Send a message that marks positions in the audio stream."""
-        marker_message, mark_id = self.create_marker_message(recipient_id)
-        await self.voice_websocket.send(marker_message)
-        self.latest_message_id = mark_id
-
-    def update_silence_timeout(self) -> None:
-        """Updates the silence timeout for the session."""
-        if self.tracker_state:
-            call_state.silence_timeout = (  # type: ignore[attr-defined]
-                self.tracker_state["slots"][SLOT_SILENCE_TIMEOUT]
-            )
-
-    async def send_text_with_buttons(
-        self,
-        recipient_id: str,
-        text: str,
-        buttons: List[Dict[str, Any]],
-        **kwargs: Any,
-    ) -> None:
-        """Uses the concise button output format for voice channels."""
-        await self.send_text_with_buttons_concise(recipient_id, text, buttons, **kwargs)
-
     async def send_text_message(
         self, recipient_id: str, text: str, **kwargs: Any
     ) -> None:
-        text = remove_emojis(text)
-        self.update_silence_timeout()
         cached_audio_bytes = self.tts_cache.get(text)
-        collected_audio_bytes = RasaAudioBytes(b"")
-        seconds_marker = -1
-        if cached_audio_bytes:
-            audio_stream = self.chunk_audio(cached_audio_bytes)
-        else:
-            # Todo: make kwargs compatible with engine config
-            synth_config = self.tts_engine.config.__class__.from_dict({})
-            try:
-                audio_stream = self.tts_engine.synthesize(text, synth_config)
-            except TTSError:
-                # TODO: add message that works without tts, e.g. loading from disc
-                audio_stream = self.chunk_audio(generate_silence())
 
+        if cached_audio_bytes:
+            await self.send_audio_bytes(recipient_id, cached_audio_bytes)
+            return
+        collected_audio_bytes = RasaAudioBytes(b"")
+        # Todo: make kwargs compatible with engine config
+        synth_config = self.tts_engine.config.__class__.from_dict({})
+        try:
+            audio_stream = self.tts_engine.synthesize(text, synth_config)
+        except TTSError:
+            # TODO: add message that works without tts, e.g. loading from disc
+            pass
         async for audio_bytes in audio_stream:
             try:
                 await self.send_audio_bytes(recipient_id, audio_bytes)
-                full_seconds_of_audio = len(collected_audio_bytes) // HERTZ
-                if full_seconds_of_audio > seconds_marker:
-                    await self.send_marker_message(recipient_id)
-                    seconds_marker = full_seconds_of_audio
-
             except (WebsocketClosed, ServerError):
                 # ignore sending error, and keep collecting and caching audio bytes
-                call_state.connection_failed = True  # type: ignore[attr-defined]
+                self.should_hangup = True
+
             collected_audio_bytes = RasaAudioBytes(collected_audio_bytes + audio_bytes)
-        try:
-            await self.send_marker_message(recipient_id)
-        except (WebsocketClosed, ServerError):
-            # ignore sending error
-            pass
-        call_state.latest_bot_audio_id = self.latest_message_id  # type: ignore[attr-defined]
 
-        if not cached_audio_bytes:
-            self.tts_cache.put(text, collected_audio_bytes)
+        self.tts_cache.put(text, collected_audio_bytes)
 
     async def send_audio_bytes(
         self, recipient_id: str, audio_bytes: RasaAudioBytes
     ) -> None:
         channel_bytes = self.rasa_audio_bytes_to_channel_bytes(audio_bytes)
-        message = self.channel_bytes_to_message(recipient_id, channel_bytes)
-        await self.voice_websocket.send(message)
-
-    async def chunk_audio(
-        self, audio_bytes: RasaAudioBytes, chunk_size: int = 2048
-    ) -> AsyncIterator[RasaAudioBytes]:
-        """Generate chunks from cached audio bytes."""
-        offset = 0
-        while offset < len(audio_bytes):
-            chunk = audio_bytes[offset : offset + chunk_size]
-            if len(chunk):
-                yield RasaAudioBytes(chunk)
-            offset += chunk_size
-        return
+        for message in self.channel_bytes_to_messages(recipient_id, channel_bytes):
+            await self.voice_websocket.send(message)
 
     async def hangup(self, recipient_id: str, **kwargs: Any) -> None:
-        call_state.should_hangup = True  # type: ignore[attr-defined]
+        self.should_hangup = True
 
 
 class VoiceInputChannel(InputChannel):
-    def __init__(
-        self,
-        server_url: str,
-        asr_config: Dict,
-        tts_config: Dict,
-        monitor_silence: bool = False,
-    ):
-        validate_voice_license_scope()
+    def __init__(self, server_url: str, asr_config: Dict, tts_config: Dict):
         self.server_url = server_url
         self.asr_config = asr_config
         self.tts_config = tts_config
-        self.monitor_silence = monitor_silence
         self.tts_cache = TTSCache(tts_config.get("cache_size", 1000))
 
-    async def handle_silence_timeout(
-        self,
-        voice_websocket: Websocket,
-        on_new_message: Callable[[UserMessage], Awaitable[Any]],
-        tts_engine: TTSEngine,
-        call_parameters: CallParameters,
-    ) -> None:
-        timeout = call_state.silence_timeout
-        if not timeout:
-            return
-        if not self.monitor_silence:
-            return
-        logger.debug("voice_channel.silence_timeout_watch_started", timeout=timeout)
-        await asyncio.sleep(timeout)
-        logger.debug("voice_channel.silence_timeout_tripped")
-        output_channel = self.create_output_channel(voice_websocket, tts_engine)
-        message = UserMessage(
-            "/silence_timeout",
-            output_channel,
-            call_parameters.stream_id,
-            input_channel=self.name(),
-            metadata=asdict(call_parameters),
-        )
-        await on_new_message(message)
-
-    @staticmethod
-    def _cancel_silence_timeout_watcher() -> None:
-        """Cancels the silent timeout task if it exists."""
-        if call_state.silence_timeout_watcher:
-            logger.debug("voice_channel.cancelling_current_timeout_watcher_task")
-            call_state.silence_timeout_watcher.cancel()
-            call_state.silence_timeout_watcher = None  # type: ignore[attr-defined]
+        # if set to a value, call will be hungup after marker is reached
+        self.hangup_after: Optional[str] = None
 
     @classmethod
     def from_credentials(cls, credentials: Optional[Dict[str, Any]]) -> InputChannel:
         credentials = credentials or {}
-        return cls(
-            credentials["server_url"],
-            credentials["asr"],
-            credentials["tts"],
-            credentials.get("monitor_silence", False),
-        )
+        return cls(credentials["server_url"], credentials["asr"], credentials["tts"])
 
     def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
         raise NotImplementedError
@@ -325,7 +179,6 @@ class VoiceInputChannel(InputChannel):
         channel_websocket: Websocket,
     ) -> None:
         """Pipe input audio to ASR and consume ASR events simultaneously."""
-        _call_state.set(CallState())
         asr_engine = asr_engine_from_config(self.asr_config)
         tts_engine = tts_engine_from_config(self.tts_config)
         await asr_engine.connect()
@@ -339,29 +192,7 @@ class VoiceInputChannel(InputChannel):
 
         async def consume_audio_bytes() -> None:
             async for message in channel_websocket:
-                is_bot_speaking_before = call_state.is_bot_speaking
                 channel_action = self.map_input_message(message)
-                is_bot_speaking_after = call_state.is_bot_speaking
-
-                if not is_bot_speaking_before and is_bot_speaking_after:
-                    logger.debug("voice_channel.bot_started_speaking")
-                    # relevant when the bot speaks multiple messages in one turn
-                    self._cancel_silence_timeout_watcher()
-
-                # we just stopped speaking, starting a watcher for silence timeout
-                if is_bot_speaking_before and not is_bot_speaking_after:
-                    logger.debug("voice_channel.bot_stopped_speaking")
-                    self._cancel_silence_timeout_watcher()
-                    call_state.silence_timeout_watcher = (  # type: ignore[attr-defined]
-                        asyncio.create_task(
-                            self.handle_silence_timeout(
-                                channel_websocket,
-                                on_new_message,
-                                tts_engine,
-                                call_parameters,
-                            )
-                        )
-                    )
                 if isinstance(channel_action, NewAudioAction):
                     await asr_engine.send_audio_chunks(channel_action.audio_bytes)
                 elif isinstance(channel_action, EndConversationAction):
@@ -378,20 +209,12 @@ class VoiceInputChannel(InputChannel):
                     call_parameters,
                 )
 
-        audio_forwarding_task = asyncio.create_task(consume_audio_bytes())
-        asr_event_task = asyncio.create_task(consume_asr_events())
         await asyncio.wait(
-            [audio_forwarding_task, asr_event_task],
+            [consume_audio_bytes(), consume_asr_events()],
             return_when=asyncio.FIRST_COMPLETED,
         )
-        if not audio_forwarding_task.done():
-            audio_forwarding_task.cancel()
-        if not asr_event_task.done():
-            asr_event_task.cancel()
         await tts_engine.close_connection()
         await asr_engine.close_connection()
-        await channel_websocket.close()
-        self._cancel_silence_timeout_watcher()
 
     def create_output_channel(
         self, voice_websocket: Websocket, tts_engine: TTSEngine
@@ -409,10 +232,7 @@ class VoiceInputChannel(InputChannel):
     ) -> None:
         """Handle a new event from the ASR system."""
         if isinstance(e, NewTranscript) and e.text:
-            logger.debug(
-                "VoiceInputChannel.handle_asr_event.new_transcript", transcript=e.text
-            )
-            call_state.is_user_speaking = False  # type: ignore[attr-defined]
+            logger.info(f"New transcript: {e.text}")
             output_channel = self.create_output_channel(voice_websocket, tts_engine)
             message = UserMessage(
                 e.text,
@@ -422,6 +242,6 @@ class VoiceInputChannel(InputChannel):
                 metadata=asdict(call_parameters),
             )
             await on_new_message(message)
-        elif isinstance(e, UserIsSpeaking):
-            self._cancel_silence_timeout_watcher()
-            call_state.is_user_speaking = True  # type: ignore[attr-defined]
+
+            if output_channel.should_hangup:
+                self.hangup_after = output_channel.latest_message_id

rasa/core/featurizers/single_state_featurizer.py

@@ -1,8 +1,7 @@
 import logging
-from typing import List, Optional, Dict, Text, Set, Any
-
 import numpy as np
 import scipy.sparse
+from typing import List, Optional, Dict, Text, Set, Any
 
 from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization
 from rasa.nlu.extractors.extractor import EntityTagSpec
@@ -361,26 +360,6 @@ class SingleStateFeaturizer:
             for action in domain.action_names_or_texts
         ]
 
-    def to_dict(self) -> Dict[str, Any]:
-        return {
-            "action_texts": self.action_texts,
-            "entity_tag_specs": self.entity_tag_specs,
-            "feature_states": self._default_feature_states,
-        }
-
-    @classmethod
-    def create_from_dict(
-        cls, data: Dict[str, Any]
-    ) -> Optional["SingleStateFeaturizer"]:
-        if not data:
-            return None
-
-        featurizer = SingleStateFeaturizer()
-        featurizer.action_texts = data["action_texts"]
-        featurizer._default_feature_states = data["feature_states"]
-        featurizer.entity_tag_specs = data["entity_tag_specs"]
-        return featurizer
-
 
 class IntentTokenizerSingleStateFeaturizer(SingleStateFeaturizer):
     """A SingleStateFeaturizer for use with policies that predict intent labels."""