rasa-pro 3.11.0__py3-none-any.whl → 3.11.0a2__py3-none-any.whl

rasa/core/channels/voice_stream/browser_audio.py

@@ -1,10 +1,7 @@
 import audioop
-import base64
-import json
-
 import structlog
 import uuid
-from typing import Any, Awaitable, Callable, Optional, Tuple
+from typing import Any, Awaitable, Callable, List, Optional
 
 from sanic import Blueprint, HTTPResponse, Request, response
 from sanic import Websocket  # type: ignore
@@ -12,19 +9,16 @@ from sanic import Websocket  # type: ignore
 
 from rasa.core.channels import UserMessage
 from rasa.core.channels.voice_ready.utils import CallParameters
-from rasa.core.channels.voice_stream.call_state import call_state
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.voice_channel import (
-    ContinueConversationAction,
-    EndConversationAction,
     NewAudioAction,
     VoiceChannelAction,
     VoiceInputChannel,
     VoiceOutputChannel,
 )
 
-logger = structlog.get_logger()
+structlogger = structlog.get_logger()
 
 
 class BrowserAudioOutputChannel(VoiceOutputChannel):
@@ -37,12 +31,10 @@ class BrowserAudioOutputChannel(VoiceOutputChannel):
     ) -> bytes:
         return audioop.ulaw2lin(rasa_audio_bytes, 4)
 
-    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
-        return json.dumps({"audio": base64.b64encode(channel_bytes).decode("utf-8")})
-
-    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
-        message_id = uuid.uuid4().hex
-        return json.dumps({"marker": message_id}), message_id
+    def channel_bytes_to_messages(
+        self, recipient_id: str, channel_bytes: bytes
+    ) -> List[Any]:
+        return [channel_bytes]
 
 
 class BrowserAudioInputChannel(VoiceInputChannel):
@@ -63,23 +55,8 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         self,
         message: Any,
     ) -> VoiceChannelAction:
-        data = json.loads(message)
-        if "audio" in data:
-            channel_bytes = base64.b64decode(data["audio"])
-            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(channel_bytes)
-            return NewAudioAction(audio_bytes)
-        elif "marker" in data:
-            if data["marker"] == call_state.latest_bot_audio_id:
-                # Just finished streaming last audio bytes
-                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
-                if call_state.should_hangup:
-                    logger.debug(
-                        "browser_audio.hangup", marker=call_state.latest_bot_audio_id
-                    )
-                    return EndConversationAction()
-            else:
-                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
-        return ContinueConversationAction()
+        audio_bytes = self.channel_bytes_to_rasa_audio_bytes(message)
+        return NewAudioAction(audio_bytes)
 
     def create_output_channel(
         self, voice_websocket: Websocket, tts_engine: TTSEngine
@@ -94,7 +71,7 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
     ) -> Blueprint:
         """Defines a Sanic bluelogger.debug."""
-        blueprint = Blueprint("browser_audio", __name__)
+        blueprint = Blueprint("socketio_webhook", __name__)
 
         @blueprint.route("/", methods=["GET"])
         async def health(_: Request) -> HTTPResponse:

rasa/core/channels/voice_stream/tts/azure.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 
 import aiohttp
 import structlog
-from aiohttp import ClientConnectorError, ClientTimeout
+from aiohttp import ClientConnectorError
 
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.tts.tts_engine import (
@@ -12,7 +12,6 @@ from rasa.core.channels.voice_stream.tts.tts_engine import (
     TTSEngineConfig,
     TTSError,
 )
-from rasa.shared.constants import AZURE_SPEECH_API_KEY_ENV_VAR
 from rasa.shared.exceptions import ConnectionException
 
 
@@ -26,15 +25,13 @@ class AzureTTSConfig(TTSEngineConfig):
 
 class AzureTTS(TTSEngine[AzureTTSConfig]):
     session: Optional[aiohttp.ClientSession] = None
-    required_env_vars = (AZURE_SPEECH_API_KEY_ENV_VAR,)
 
     def __init__(self, config: Optional[AzureTTSConfig] = None):
         super().__init__(config)
-        timeout = ClientTimeout(total=self.config.timeout)
         # Have to create this class-shared session lazily at run time otherwise
         # the async event loop doesn't work
         if self.__class__.session is None or self.__class__.session.closed:
-            self.__class__.session = aiohttp.ClientSession(timeout=timeout)
+            self.__class__.session = aiohttp.ClientSession()
 
     async def synthesize(
         self, text: str, config: Optional[AzureTTSConfig] = None
@@ -63,12 +60,10 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
                     raise TTSError(f"TTS failed: {response.text()}")
         except ClientConnectorError as e:
             raise TTSError(e)
-        except TimeoutError as e:
-            raise TTSError(e)
 
     @staticmethod
     def get_request_headers() -> dict[str, str]:
-        azure_speech_api_key = os.environ[AZURE_SPEECH_API_KEY_ENV_VAR]
+        azure_speech_api_key = os.environ["AZURE_SPEECH_API_KEY"]
         return {
             "Ocp-Apim-Subscription-Key": azure_speech_api_key,
             "Content-Type": "application/ssml+xml",
@@ -97,7 +92,6 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
         return AzureTTSConfig(
             language="en-US",
             voice="en-US-JennyNeural",
-            timeout=10,
             speech_region="germanywestcentral",
         )
 

rasa/core/channels/voice_stream/tts/cartesia.py

@@ -3,19 +3,20 @@ from typing import AsyncIterator, Dict, Optional
 import os
 import aiohttp
 import structlog
-from aiohttp import ClientConnectorError, ClientTimeout
+from aiohttp import ClientConnectorError
 
 from rasa.core.channels.voice_stream.tts.tts_engine import (
     TTSEngineConfig,
 )
 
-from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine, TTSError
-from rasa.shared.constants import CARTESIA_API_KEY_ENV_VAR
 from rasa.shared.exceptions import ConnectionException
 
 structlogger = structlog.get_logger()
 
+CARTESIA_API_KEY = "CARTESIA_API_KEY"
+
 
 @dataclass
 class CartesiaTTSConfig(TTSEngineConfig):
@@ -25,15 +26,13 @@ class CartesiaTTSConfig(TTSEngineConfig):
 
 class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
     session: Optional[aiohttp.ClientSession] = None
-    required_env_vars = (CARTESIA_API_KEY_ENV_VAR,)
 
     def __init__(self, config: Optional[CartesiaTTSConfig] = None):
         super().__init__(config)
-        timeout = ClientTimeout(total=self.config.timeout)
         # Have to create this class-shared session lazily at run time otherwise
         # the async event loop doesn't work
         if self.__class__.session is None or self.__class__.session.closed:
-            self.__class__.session = aiohttp.ClientSession(timeout=timeout)
+            self.__class__.session = aiohttp.ClientSession()
 
     @staticmethod
     def get_tts_endpoint() -> str:
@@ -56,13 +55,13 @@ class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
             "output_format": {
                 "container": "raw",
                 "encoding": "pcm_mulaw",
-                "sample_rate": HERTZ,
+                "sample_rate": 8000,
             },
         }
 
     @staticmethod
     def get_request_headers(config: CartesiaTTSConfig) -> dict[str, str]:
-        cartesia_api_key = os.environ[CARTESIA_API_KEY_ENV_VAR]
+        cartesia_api_key = os.environ.get(CARTESIA_API_KEY)
         return {
             "Cartesia-Version": str(config.version),
             "Content-Type": "application/json",
@@ -89,15 +88,13 @@ class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
                     return
                 else:
                     structlogger.error(
-                        "cartesia.synthesize.rest.failed",
+                        "azure.synthesize.rest.failed",
                         status_code=response.status,
                         msg=response.text(),
                     )
                     raise TTSError(f"TTS failed: {response.text()}")
         except ClientConnectorError as e:
             raise TTSError(e)
-        except TimeoutError as e:
-            raise TTSError(e)
 
     def engine_bytes_to_rasa_audio_bytes(self, chunk: bytes) -> RasaAudioBytes:
         """Convert the generated tts audio bytes into rasa audio bytes."""
@@ -108,7 +105,6 @@ class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
         return CartesiaTTSConfig(
             language="en",
             voice="248be419-c632-4f23-adf1-5324ed7dbf1d",
-            timeout=10,
             model_id="sonic-english",
             version="2024-06-10",
         )

rasa/core/channels/voice_stream/tts/tts_engine.py

@@ -1,10 +1,9 @@
-from typing import AsyncIterator, Dict, Generic, Optional, Tuple, Type, TypeVar
+from typing import AsyncIterator, Dict, Generic, Optional, Type, TypeVar
 from dataclasses import dataclass
 
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.util import MergeableConfig
 from rasa.shared.exceptions import RasaException
-from rasa.shared.utils.common import validate_environment
 
 
 class TTSError(RasaException):
@@ -19,20 +18,11 @@ E = TypeVar("E", bound="TTSEngine")
 class TTSEngineConfig(MergeableConfig):
     language: Optional[str] = None
     voice: Optional[str] = None
-    timeout: Optional[int] = None
 
 
 class TTSEngine(Generic[T]):
-    required_env_vars: Tuple[str, ...] = ()
-    required_packages: Tuple[str, ...] = ()
-
     def __init__(self, config: Optional[T] = None):
         self.config = self.get_default_config().merge(config)
-        validate_environment(
-            self.required_env_vars,
-            self.required_packages,
-            f"TTS Engine {self.__class__.__name__}",
-        )
 
     async def close_connection(self) -> None:
         """Cleanup the connection if necessary."""

rasa/core/channels/voice_stream/twilio_media_streams.py

@@ -1,9 +1,8 @@
 import base64
 import json
-import uuid
-
 import structlog
-from typing import Any, Awaitable, Callable, Dict, Optional, Text, Tuple
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Text
+import uuid
 
 from sanic import Blueprint, HTTPResponse, Request, response
 from sanic import Websocket  # type: ignore
@@ -11,7 +10,6 @@ from sanic import Websocket  # type: ignore
 
 from rasa.core.channels import UserMessage
 from rasa.core.channels.voice_ready.utils import CallParameters
-from rasa.core.channels.voice_stream.call_state import call_state
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.voice_channel import (
@@ -23,7 +21,7 @@ from rasa.core.channels.voice_stream.voice_channel import (
     VoiceOutputChannel,
 )
 
-logger = structlog.get_logger(__name__)
+structlogger = structlog.get_logger()
 
 
 def map_call_params(data: Dict[Text, Any]) -> CallParameters:
@@ -49,18 +47,10 @@ class TwilioMediaStreamsOutputChannel(VoiceOutputChannel):
     ) -> bytes:
         return base64.b64encode(rasa_audio_bytes)
 
-    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
+    def channel_bytes_to_messages(
+        self, recipient_id: str, channel_bytes: bytes
+    ) -> List[Any]:
         message_id = uuid.uuid4().hex
-        mark_message = json.dumps(
-            {
-                "event": "mark",
-                "streamSid": recipient_id,
-                "mark": {"name": message_id},
-            }
-        )
-        return mark_message, message_id
-
-    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
         media_message = json.dumps(
             {
                 "event": "media",
@@ -70,7 +60,15 @@ class TwilioMediaStreamsOutputChannel(VoiceOutputChannel):
                 },
             }
         )
-        return media_message
+        mark_message = json.dumps(
+            {
+                "event": "mark",
+                "streamSid": recipient_id,
+                "mark": {"name": message_id},
+            }
+        )
+        self.latest_message_id = message_id
+        return [media_message, mark_message]
 
 
 class TwilioMediaStreamsInputChannel(VoiceInputChannel):
@@ -105,16 +103,9 @@ class TwilioMediaStreamsInputChannel(VoiceInputChannel):
         elif data["event"] == "stop":
             return EndConversationAction()
         elif data["event"] == "mark":
-            if data["mark"]["name"] == call_state.latest_bot_audio_id:
-                # Just finished streaming last audio bytes
-                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
-                if call_state.should_hangup:
-                    logger.debug(
-                        "twilio_streams.hangup", marker=call_state.latest_bot_audio_id
-                    )
-                    return EndConversationAction()
-            else:
-                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+            if data["mark"]["name"] == self.hangup_after:
+                structlogger.debug("twilio_streams.hangup", marker=self.hangup_after)
+                return EndConversationAction()
         return ContinueConversationAction()
 
     def create_output_channel(
@@ -140,7 +131,7 @@ class TwilioMediaStreamsInputChannel(VoiceInputChannel):
         self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
     ) -> Blueprint:
         """Defines a Sanic bluelogger.debug."""
-        blueprint = Blueprint("twilio_media_streams", __name__)
+        blueprint = Blueprint("socketio_webhook", __name__)
 
         @blueprint.route("/", methods=["GET"])
         async def health(_: Request) -> HTTPResponse:

rasa/core/channels/voice_stream/util.py

@@ -5,7 +5,7 @@ from typing import Optional, Type, TypeVar
 
 import structlog
 
-from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.shared.exceptions import RasaException
 
 structlogger = structlog.get_logger()
@@ -23,16 +23,16 @@ def read_wav_to_rasa_audio_bytes(file_name: str) -> Optional[RasaAudioBytes]:
         wave_data = audioop.lin2lin(wave_data, wave_object.getsampwidth(), 1)
         # 8 bit is unsigned
         # wave_data = audioop.bias(wave_data, 1, 128)
-    if wave_object.getframerate() != HERTZ:
+    if wave_object.getframerate() != 8000:
         wave_data, _ = audioop.ratecv(
-            wave_data, 1, 1, wave_object.getframerate(), HERTZ, None
+            wave_data, 1, 1, wave_object.getframerate(), 8000, None
         )
     wave_data = audioop.lin2ulaw(wave_data, 1)
     return RasaAudioBytes(wave_data)
 
 
 def generate_silence(length_in_seconds: float = 1.0) -> RasaAudioBytes:
-    return RasaAudioBytes(b"\00" * int(length_in_seconds * HERTZ))
+    return RasaAudioBytes(b"\00" * int(length_in_seconds * 8000))
 
 
 T = TypeVar("T", bound="MergeableConfig")