rasa-pro 3.12.22__py3-none-any.whl → 3.13.0__py3-none-any.whl

rasa/core/channels/studio_chat.py

@@ -1,5 +1,8 @@
 import asyncio
+import audioop
+import base64
 import json
+import uuid
 from functools import partial
 from typing import (
     TYPE_CHECKING,
@@ -10,13 +13,25 @@ from typing import (
     List,
     Optional,
     Text,
+    Tuple,
 )
 
 import structlog
-from sanic import Sanic
 
+from rasa.core.channels import UserMessage
 from rasa.core.channels.socketio import SocketBlueprint, SocketIOInput
-from rasa.core.exceptions import AgentNotReady
+from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.call_state import call_state
+from rasa.core.channels.voice_stream.tts import TTSEngine
+from rasa.core.channels.voice_stream.voice_channel import (
+    ContinueConversationAction,
+    EndConversationAction,
+    NewAudioAction,
+    VoiceChannelAction,
+    VoiceInputChannel,
+    VoiceOutputChannel,
+)
 from rasa.hooks import hookimpl
 from rasa.plugin import plugin_manager
 from rasa.shared.core.constants import ACTION_LISTEN_NAME
@@ -24,7 +39,10 @@ from rasa.shared.core.events import ActionExecuted
 from rasa.shared.core.trackers import EventVerbosity
 
 if TYPE_CHECKING:
-    from rasa.core.channels.channel import UserMessage
+    from sanic import Sanic, Websocket  # type: ignore[attr-defined]
+    from socketio import AsyncServer
+
+    from rasa.core.channels.channel import InputChannel, UserMessage
     from rasa.shared.core.trackers import DialogueStateTracker
 
 
@@ -84,7 +102,9 @@ class StudioTrackerUpdatePlugin:
 
     def handle_tracker_update(self, tracker: "DialogueStateTracker") -> None:
         """Handles a tracker update when triggered by a hook."""
-        structlogger.info("studio_chat.after_tracker_update", tracker=tracker)
+        structlogger.info(
+            "studio_chat.after_tracker_update", sender_id=tracker.sender_id
+        )
         # directly create a dump to avoid the tracker getting modified by another
         # function before it gets published (since the publishing is scheduled
         # as an async task)
@@ -101,32 +121,83 @@ class StudioTrackerUpdatePlugin:
         self._cancel_tasks()
 
 
-class StudioChatInput(SocketIOInput):
+class StudioChatInput(SocketIOInput, VoiceInputChannel):
     """Input channel for the communication between Rasa Studio and Rasa Pro."""
 
+    requires_voice_license = False
+
     @classmethod
     def name(cls) -> Text:
         return "studio_chat"
 
     def __init__(
         self,
-        *args: Any,
-        **kwargs: Any,
+        server_url: str,
+        asr_config: Dict,
+        tts_config: Dict,
+        user_message_evt: Text = "user_uttered",
+        bot_message_evt: Text = "bot_uttered",
+        namespace: Optional[Text] = None,
+        session_persistence: bool = False,
+        socketio_path: Optional[Text] = "/socket.io",
+        jwt_key: Optional[Text] = None,
+        jwt_method: Optional[Text] = "HS256",
+        metadata_key: Optional[Text] = "metadata",
     ) -> None:
-        """Creates a ``SocketIOInput`` object."""
+        """Creates a `StudioChatInput` object."""
         from rasa.core.agent import Agent
 
-        super().__init__(*args, **kwargs)
         self.agent: Optional[Agent] = None
 
+        # Initialize the SocketIO input channel
+        SocketIOInput.__init__(
+            self,
+            user_message_evt=user_message_evt,
+            bot_message_evt=bot_message_evt,
+            namespace=namespace,
+            session_persistence=session_persistence,
+            socketio_path=socketio_path,
+            jwt_key=jwt_key,
+            jwt_method=jwt_method,
+            metadata_key=metadata_key,
+        )
+
+        # Initialize the Voice Input Channel
+        VoiceInputChannel.__init__(
+            self,
+            server_url=server_url,
+            asr_config=asr_config,
+            tts_config=tts_config,
+        )
+
+        # Dictionaries to manage active connections and background tasks
+        # `active_connections` holds the active voice sessions
+        # `background_tasks` holds the asyncio tasks for voice streaming
+        self.active_connections: Dict[str, SocketIOVoiceWebsocketAdapter] = {}
+        self.background_tasks: Dict[str, asyncio.Task] = {}
+
         self._register_tracker_update_hook()
 
-    async def emit(self, event: str, data: Dict, room: str) -> None:
-        """Emits an event to the websocket."""
-        if not self.sio:
-            structlogger.error("studio_chat.emit.sio_not_initialized")
-            return
-        await self.sio.emit(event, data, room=room)
+    @classmethod
+    def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> "InputChannel":
+        """Creates a StudioChatInput channel from credentials."""
+        credentials = credentials or {}
+
+        return cls(
+            # Voice specific parameters
+            server_url=credentials.get("server_url", ""),
+            asr_config=credentials.get("asr", {}),
+            tts_config=credentials.get("tts", {}),
+            # SocketIO parameters
+            user_message_evt=credentials.get("user_message_evt", "user_uttered"),
+            bot_message_evt=credentials.get("bot_message_evt", "bot_uttered"),
+            namespace=credentials.get("namespace"),
+            session_persistence=credentials.get("session_persistence", False),
+            socketio_path=credentials.get("socketio_path", "/socket.io"),
+            jwt_key=credentials.get("jwt_key"),
+            jwt_method=credentials.get("jwt_method", "HS256"),
+            metadata_key=credentials.get("metadata_key", "metadata"),
+        )
 
     def _register_tracker_update_hook(self) -> None:
         plugin_manager().register(StudioTrackerUpdatePlugin(self))
@@ -137,7 +208,10 @@ class StudioChatInput(SocketIOInput):
 
     async def publish_tracker_update(self, sender_id: str, tracker_dump: Dict) -> None:
         """Publishes a tracker update notification to the websocket."""
-        await self.emit("tracker", tracker_dump, room=sender_id)
+        if not self.sio:
+            structlogger.error("studio_chat.on_tracker_updated.sio_not_initialized")
+            return
+        await self.sio.emit("tracker", tracker_dump, room=sender_id)
 
     async def on_message_proxy(
         self,
@@ -150,15 +224,8 @@ class StudioChatInput(SocketIOInput):
         """
         await on_new_message(message)
 
-        if not self.agent or not self.agent.is_ready():
+        if not self.agent:
             structlogger.error("studio_chat.on_message_proxy.agent_not_initialized")
-            await self.emit_error(
-                "The Rasa Pro model could not be loaded. "
-                "Please check the training and deployment logs "
-                "for more information.",
-                message.sender_id,
-                AgentNotReady("The Rasa Pro model could not be loaded."),
-            )
             return
 
         tracker = await self.agent.tracker_store.retrieve(message.sender_id)
@@ -168,17 +235,6 @@ class StudioChatInput(SocketIOInput):
 
         await self.on_tracker_updated(tracker)
 
-    async def emit_error(self, message: str, room: str, e: Exception) -> None:
-        await self.emit(
-            "error",
-            {
-                "message": message,
-                "error": str(e),
-                "exception": str(type(e).__name__),
-            },
-            room=room,
-        )
-
     async def handle_tracker_update(self, sid: str, data: Dict) -> None:
         from rasa.shared.core.trackers import DialogueStateTracker
 
@@ -195,44 +251,134 @@ class StudioChatInput(SocketIOInput):
             structlogger.error("studio_chat.sio.domain_not_initialized")
             return None
 
-        tracker: Optional[DialogueStateTracker] = None
-
         async with self.agent.lock_store.lock(data["sender_id"]):
-            try:
-                tracker = DialogueStateTracker.from_dict(
-                    data["sender_id"], data["events"], domain.slots
-                )
+            tracker = DialogueStateTracker.from_dict(
+                data["sender_id"], data["events"], domain.slots
+            )
 
-                # will override an existing tracker with the same id!
+            # will override an existing tracker with the same id!
+            await self.agent.tracker_store.save(tracker)
+
+            processor = self.agent.processor
+            if processor and does_need_action_prediction(tracker):
+                output_channel = self.get_output_channel()
+
+                await processor._run_prediction_loop(output_channel, tracker)
                 await self.agent.tracker_store.save(tracker)
 
-                processor = self.agent.processor
-                if processor and does_need_action_prediction(tracker):
-                    output_channel = self.get_output_channel()
-
-                    await processor._run_prediction_loop(output_channel, tracker)
-                    await processor.run_anonymization_pipeline(tracker)
-                    await self.agent.tracker_store.save(tracker)
-            except Exception as e:
-                structlogger.error(
-                    "studio_chat.sio.handle_tracker_update.error",
-                    error=e,
-                    sender_id=data["sender_id"],
-                )
-                await self.emit_error(
-                    "An error occurred while updating the conversation.",
-                    data["sender_id"],
-                    e,
-                )
-
-        if not tracker:
-            # in case the tracker couldn't be updated, we retrieve the prior
-            # version and use that to populate the update
-            tracker = await self.agent.tracker_store.get_or_create_tracker(
-                data["sender_id"]
-            )
         await self.on_tracker_updated(tracker)
 
+    def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
+        """Voice method to convert channel bytes to RasaAudioBytes."""
+        return RasaAudioBytes(audioop.lin2ulaw(input_bytes, 4))
+
+    async def collect_call_parameters(
+        self, channel_websocket: "Websocket"
+    ) -> Optional[CallParameters]:
+        """Voice method to collect call parameters"""
+        session_id = channel_websocket.session_id
+        return CallParameters(session_id, "local", "local", stream_id=session_id)
+
+    def map_input_message(
+        self,
+        message: Any,
+        ws: "Websocket",
+    ) -> VoiceChannelAction:
+        """Voice method to map websocket messages to actions."""
+        if "audio" in message:
+            channel_bytes = base64.b64decode(message["audio"])
+            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(channel_bytes)
+            return NewAudioAction(audio_bytes)
+        elif "marker" in message:
+            if message["marker"] == call_state.latest_bot_audio_id:
+                # Just finished streaming last audio bytes
+                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
+                if call_state.should_hangup:
+                    structlogger.debug(
+                        "studio_chat.hangup", marker=call_state.latest_bot_audio_id
+                    )
+                    return EndConversationAction()
+            else:
+                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+        return ContinueConversationAction()
+
+    def create_output_channel(
+        self, voice_websocket: "Websocket", tts_engine: TTSEngine
+    ) -> VoiceOutputChannel:
+        """Create a voice output channel"""
+        return StudioVoiceOutputChannel(
+            voice_websocket,
+            tts_engine,
+            self.tts_cache,
+        )
+
+    def _start_voice_session(
+        self,
+        session_id: str,
+        sid: str,
+        on_new_message: Callable[[UserMessage], Awaitable[Any]],
+    ) -> None:
+        """Create SocketIO WebSocket Adaptor & start async task for voice streaming."""
+        if sid in self.active_connections:
+            structlogger.warning(
+                "studio_chat._start_voice_session.session_already_active",
+                session_id=sid,
+            )
+            return
+
+        structlogger.info(
+            "studio_chat._start_voice_session.starting_session", session_id=sid
+        )
+
+        # Create a websocket adapter for this connection
+        ws_adapter = SocketIOVoiceWebsocketAdapter(
+            sio=self.sio,
+            session_id=session_id,
+            sid=sid,
+            bot_message_evt=self.bot_message_evt,
+        )
+        self.active_connections[sid] = ws_adapter
+
+        # Start voice streaming in an async task
+        task = asyncio.create_task(
+            self._handle_voice_streaming(on_new_message, ws_adapter, sid)
+        )
+        self.background_tasks[sid] = task
+        task.add_done_callback(lambda _: self._cleanup_tasks_for_sid(sid))
+
+    async def _handle_voice_streaming(
+        self,
+        on_new_message: Callable[[UserMessage], Awaitable[Any]],
+        ws_adapter: "Websocket",
+        sid: str,
+    ) -> None:
+        """Handle voice streaming for a Socket.IO connection."""
+        try:
+            await self.run_audio_streaming(on_new_message, ws_adapter)
+        except Exception as e:
+            structlogger.exception(
+                "studio_voice.voice_streaming.error",
+                error=str(e),
+                sid=sid,
+            )
+            if sid in self.active_connections:
+                del self.active_connections[sid]
+
+    def _cleanup_tasks_for_sid(self, sid: str) -> None:
+        if sid in self.background_tasks:
+            task = self.background_tasks.pop(sid)
+            task.cancel()
+        if sid in self.active_connections:
+            del self.active_connections[sid]
+
+    @hookimpl  # type: ignore[misc]
+    def after_server_stop(self) -> None:
+        """Cleanup background tasks and active connections when the server stops."""
+        structlogger.info("studio_chat.after_server_stop.cleanup")
+        self.active_connections.clear()
+        for task in self.background_tasks.values():
+            task.cancel()
+
     def blueprint(
         self, on_new_message: Callable[["UserMessage"], Awaitable[Any]]
     ) -> SocketBlueprint:
@@ -245,11 +391,126 @@ class StudioChatInput(SocketIOInput):
             return socket_blueprint
 
         @socket_blueprint.listener("after_server_start")  # type: ignore[misc]
-        async def after_server_start(app: Sanic, _: asyncio.AbstractEventLoop) -> None:
+        async def after_server_start(
+            app: "Sanic", _: asyncio.AbstractEventLoop
+        ) -> None:
             self.agent = app.ctx.agent
 
+        @self.sio.on("disconnect", namespace=self.namespace)
+        async def disconnect(sid: Text) -> None:
+            structlogger.debug("studio_chat.sio.disconnect", sid=sid)
+            self._cleanup_tasks_for_sid(sid)
+
+        @self.sio.on("session_request", namespace=self.namespace)
+        async def session_request(sid: Text, data: Optional[Dict]) -> None:
+            """Overrides the base SocketIOInput session_request handler.
+
+            Args:
+              sid: ID of the session (from SocketIO).
+              data:
+                - session_id: Studio Chat channel is used with a Bridge Architecture
+                  (Model Service's Socket Bridge), so we use session_id to remain
+                  consistent across the bridge. Session ID becomes the sender_id
+                  for the UserMessage.
+                - is_voice: Boolean indicating if its a voice session.
+            """
+            # Call parent session_request handler first
+            await self.handle_session_request(sid, data)
+
+            # start a voice session if requested
+            if data and data.get("is_voice", False):
+                self._start_voice_session(data["session_id"], sid, on_new_message)
+
+        @self.sio.on(self.user_message_evt, namespace=self.namespace)
+        async def handle_message(sid: Text, data: Dict) -> None:
+            """Overrides the base SocketIOInput handle_message handler."""
+            # Handle voice messages
+            if "audio" in data or "marker" in data:
+                if sid in self.active_connections:
+                    # Route audio messages to the voice adapter queue
+                    ws = self.active_connections[sid]
+                    ws.put_message(data)
+                return
+
+            # Handle text messages
+            await self.handle_user_message(sid, data, on_new_message)
+
         @self.sio.on("update_tracker", namespace=self.namespace)
         async def on_update_tracker(sid: Text, data: Dict) -> None:
             await self.handle_tracker_update(sid, data)
 
         return socket_blueprint
+
+
+class StudioVoiceOutputChannel(VoiceOutputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "studio_chat"
+
+    def rasa_audio_bytes_to_channel_bytes(
+        self, rasa_audio_bytes: RasaAudioBytes
+    ) -> bytes:
+        return audioop.ulaw2lin(rasa_audio_bytes, 4)
+
+    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
+        return json.dumps({"audio": base64.b64encode(channel_bytes).decode("utf-8")})
+
+    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
+        message_id = uuid.uuid4().hex
+        return json.dumps({"marker": message_id}), message_id
+
+
+class SocketIOVoiceWebsocketAdapter:
+    """Adapter to make Socket.IO work like a Sanic WebSocket for voice channels."""
+
+    def __init__(
+        self, sio: "AsyncServer", session_id: str, sid: str, bot_message_evt: str
+    ) -> None:
+        self.sio = sio
+        self.bot_message_evt = bot_message_evt
+        self._closed = False
+        self._receive_queue: asyncio.Queue[Any] = asyncio.Queue()
+
+        # the messages need to be emitted on room=sid
+        self.sid = sid
+
+        # used by collect_call_parameters
+        # ultimately, this becomes the sender_id
+        self.session_id = session_id
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    async def send(self, data: Any) -> None:
+        """Send data to the client."""
+        if not self.closed:
+            await self.sio.emit(self.bot_message_evt, data, room=self.sid)
+
+    async def recv(self) -> Any:
+        """Receive data from the client."""
+        if self.closed:
+            raise ConnectionError("WebSocket is closed")
+        return await self._receive_queue.get()
+
+    def put_message(self, message: Any) -> None:
+        """Put message in the internal receive queue."""
+        self._receive_queue.put_nowait(message)
+
+    async def close(self, code: int = 1000, reason: str = "") -> None:
+        """Close the connection."""
+        self._closed = True
+        # at this point, the client should have disconnected
+
+    def __aiter__(self) -> "SocketIOVoiceWebsocketAdapter":
+        """Allow the adapter to be used in an async for loop."""
+        return self
+
+    async def __anext__(self) -> Any:
+        if self.closed:
+            raise StopAsyncIteration
+        try:
+            message = await self.recv()
+            return message
+        except Exception:
+            raise StopAsyncIteration

rasa/core/channels/vier_cvg.py

@@ -129,9 +129,8 @@ class CVGOutput(OutputChannel):
                 )
 
             logger.info(
-                "Creating incoming UserMessage: {text=%s, output_channel=%s, sender_id=%s, metadata=%s}"  # noqa: E501
+                "Creating incoming UserMessage: {output_channel=%s, sender_id=%s, metadata=%s}"  # noqa: E501
                 % (
-                    user_message.text,
                     user_message.output_channel,
                     user_message.sender_id,
                     user_message.metadata,

rasa/core/channels/voice_ready/audiocodes.py

@@ -149,7 +149,7 @@ class Conversation:
             structlogger.warning(
                 "audiocodes.handle.activities.empty_input_channel_name",
                 event_info=(
-                    "Audiocodes input channel name is empty "
+                    f"Audiocodes input channel name is empty "
                     f"for conversation {self.conversation_id}"
                 ),
             )
@@ -177,7 +177,7 @@ class Conversation:
             else:
                 structlogger.warning(
                     "audiocodes.handle.activities.unknown_activity_type",
-                    activity=activity,
+                    activity_type=activity["type"],
                 )
                 continue
 
@@ -193,16 +193,9 @@ class Conversation:
             try:
                 await on_new_message(user_msg)
             except Exception as e:  # skipcq: PYL-W0703
-                if isinstance(user_msg.text, dict):
-                    anonymized_info = json.dumps(user_msg.text)
-                elif isinstance(user_msg.text, str):
-                    anonymized_info = user_msg.text
-                else:
-                    anonymized_info = INFO_UNKNOWN
-
                 structlogger.exception(
                     "audiocodes.handle.activities.failure",
-                    user_message=copy.deepcopy(anonymized_info),
+                    sender_id=self.conversation_id,
                     error=e,
                     exc_info=True,
                 )
@@ -495,8 +488,8 @@ class AudiocodesInput(InputChannel):
         await on_new_message(
             UserMessage(
                 text=f"{INTENT_MESSAGE_PREFIX}session_end",
-                input_channel=self.name(),
                 output_channel=None,
+                input_channel=self.name(),
                 sender_id=conversation_id,
                 metadata=reason,
             )

rasa/core/channels/voice_ready/jambonz.py

@@ -17,7 +17,10 @@ from rasa.core.channels.voice_ready.jambonz_protocol import (
     send_ws_text_message,
     websocket_message_handler,
 )
-from rasa.core.channels.voice_ready.utils import validate_voice_license_scope
+from rasa.core.channels.voice_ready.utils import (
+    validate_username_password_credentials,
+    validate_voice_license_scope,
+)
 from rasa.shared.exceptions import RasaException
 from rasa.shared.utils.common import mark_as_beta_feature
 from rasa.utils.io import remove_emojis
@@ -41,11 +44,7 @@ class JambonzVoiceReadyInput(InputChannel):
 
         username = credentials.get("username")
         password = credentials.get("password")
-        if (username is None) != (password is None):
-            raise RasaException(
-                "In Jambonz channel, either both username and password "
-                "or neither should be provided. "
-            )
+        validate_username_password_credentials(username, password, "Jambonz")
 
         return cls(username, password)
 

rasa/core/channels/voice_ready/twilio_voice.py

@@ -16,9 +16,12 @@ from rasa.core.channels.channel import (
     create_auth_requested_response_provider,
     requires_basic_auth,
 )
-from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_ready.utils import (
+    CallParameters,
+    validate_username_password_credentials,
+)
 from rasa.shared.core.events import BotUttered
-from rasa.shared.exceptions import InvalidConfigException, RasaException
+from rasa.shared.exceptions import InvalidConfigException
 
 logger = structlog.get_logger(__name__)
 
@@ -127,11 +130,7 @@ class TwilioVoiceInput(InputChannel):
 
         username = credentials.get("username")
         password = credentials.get("password")
-        if (username is None) != (password is None):
-            raise RasaException(
-                "In TwilioVoice channel, either both username and password "
-                "or neither should be provided. "
-            )
+        validate_username_password_credentials(username, password, "TwilioVoice")
 
         return cls(
             credentials.get(
@@ -180,8 +179,9 @@ class TwilioVoiceInput(InputChannel):
         if self.assistant_voice not in self.SUPPORTED_VOICES:
             self._raise_invalid_voice_exception()
 
-        if (self.username is None) != (self.password is None):
-            self._raise_invalid_credentials_exception()
+        validate_username_password_credentials(
+            self.username, self.password, "TwilioVoice"
+        )
 
         try:
             int(self.speech_timeout)
@@ -389,9 +389,10 @@ class TwilioVoiceInput(InputChannel):
         return voice_response
 
     def _raise_invalid_credentials_exception(self) -> None:
-        raise InvalidConfigException(
-            "In TwilioVoice channel, either both username and password "
-            "or neither should be provided. "
+        # This method is now redundant since we use the shared validation function
+        # but keeping it for backward compatibility if any external code calls it
+        validate_username_password_credentials(
+            self.username, self.password, "TwilioVoice"
         )
 
 

rasa/core/channels/voice_ready/utils.py

@@ -3,9 +3,31 @@ from typing import Optional
 
 import structlog
 
+from rasa.shared.exceptions import InvalidConfigException
+
 structlogger = structlog.get_logger()
 
 
+def validate_username_password_credentials(
+    username: Optional[str], password: Optional[str], channel_name: str
+) -> None:
+    """Validate that username and password are both provided or both None.
+
+    Args:
+        username: The username credential, or None
+        password: The password credential, or None
+        channel_name: The name of the channel for error message
+
+    Raises:
+        InvalidConfigException: If only one of username/password is provided
+    """
+    if (not username) != (not password):
+        raise InvalidConfigException(
+            f"In {channel_name} channel, either both username and password "
+            "or neither should be provided."
+        )
+
+
 def validate_voice_license_scope() -> None:
     from rasa.utils.licensing import (
         PRODUCT_AREA,