rasa-pro 3.12.0.dev13__py3-none-any.whl → 3.12.0rc2__py3-none-any.whl

rasa/core/channels/voice_stream/audiocodes.py

@@ -0,0 +1,264 @@
+import asyncio
+import base64
+import json
+from typing import Any, Awaitable, Callable, Dict, Optional, Text
+
+import structlog
+from sanic import (  # type: ignore[attr-defined]
+    Blueprint,
+    HTTPResponse,
+    Request,
+    Websocket,
+    response,
+)
+
+from rasa.core.channels import UserMessage
+from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.call_state import (
+    call_state,
+)
+from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
+from rasa.core.channels.voice_stream.voice_channel import (
+    ContinueConversationAction,
+    EndConversationAction,
+    NewAudioAction,
+    VoiceChannelAction,
+    VoiceInputChannel,
+    VoiceOutputChannel,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+def map_call_params(data: Dict[Text, Any]) -> CallParameters:
+    """Map the audiocodes stream parameters to the CallParameters dataclass."""
+    return CallParameters(
+        call_id=data["conversationId"],
+        user_phone=data["caller"],
+        # Bot phone is not available in the Audiocodes API
+        direction="inbound",  # AudioCodes calls are always inbound
+    )
+
+
+class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "ac_voice"
+
+    def rasa_audio_bytes_to_channel_bytes(
+        self, rasa_audio_bytes: RasaAudioBytes
+    ) -> bytes:
+        return base64.b64encode(rasa_audio_bytes)
+
+    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
+        media_message = json.dumps(
+            {
+                "type": "playStream.chunk",
+                "streamId": str(call_state.stream_id),
+                "audioChunk": channel_bytes.decode("utf-8"),
+            }
+        )
+        return media_message
+
+    async def send_start_marker(self, recipient_id: str) -> None:
+        """Send playStream.start before first audio chunk."""
+        call_state.stream_id += 1  # type: ignore[attr-defined]
+        media_message = json.dumps(
+            {
+                "type": "playStream.start",
+                "streamId": str(call_state.stream_id),
+            }
+        )
+        logger.debug("Sending start marker", stream_id=call_state.stream_id)
+        await self.voice_websocket.send(media_message)
+
+    async def send_intermediate_marker(self, recipient_id: str) -> None:
+        """Audiocodes doesn't need intermediate markers, so do nothing."""
+        pass
+
+    async def send_end_marker(self, recipient_id: str) -> None:
+        """Send playStream.stop after last audio chunk."""
+        media_message = json.dumps(
+            {
+                "type": "playStream.stop",
+                "streamId": str(call_state.stream_id),
+            }
+        )
+        logger.debug("Sending end marker", stream_id=call_state.stream_id)
+        await self.voice_websocket.send(media_message)
+
+
+class AudiocodesVoiceInputChannel(VoiceInputChannel):
+    @classmethod
+    def name(cls) -> str:
+        return "ac_voice"
+
+    def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
+        return RasaAudioBytes(base64.b64decode(input_bytes))
+
+    async def collect_call_parameters(
+        self, channel_websocket: Websocket
+    ) -> Optional[CallParameters]:
+        async for message in channel_websocket:
+            data = json.loads(message)
+            if data["type"] == "session.initiate":
+                # retrieve parameters set in the webhook - contains info about the
+                # caller
+                logger.info("received initiate message", data=data)
+                self._send_accepted(channel_websocket, data)
+                return map_call_params(data)
+            else:
+                logger.warning("ac_voice.unknown_message", data=data)
+        return None
+
+    def map_input_message(
+        self,
+        message: Any,
+        ws: Websocket,
+    ) -> VoiceChannelAction:
+        data = json.loads(message)
+        if data["type"] == "activities":
+            activities = data["activities"]
+            for activity in activities:
+                logger.debug("ac_voice.activity", data=activity)
+                if activity["name"] == "start":
+                    pass
+                elif activity["name"] == "dtmf":
+                    # TODO: handle DTMF input
+                    pass
+                elif activity["name"] == "playFinished":
+                    logger.debug("ac_voice.playFinished", data=activity)
+                    if call_state.should_hangup:
+                        logger.info("audiocodes.hangup")
+                        self._send_hangup(ws, data)
+                        # the conversation should continue until
+                        # we receive a end message from audiocodes
+                    pass
+                else:
+                    logger.warning("ac_voice.unknown_activity", data=activity)
+        elif data["type"] == "userStream.start":
+            logger.debug("ac_voice.userStream.start", data=data)
+            self._send_recognition_started(ws, data)
+        elif data["type"] == "userStream.chunk":
+            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(data["audioChunk"])
+            return NewAudioAction(audio_bytes)
+        elif data["type"] == "userStream.stop":
+            logger.debug("ac_voice.stop_recognition", data=data)
+            self._send_recognition_ended(ws, data)
+        elif data["type"] == "session.resume":
+            logger.debug("ac_voice.resume", data=data)
+            self._send_accepted(ws, data)
+        elif data["type"] == "session.end":
+            logger.debug("ac_voice.end", data=data)
+            return EndConversationAction()
+        elif data["type"] == "connection.validate":
+            # not part of call flow; only sent when integration is created
+            self._send_validated(ws, data)
+        else:
+            logger.warning("ac_voice.unknown_message", data=data)
+
+        return ContinueConversationAction()
+
+    def _send_accepted(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        supported_formats = data.get("supportedMediaFormats", [])
+        preferred_format = "raw/mulaw"
+
+        if preferred_format not in supported_formats:
+            logger.warning(
+                "ac_voice.format_not_supported",
+                supported_formats=supported_formats,
+                preferred_format=preferred_format,
+            )
+            raise
+
+        payload = {
+            "type": "session.accepted",
+            "mediaFormat": "raw/mulaw",
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_recognition_started(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {"type": "userStream.started"}
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_recognition_ended(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {"type": "userStream.stopped"}
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_hypothesis(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        """
+        TODO: The hypothesis message is sent by the bot to provide partial
+        recognition results. Using this message is recommended,
+        as VAIC relies on it for performing barge-in.
+        """
+        pass
+
+    def _send_recognition(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        """
+        TODO: The recognition message is sent by the bot to provide
+        the final recognition result. Using this message is recommended
+        mainly for logging purposes.
+        """
+        pass
+
+    def _send_hangup(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {
+            "conversationId": data["conversationId"],
+            "type": "activities",
+            "activities": [{"type": "event", "name": "hangup"}],
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def _send_validated(self, ws: Websocket, data: Dict[Text, Any]) -> None:
+        payload = {
+            "type": "connection.validated",
+            "success": True,
+        }
+        _schedule_async_task(ws.send(json.dumps(payload)))
+
+    def create_output_channel(
+        self, voice_websocket: Websocket, tts_engine: TTSEngine
+    ) -> VoiceOutputChannel:
+        return AudiocodesVoiceOutputChannel(
+            voice_websocket,
+            tts_engine,
+            self.tts_cache,
+        )
+
+    def blueprint(
+        self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
+    ) -> Blueprint:
+        """Defines a Sanic bluelogger.debug."""
+        blueprint = Blueprint("ac_voice", __name__)
+
+        @blueprint.route("/", methods=["GET"])
+        async def health(_: Request) -> HTTPResponse:
+            return response.json({"status": "ok"})
+
+        @blueprint.websocket("/websocket")  # type: ignore
+        async def receive(request: Request, ws: Websocket) -> None:
+            # TODO: validate API key header
+            logger.info("audiocodes.receive", message="Starting audio streaming")
+            try:
+                await self.run_audio_streaming(on_new_message, ws)
+            except Exception as e:
+                logger.exception(
+                    "audiocodes.receive",
+                    message="Error during audio streaming",
+                    error=e,
+                )
+                # return 500 error
+                raise
+
+        return blueprint
+
+
+def _schedule_async_task(coro: Awaitable[Any]) -> None:
+    """Helper function to schedule a coroutine in the event loop.
+
+    Args:
+        coro: The coroutine to schedule
+    """
+    loop = asyncio.get_running_loop()
+    loop.call_soon_threadsafe(lambda: loop.create_task(coro))

rasa/core/channels/voice_stream/browser_audio.py

@@ -106,6 +106,9 @@ class BrowserAudioInputChannel(VoiceInputChannel):
 
         @blueprint.websocket("/websocket")  # type: ignore
         async def handle_message(request: Request, ws: Websocket) -> None:
-            await self.run_audio_streaming(on_new_message, ws)
+            try:
+                await self.run_audio_streaming(on_new_message, ws)
+            except Exception as e:
+                logger.error("browser_audio.handle_message.error", error=e)
 
         return blueprint

rasa/core/channels/voice_stream/call_state.py

@@ -25,6 +25,9 @@ class CallState:
     server_sequence_number: int = 0
     audio_buffer: bytearray = field(default_factory=bytearray)
 
+    # Audiocodes requires a stream ID at start and end of stream
+    stream_id: int = 0
+
 
 _call_state: ContextVar[CallState] = ContextVar("call_state")
 call_state = LocalProxy(_call_state)

rasa/core/channels/voice_stream/genesys.py

@@ -104,7 +104,10 @@ class GenesysOutputChannel(VoiceOutputChannel):
                 current_position = end_position
 
     async def send_marker_message(self, recipient_id: str) -> None:
-        """Send a message that marks positions in the audio stream."""
+        """
+        Send a message that marks positions in the audio stream.
+        Genesys does not support this feature, so we do nothing here.
+        """
         pass
 
 
@@ -190,6 +193,8 @@ class GenesysInputChannel(VoiceInputChannel):
                 if call_state.should_hangup:
                     logger.info("genesys.hangup")
                     self.disconnect(ws, data)
+                    # the conversation should continue until
+                    # we receive a close message from Genesys
             elif msg_type == "dtmf":
                 logger.info("genesys.handle_dtmf", message=data)
             elif msg_type == "error":
@@ -259,7 +264,6 @@ class GenesysInputChannel(VoiceInputChannel):
         logger.debug("genesys.handle_close.closed", response=response)
 
         _schedule_ws_task(ws.send(json.dumps(response)))
-        _schedule_ws_task(ws.close())
 
     def disconnect(self, ws: Websocket, data: dict) -> None:
         """

rasa/core/channels/voice_stream/tts/azure.py

@@ -21,6 +21,7 @@ structlogger = structlog.get_logger()
 @dataclass
 class AzureTTSConfig(TTSEngineConfig):
     speech_region: Optional[str] = None
+    endpoint: Optional[str] = None
 
 
 class AzureTTS(TTSEngine[AzureTTSConfig]):
@@ -76,7 +77,13 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
 
     @staticmethod
     def get_tts_endpoint(config: AzureTTSConfig) -> str:
-        return f"https://{config.speech_region}.tts.speech.microsoft.com/cognitiveservices/v1"
+        if config.endpoint is not None:
+            return config.endpoint
+        else:
+            return (
+                f"https://{config.speech_region}.tts.speech.microsoft.com/"
+                f"cognitiveservices/v1"
+            )
 
     @staticmethod
     def create_request_body(text: str, conf: AzureTTSConfig) -> str:
@@ -99,6 +106,7 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
             voice="en-US-JennyNeural",
             timeout=10,
             speech_region="eastus",
+            endpoint=None,
         )
 
     @classmethod

rasa/core/channels/voice_stream/tts/cartesia.py

@@ -24,6 +24,7 @@ structlogger = structlog.get_logger()
 class CartesiaTTSConfig(TTSEngineConfig):
     model_id: Optional[str] = None
     version: Optional[str] = None
+    endpoint: Optional[str] = None
 
 
 class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
@@ -38,11 +39,6 @@ class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
         if self.__class__.session is None or self.__class__.session.closed:
             self.__class__.session = aiohttp.ClientSession(timeout=timeout)
 
-    @staticmethod
-    def get_tts_endpoint() -> str:
-        """Create the endpoint string for cartesia."""
-        return "https://api.cartesia.ai/tts/sse"
-
     @staticmethod
     def get_request_body(text: str, config: CartesiaTTSConfig) -> Dict:
         """Create the request body for cartesia."""
@@ -79,7 +75,7 @@ class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
         config = self.config.merge(config)
         payload = self.get_request_body(text, config)
         headers = self.get_request_headers(config)
-        url = self.get_tts_endpoint()
+        url = self.config.endpoint
         if self.session is None:
             raise ConnectionException("Client session is not initialized")
         try:
@@ -101,13 +97,22 @@ class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
                                     channel_bytes
                                 )
                     return
+                elif response.status == 401:
+                    structlogger.error(
+                        "cartesia.synthesize.rest.unauthorized",
+                        status_code=response.status,
+                    )
+                    raise TTSError(
+                        "Unauthorized. Please make sure you have the correct API key."
+                    )
                 else:
+                    response_text = await response.text()
                     structlogger.error(
                         "cartesia.synthesize.rest.failed",
                         status_code=response.status,
-                        msg=response.text(),
+                        msg=response_text,
                     )
-                    raise TTSError(f"TTS failed: {response.text()}")
+                    raise TTSError(f"TTS failed: {response_text}")
         except ClientConnectorError as e:
             raise TTSError(e)
         except TimeoutError as e:
@@ -125,6 +130,7 @@ class CartesiaTTS(TTSEngine[CartesiaTTSConfig]):
             timeout=10,
             model_id="sonic-english",
             version="2024-06-10",
+            endpoint="https://api.cartesia.ai/tts/sse",
         )
 
     @classmethod

rasa/core/channels/voice_stream/voice_channel.py

@@ -148,6 +148,19 @@ class VoiceOutputChannel(OutputChannel):
         await self.voice_websocket.send(marker_message)
         self.latest_message_id = mark_id
 
+    async def send_start_marker(self, recipient_id: str) -> None:
+        """Send a marker message before the first audio chunk."""
+        # Default implementation uses the generic marker message
+        await self.send_marker_message(recipient_id)
+
+    async def send_intermediate_marker(self, recipient_id: str) -> None:
+        """Send a marker message during audio streaming."""
+        await self.send_marker_message(recipient_id)
+
+    async def send_end_marker(self, recipient_id: str) -> None:
+        """Send a marker message after the last audio chunk."""
+        await self.send_marker_message(recipient_id)
+
     def update_silence_timeout(self) -> None:
         """Updates the silence timeout for the session."""
         if self.tracker_state:
@@ -173,6 +186,13 @@ class VoiceOutputChannel(OutputChannel):
         cached_audio_bytes = self.tts_cache.get(text)
         collected_audio_bytes = RasaAudioBytes(b"")
         seconds_marker = -1
+
+        # Send start marker before first chunk
+        try:
+            await self.send_start_marker(recipient_id)
+        except (WebsocketClosed, ServerError):
+            call_state.connection_failed = True  # type: ignore[attr-defined]
+
         if cached_audio_bytes:
             audio_stream = self.chunk_audio(cached_audio_bytes)
         else:
@@ -189,15 +209,16 @@ class VoiceOutputChannel(OutputChannel):
                 await self.send_audio_bytes(recipient_id, audio_bytes)
                 full_seconds_of_audio = len(collected_audio_bytes) // HERTZ
                 if full_seconds_of_audio > seconds_marker:
-                    await self.send_marker_message(recipient_id)
+                    await self.send_intermediate_marker(recipient_id)
                     seconds_marker = full_seconds_of_audio
 
             except (WebsocketClosed, ServerError):
                 # ignore sending error, and keep collecting and caching audio bytes
                 call_state.connection_failed = True  # type: ignore[attr-defined]
             collected_audio_bytes = RasaAudioBytes(collected_audio_bytes + audio_bytes)
+
         try:
-            await self.send_marker_message(recipient_id)
+            await self.send_end_marker(recipient_id)
         except (WebsocketClosed, ServerError):
             # ignore sending error
             pass

rasa/core/constants.py

@@ -110,3 +110,5 @@ UTTER_SOURCE_METADATA_KEY = "utter_source"
 DOMAIN_GROUND_TRUTH_METADATA_KEY = "domain_ground_truth"
 ACTIVE_FLOW_METADATA_KEY = "active_flow"
 STEP_ID_METADATA_KEY = "step_id"
+KEY_IS_CALM_SYSTEM = "is_calm_system"
+KEY_IS_COEXISTENCE_ASSISTANT = "is_coexistence_assistant"

rasa/core/nlg/contextual_response_rephraser.py

@@ -64,7 +64,7 @@ DEFAULT_LLM_CONFIG = {
 DEFAULT_RESPONSE_VARIATION_PROMPT_TEMPLATE = """The following is a conversation with
 an AI assistant. The assistant is helpful, creative, clever, and very friendly.
 Rephrase the suggested AI response staying close to the original message and retaining
-its meaning. Use simple english.
+its meaning. Use simple {{language}}.
 
 Context / previous conversation with the user:
 {{history}}
@@ -164,6 +164,22 @@ class ContextualResponseRephraser(
         response[PROMPTS] = prompts
         return response
 
+    @staticmethod
+    def get_language_label(tracker: DialogueStateTracker) -> str:
+        """Fetches the label of the language to be used for the rephraser.
+
+        Args:
+            tracker: The tracker to get the language from.
+
+        Returns:
+            The label of the current language, or "English" if no language is set.
+        """
+        return (
+            tracker.current_language.label
+            if tracker.current_language
+            else tracker.default_language.label
+        )
+
     def _last_message_if_human(self, tracker: DialogueStateTracker) -> Optional[str]:
         """Returns the latest message from the tracker.
 
@@ -281,6 +297,7 @@ class ContextualResponseRephraser(
             suggested_response=response_text,
             current_input=current_input,
             slots=tracker.current_slot_values(),
+            language=self.get_language_label(tracker),
         )
         log_llm(
             logger=structlogger,

rasa/core/nlg/generator.py

@@ -1,6 +1,9 @@
-import logging
 from typing import Any, Dict, List, Optional, Text, Union
 
+import structlog
+from jinja2 import Template
+from pypred import Predicate
+
 import rasa.shared.utils.common
 import rasa.shared.utils.io
 from rasa.shared.constants import CHANNEL, RESPONSE_CONDITION
@@ -8,7 +11,7 @@ from rasa.shared.core.domain import Domain
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.utils.endpoints import EndpointConfig
 
-logger = logging.getLogger(__name__)
+structlogger = structlog.get_logger()
 
 
 class NaturalLanguageGenerator:
@@ -74,7 +77,11 @@ def _create_from_endpoint_config(
     else:
         nlg = _load_from_module_name_in_endpoint_config(endpoint_config, domain)
 
-    logger.debug(f"Instantiated NLG to '{nlg.__class__.__name__}'.")
+    structlogger.debug(
+        "rasa.core.nlg.generator.create",
+        nlg_class_name=nlg.__class__.__name__,
+        event_info=f"Instantiated NLG to '{nlg.__class__.__name__}'.",
+    )
     return nlg
 
 
@@ -112,18 +119,15 @@ class ResponseVariationFilter:
     ) -> bool:
         """Checks if the conditional response variation matches the filled slots."""
         constraints = response.get(RESPONSE_CONDITION, [])
-        for constraint in constraints:
-            name = constraint["name"]
-            value = constraint["value"]
-            filled_slots_value = filled_slots.get(name)
-            if isinstance(filled_slots_value, str) and isinstance(value, str):
-                if filled_slots_value.casefold() != value.casefold():
+        if isinstance(constraints, str) and not _evaluate_predicate(
+            constraints, filled_slots
+        ):
+            return False
+
+        elif isinstance(constraints, list):
+            for constraint in constraints:
+                if not _evaluate_and_deprecate_condition(constraint, filled_slots):
                     return False
-            # slot values can be of different data types
-            # such as int, float, bool, etc. hence, this check
-            # executes when slot values are not strings
-            elif filled_slots_value != value:
-                return False
 
         return True
 
@@ -180,7 +184,21 @@ class ResponseVariationFilter:
         if conditional_no_channel:
             return conditional_no_channel
 
-        return default_no_channel
+        if default_no_channel:
+            return default_no_channel
+
+        # if there is no response variation selected,
+        # return the internal error response to prevent
+        # the bot from staying silent
+        structlogger.error(
+            "rasa.core.nlg.generator.responses_for_utter_action.no_response",
+            utter_action=utter_action,
+            event_info=f"No response variation selected for the predicted "
+            f"utterance {utter_action}. Please check you have provided "
+            f"a default variation and that all the conditions are valid. "
+            f"Returning the internal error response.",
+        )
+        return self.responses.get("utter_internal_error_rasa", [])
 
     def get_response_variation_id(
         self,
@@ -228,3 +246,53 @@ class ResponseVariationFilter:
             response_ids.add(response_variation_id)
 
         return True
+
+
+def _evaluate_and_deprecate_condition(
+    constraint: Dict[Text, Any], filled_slots: Dict[Text, Any]
+) -> bool:
+    """Evaluates the condition of a response variation."""
+    rasa.shared.utils.io.raise_deprecation_warning(
+        "Using a dictionary as a condition in a response variation is deprecated. "
+        "Please use a pypred string predicate instead. "
+        "Dictionary conditions will be removed in Rasa Open Source 4.0.0 .",
+        warn_until_version="4.0.0",
+    )
+
+    name = constraint["name"]
+    value = constraint["value"]
+    filled_slots_value = filled_slots.get(name)
+    if isinstance(filled_slots_value, str) and isinstance(value, str):
+        if filled_slots_value.casefold() != value.casefold():
+            return False
+    # slot values can be of different data types
+    # such as int, float, bool, etc. hence, this check
+    # executes when slot values are not strings
+    elif filled_slots_value != value:
+        return False
+
+    return True
+
+
+def _evaluate_predicate(constraint: str, filled_slots: Dict[Text, Any]) -> bool:
+    """Evaluates the condition of a response variation."""
+    context = {"slots": filled_slots}
+    document = context.copy()
+    try:
+        rendered_template = Template(constraint).render(context)
+        predicate = Predicate(rendered_template)
+        result = predicate.evaluate(document)
+        structlogger.debug(
+            "rasa.core.nlg.generator.evaluate_conditional_response_predicate",
+            predicate=predicate.description(),
+            result=result,
+        )
+        return result
+    except (TypeError, Exception) as e:
+        structlogger.error(
+            "rasa.core.nlg.generator.evaluate_conditional_response_predicate.error",
+            predicate=constraint,
+            document=document,
+            error=str(e),
+        )
+        return False

rasa/core/nlg/response.py

@@ -49,9 +49,12 @@ class TemplatedNaturalLanguageGenerator(NaturalLanguageGenerator):
                 selected_response = np.random.choice(suitable_responses)
                 condition = selected_response.get(RESPONSE_CONDITION)
                 if condition:
-                    formatted_response_conditions = self._format_response_conditions(
-                        condition
-                    )
+                    if isinstance(condition, list):
+                        formatted_response_conditions = (
+                            self._format_response_conditions(condition)
+                        )
+                    else:
+                        formatted_response_conditions = condition
                     logger.debug(
                         "Selecting response variation with conditions:"
                         f"{formatted_response_conditions}"