rasa-pro 3.14.0a16__py3-none-any.whl → 3.14.0a18__py3-none-any.whl

rasa/core/channels/voice_stream/asr/asr_event.py

@@ -15,7 +15,7 @@ class NewTranscript(ASREvent):
 
 @dataclass
 class UserIsSpeaking(ASREvent):
-    pass
+    text: str
 
 
 @dataclass

rasa/core/channels/voice_stream/asr/azure.py

@@ -1,7 +1,7 @@
 import asyncio
 import os
 from dataclasses import dataclass
-from typing import Any, AsyncIterator, Dict, Optional
+from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Optional
 
 import structlog
 
@@ -15,6 +15,9 @@ from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
 from rasa.shared.constants import AZURE_SPEECH_API_KEY_ENV_VAR
 from rasa.shared.exceptions import ConnectionException
 
+if TYPE_CHECKING:
+    from azure.cognitiveservices.speech import SpeechRecognitionEventArgs
+
 logger = structlog.get_logger(__name__)
 
 
@@ -43,9 +46,9 @@ class AzureASR(ASREngine[AzureASRConfig]):
         )
         self.main_loop = asyncio.get_running_loop()
 
-    def signal_user_is_speaking(self, event: Any) -> None:
+    def signal_user_is_speaking(self, event: "SpeechRecognitionEventArgs") -> None:
         """Replace the azure event with a generic is speaking event."""
-        self.fill_queue(UserIsSpeaking())
+        self.fill_queue(UserIsSpeaking(event.result.text))
 
     def fill_queue(self, event: Any) -> None:
         """Either puts the event or a dedicated ASR Event into the queue."""

rasa/core/channels/voice_stream/asr/deepgram.py

@@ -117,7 +117,7 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
                         self.accumulated_transcript, transcript
                     )
             elif transcript:
-                return UserIsSpeaking()
+                return UserIsSpeaking(transcript)
         # event that comes after utterance_end_ms of no new transcript
         elif data_type == "UtteranceEnd":
             if self.accumulated_transcript:

rasa/core/channels/voice_stream/audiocodes.py

@@ -89,6 +89,7 @@ class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
         # This is an approximation, as the bot will be sent the audio chunks next
         # which are played to the user immediately.
         call_state.is_bot_speaking = True
+        VoiceInputChannel._cancel_silence_timeout_watcher()
 
     async def send_intermediate_marker(self, recipient_id: str) -> None:
         """Audiocodes doesn't need intermediate markers, so do nothing."""
@@ -116,6 +117,7 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
         server_url: str,
         asr_config: Dict,
         tts_config: Dict,
+        interruptions: Optional[Dict[str, int]] = None,
         token: Optional[Text] = None,
     ):
         mark_as_beta_feature("Audiocodes (audiocodes_stream) Channel")
@@ -123,6 +125,7 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
             server_url=server_url,
             asr_config=asr_config,
             tts_config=tts_config,
+            interruptions=interruptions,
         )
         self.token = token
 

rasa/core/channels/voice_stream/browser_audio.py

@@ -3,7 +3,9 @@ from __future__ import annotations
 import audioop
 import base64
 import json
+import os
 import uuid
+import wave
 from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
 
 import structlog
@@ -70,10 +72,45 @@ class BrowserAudioOutputChannel(VoiceOutputChannel):
 
 class BrowserAudioInputChannel(VoiceInputChannel):
     def __init__(
-        self, server_url: str, asr_config: Dict[str, Any], tts_config: Dict[str, Any]
+        self,
+        server_url: str,
+        asr_config: Dict[str, Any],
+        tts_config: Dict[str, Any],
+        recording: bool = False,
+        interruptions: Optional[Dict[str, int]] = None,
     ) -> None:
         """Initializes the browser audio input channel."""
-        super().__init__(server_url, asr_config, tts_config)
+        super().__init__(server_url, asr_config, tts_config, interruptions)
+
+        # For debugging, recording of user audio might be useful
+        # to identify audio quality issues or transcription errors
+        self._recording_enabled = recording
+        self._wav_file: Optional[wave.Wave_write] = None
+
+    def _start_recording(self, call_id: str, user_id: str) -> None:
+        os.makedirs("recordings", exist_ok=True)
+        filename = f"{user_id}_{call_id}.wav"
+        file_path = os.path.join("recordings", filename)
+
+        if not self._recording_enabled:
+            return
+
+        self._wav_file = wave.open(file_path, "wb")
+        self._wav_file.setnchannels(1)  # Mono audio
+        self._wav_file.setsampwidth(4)  # 32-bit audio (4 bytes)
+        self._wav_file.setframerate(8000)  # 8kHz sample rate
+        logger.info("voice_channel.user_audio_recording.started", file_path=file_path)
+
+    def _append_audio_to_recording(self, audio_bytes: bytes) -> None:
+        if self._wav_file and self._recording_enabled:
+            self._wav_file.writeframes(audio_bytes)
+
+    def _stop_recording(self) -> None:
+        """Close the recording file if it's open."""
+        if self._wav_file:
+            self._wav_file.close()
+            self._wav_file = None
+            logger.debug("voice_channel.user_audio_recording.stopped")
 
     @classmethod
     def name(cls) -> str:
@@ -94,7 +131,7 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         credentials: Optional[Dict[str, Any]],
     ) -> BrowserAudioInputChannel:
         cls.validate_basic_credentials(credentials)
-        new_creds = repack_voice_credentials(credentials)
+        new_creds = repack_voice_credentials(credentials or {})
         return cls(**new_creds)
 
     def map_input_message(
@@ -105,6 +142,7 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         data = json.loads(message)
         if "audio" in data:
             channel_bytes = base64.b64decode(data["audio"])
+            self._append_audio_to_recording(channel_bytes)
             audio_bytes = self.channel_bytes_to_rasa_audio_bytes(channel_bytes)
             return NewAudioAction(audio_bytes)
         elif "marker" in data:
@@ -120,6 +158,13 @@ class BrowserAudioInputChannel(VoiceInputChannel):
                 call_state.is_bot_speaking = True
         return ContinueConversationAction()
 
+    async def interrupt_playback(
+        self, ws: Websocket, call_parameters: CallParameters
+    ) -> None:
+        """Interrupt the current playback of audio."""
+        logger.debug("browser_audio.interrupt_playback")
+        await ws.send(json.dumps({"interruptPlayback": True}))
+
     def create_output_channel(
         self, voice_websocket: Websocket, tts_engine: TTSEngine
     ) -> VoiceOutputChannel:
@@ -142,8 +187,13 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         @blueprint.websocket("/websocket")  # type: ignore
         async def handle_message(request: Request, ws: Websocket) -> None:
             try:
+                call_parameters = await self.collect_call_parameters(ws)
+                if call_parameters and call_parameters.call_id:
+                    self._start_recording(call_parameters.call_id, "local")
                 await self.run_audio_streaming(on_new_message, ws)
             except Exception as e:
                 logger.error("browser_audio.handle_message.error", error=e)
+            finally:
+                self._stop_recording()
 
         return blueprint

rasa/core/channels/voice_stream/genesys.py

@@ -99,10 +99,11 @@ class GenesysInputChannel(VoiceInputChannel):
         server_url: str,
         asr_config: Dict,
         tts_config: Dict,
+        interruptions: Optional[Dict[str, int]] = None,
         api_key: Optional[Text] = None,
         client_secret: Optional[Text] = None,
     ) -> None:
-        super().__init__(server_url, asr_config, tts_config)
+        super().__init__(server_url, asr_config, tts_config, interruptions)
         self.api_key = api_key
         self.client_secret = client_secret
 

rasa/core/channels/voice_stream/jambonz.py

@@ -81,6 +81,7 @@ class JambonzStreamInputChannel(VoiceInputChannel):
         server_url: str,
         asr_config: Dict,
         tts_config: Dict,
+        interruptions: Optional[Dict[str, int]] = None,
         username: Optional[Text] = None,
         password: Optional[Text] = None,
     ) -> None:
@@ -90,7 +91,7 @@ class JambonzStreamInputChannel(VoiceInputChannel):
             username: Optional username for basic auth
             password: Optional password for basic auth
         """
-        super().__init__(server_url, asr_config, tts_config)
+        super().__init__(server_url, asr_config, tts_config, interruptions)
         self.username = username
         self.password = password
 
@@ -185,6 +186,13 @@ class JambonzStreamInputChannel(VoiceInputChannel):
             self.tts_cache,
         )
 
+    async def interrupt_playback(
+        self, ws: Websocket, call_parameters: CallParameters
+    ) -> None:
+        """Interrupt the current playback of audio."""
+        logger.debug("jambonz.interrupt_playback")
+        await ws.send(json.dumps({"type": "killAudio"}))
+
     def blueprint(
         self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
     ) -> Blueprint:

rasa/core/channels/voice_stream/twilio_media_streams.py

@@ -105,6 +105,7 @@ class TwilioMediaStreamsInputChannel(VoiceInputChannel):
         server_url: str,
         asr_config: Dict,
         tts_config: Dict,
+        interruptions: Optional[Dict[str, int]] = None,
         username: Optional[Text] = None,
         password: Optional[Text] = None,
     ):
@@ -112,6 +113,7 @@ class TwilioMediaStreamsInputChannel(VoiceInputChannel):
             server_url=server_url,
             asr_config=asr_config,
             tts_config=tts_config,
+            interruptions=interruptions,
         )
         self.username = username
         self.password = password
@@ -195,6 +197,20 @@ class TwilioMediaStreamsInputChannel(VoiceInputChannel):
             self.tts_cache,
         )
 
+    async def interrupt_playback(
+        self, ws: Websocket, call_parameters: CallParameters
+    ) -> None:
+        """Interrupt the current playback of audio."""
+        logger.debug("twilio_media_streams.interrupt_playback")
+        await ws.send(
+            json.dumps(
+                {
+                    "event": "clear",
+                    "streamSid": call_parameters.stream_id,
+                }
+            )
+        )
+
     def blueprint(
         self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
     ) -> Blueprint:

rasa/core/channels/voice_stream/voice_channel.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import copy
+import string
 import time
 from dataclasses import asdict, dataclass
 from typing import Any, AsyncIterator, Awaitable, Callable, Dict, List, Optional, Tuple
@@ -53,6 +54,13 @@ from rasa.utils.io import remove_emojis
 logger = structlog.get_logger(__name__)
 
 # define constants for the voice channel
+DEFAULT_INTERRUPTION_MIN_WORDS = 3
+
+
+@dataclass
+class InterruptionConfig:
+    enabled: bool = False
+    min_words: int = DEFAULT_INTERRUPTION_MIN_WORDS
 
 
 @dataclass
@@ -270,6 +278,10 @@ class VoiceOutputChannel(OutputChannel):
         except (WebsocketClosed, ServerError):
             call_state.connection_failed = True
 
+        # Is the response interruptible?
+        allow_interruptions = kwargs.get("allow_interruptions", True)
+        call_state.channel_data["allow_interruptions"] = allow_interruptions
+
         if cached_audio_bytes:
             audio_stream = self.chunk_audio(cached_audio_bytes)
         else:
@@ -368,6 +380,7 @@ class VoiceInputChannel(InputChannel):
         server_url: str,
         asr_config: Dict,
         tts_config: Dict,
+        interruptions: Optional[Dict[str, Any]] = None,
     ):
         if self.requires_voice_license:
             validate_voice_license_scope()
@@ -376,12 +389,21 @@ class VoiceInputChannel(InputChannel):
         self.asr_config = asr_config
         self.tts_config = tts_config
         self.tts_cache = TTSCache(tts_config.get("cache_size", 1000))
+        if interruptions:
+            self.interruption_config = InterruptionConfig(**interruptions)
+        else:
+            self.interruption_config = InterruptionConfig()
+
+        if self.interruption_config.enabled:
+            mark_as_beta_feature(f"Interruption Handling in {self.name()}")
 
         logger.info(
             "voice_channel.initialized",
+            name=self.name(),
             server_url=self.server_url,
             asr_config=self.asr_config,
             tts_config=self.tts_config,
+            interruption_config=self.interruption_config,
         )
 
     def get_sender_id(self, call_parameters: CallParameters) -> str:
@@ -463,6 +485,43 @@ class VoiceInputChannel(InputChannel):
         """Map a channel input message to a voice channel action."""
         raise NotImplementedError
 
+    def should_interrupt(self, e: ASREvent) -> bool:
+        """Determine if the current ASR event should interrupt playback.
+        Returns True if the bot response is interruptible
+        And if the user spoke more than 3 words.
+
+        Arguments:
+            e: The ASR event to evaluate.
+
+        Returns:
+            True if the event should interrupt playback, False otherwise.
+        """
+        # Are interruptions are enabled for the channel?
+        if not self.interruption_config.enabled:
+            return False
+
+        # Is the bot response interruptible?
+        if not call_state.channel_data.get("allow_interruptions", True):
+            return False
+
+        # Did the user speak more than 3 words?
+        min_words = self.interruption_config.min_words
+        if isinstance(e, UserIsSpeaking):
+            translator = str.maketrans("", "", string.punctuation)
+            words = e.text.translate(translator).split()
+            return len(words) >= min_words
+        return False
+
+    async def interrupt_playback(
+        self, ws: Websocket, call_parameters: CallParameters
+    ) -> None:
+        """Interrupt the current playback of audio.
+
+        This function is used for interruption handling.
+        As not all channels support flushing bot audio buffer,
+        if a channel does not implement it. It has no effect."""
+        pass
+
     async def run_audio_streaming(
         self,
         on_new_message: Callable[[UserMessage], Awaitable[Any]],
@@ -598,6 +657,8 @@ class VoiceInputChannel(InputChannel):
                 call_state.user_speech_start_time = time.time()
             self._cancel_silence_timeout_watcher()
             call_state.is_user_speaking = True
+            if self.should_interrupt(e):
+                await self.interrupt_playback(voice_websocket, call_parameters)
         elif isinstance(e, UserSilence):
             output_channel = self.create_output_channel(voice_websocket, tts_engine)
             message = UserMessage(

rasa/core/concurrent_lock_store.py

@@ -4,6 +4,7 @@ from collections import deque
 from typing import Deque, Optional, Text
 
 import structlog
+from pydantic import ValidationError
 
 from rasa.core.lock import Ticket, TicketLock
 from rasa.core.lock_store import (
@@ -12,6 +13,12 @@ from rasa.core.lock_store import (
     LockError,
     LockStore,
 )
+from rasa.core.redis_connection_factory import (
+    DeploymentMode,
+    RedisConfig,
+    RedisConnectionFactory,
+)
+from rasa.shared.exceptions import RasaException
 from rasa.utils.endpoints import EndpointConfig
 
 DEFAULT_REDIS_DB = 1
@@ -74,9 +81,10 @@ class ConcurrentRedisLockStore(LockStore):
                 alphanumeric.
             socket_timeout - Timeout in seconds after which an exception will be raised
                 in case Redis doesn't respond within `socket_timeout` seconds.
+            deployment_mode - Redis deployment mode: standard, cluster, or sentinel.
+            endpoints - List of endpoints for cluster/sentinel mode in host:port format.
+            sentinel_service - Sentinel service name.
         """
-        import redis
-
         host = endpoint_config.kwargs.get("host", DEFAULT_HOSTNAME)
         port = endpoint_config.kwargs.get("port", DEFAULT_PORT)
         db = endpoint_config.kwargs.get("db", DEFAULT_REDIS_DB)
@@ -90,20 +98,33 @@ class ConcurrentRedisLockStore(LockStore):
         socket_timeout = endpoint_config.kwargs.get(
             "socket_timeout", DEFAULT_SOCKET_TIMEOUT_IN_SECONDS
         )
-
-        self.red = redis.StrictRedis(
-            host=host,
-            port=int(port),
-            db=int(db),
-            username=username,
-            password=password,
-            ssl=use_ssl,
-            ssl_certfile=ssl_certfile,
-            ssl_keyfile=ssl_keyfile,
-            ssl_ca_certs=ssl_ca_certs,
-            socket_timeout=socket_timeout,
+        deployment_mode = endpoint_config.kwargs.get(
+            "deployment_mode", DeploymentMode.STANDARD.value
         )
+        endpoints = endpoint_config.kwargs.get("endpoints", [])
+        sentinel_service = endpoint_config.kwargs.get("sentinel_service")
 
+        try:
+            redis_config = RedisConfig(
+                host=host,
+                port=port,
+                db=db,
+                username=username,
+                password=password,
+                use_ssl=use_ssl,
+                ssl_certfile=ssl_certfile,
+                ssl_keyfile=ssl_keyfile,
+                ssl_ca_certs=ssl_ca_certs,
+                socket_timeout=socket_timeout,
+                deployment_mode=deployment_mode,
+                endpoints=endpoints,
+                sentinel_service=sentinel_service,
+            )
+            self.red = RedisConnectionFactory.create_connection(redis_config)
+        except ValidationError as e:
+            raise RasaException(f"Invalid Redis configuration: {e}")
+
+        self.deployment_mode = deployment_mode
         self.key_prefix = DEFAULT_CONCURRENT_REDIS_LOCK_STORE_KEY_PREFIX
         if key_prefix:
             structlogger.debug(
@@ -129,6 +150,32 @@ class ConcurrentRedisLockStore(LockStore):
                 ),
             )
 
+    def _get_keys_by_pattern(self, pattern: Text) -> list:
+        """Get keys by pattern, using SCAN for cluster mode and KEYS for others."""
+        if self.deployment_mode == DeploymentMode.CLUSTER.value:
+            # In cluster mode, use SCAN to get keys more reliably
+            keys = []
+            cursor = 0
+
+            while True:
+                try:
+                    cursor, batch_keys = self.red.scan(cursor, match=pattern, count=100)
+                    keys.extend(batch_keys)
+                    if cursor == 0:
+                        break
+                except Exception as e:
+                    structlogger.warning(
+                        "concurrent_redis_lock_store._get_keys_by_pattern.scan_interrupted",
+                        event_info=f"SCAN interrupted in cluster mode: {e}. "
+                        f"Returning {len(keys)} keys found so far.",
+                    )
+                    break
+        else:
+            # Standard and sentinel modes use KEYS
+            keys = self.red.keys(pattern)
+
+        return keys
+
     def issue_ticket(
         self, conversation_id: Text, lock_lifetime: float = LOCK_LIFETIME
     ) -> int:
@@ -157,11 +204,14 @@ class ConcurrentRedisLockStore(LockStore):
         tickets: Deque[Ticket] = deque()
 
         pattern = self.key_prefix + conversation_id + ":" + "[0-9]*"
-        redis_keys = self.red.keys(pattern)
+        redis_keys = self._get_keys_by_pattern(pattern)
 
         for key in redis_keys:
             serialised_ticket = self.red.get(key)
             if serialised_ticket:
+                # Handle bytes to string conversion for JSON parsing
+                if isinstance(serialised_ticket, bytes):
+                    serialised_ticket = serialised_ticket.decode("utf-8")
                 ticket = Ticket.from_dict(json.loads(serialised_ticket))
                 tickets.appendleft(ticket)
 
@@ -172,7 +222,7 @@ class ConcurrentRedisLockStore(LockStore):
     def delete_lock(self, conversation_id: Text) -> None:
         """Deletes lock for conversation ID."""
         pattern = self.key_prefix + conversation_id + ":*"
-        redis_keys = self.red.keys(pattern)
+        redis_keys = self._get_keys_by_pattern(pattern)
 
         if not redis_keys:
             structlogger.debug(

rasa/core/iam_credentials_providers/aws_iam_credentials_providers.py

@@ -1,7 +1,11 @@
+import os
+import threading
+import time
 from typing import Optional
 
 import boto3
 import structlog
+from aws_msk_iam_sasl_signer import MSKAuthTokenProvider
 from botocore.exceptions import BotoCoreError
 
 from rasa.core.iam_credentials_providers.credentials_provider_protocol import (
@@ -10,6 +14,7 @@ from rasa.core.iam_credentials_providers.credentials_provider_protocol import (
     SupportedServiceType,
     TemporaryCredentials,
 )
+from rasa.shared.exceptions import ConnectionException
 
 structlogger = structlog.get_logger(__name__)
 
@@ -17,7 +22,7 @@ structlogger = structlog.get_logger(__name__)
 class AWSRDSIAMCredentialsProvider(IAMCredentialsProvider):
     """Generates temporary credentials for AWS RDS using IAM roles."""
 
-    def __init__(self, username: str, host: str, port: int):
+    def __init__(self, username: str, host: str, port: int) -> None:
         """Initializes the provider."""
         self.username = username
         self.host = host
@@ -52,6 +57,73 @@ class AWSRDSIAMCredentialsProvider(IAMCredentialsProvider):
             return TemporaryCredentials(auth_token=None)
 
 
+class AWSMSKafkaIAMCredentialsProvider(IAMCredentialsProvider):
+    """Generates temporary credentials for AWS MSK using IAM roles."""
+
+    def __init__(self) -> None:
+        self.region = os.getenv("AWS_DEFAULT_REGION", os.getenv("AWS_REGION"))
+        self._token: Optional[str] = None
+        self._expires_at: float = 0
+        self.refresh_margin_seconds = 60  # Refresh 60 seconds before expiry
+        # ensure thread safety when refreshing token because the
+        # kafka client library we use (confluent-kafka) is multithreaded
+        self.lock = threading.Lock()
+
+    @property
+    def token(self) -> Optional[str]:
+        return self._token
+
+    @token.setter
+    def token(self, value: str) -> None:
+        self._token = value
+
+    @property
+    def expires_at(self) -> float:
+        return self._expires_at
+
+    @expires_at.setter
+    def expires_at(self, value: float) -> None:
+        self._expires_at = value
+
+    def get_credentials(self) -> TemporaryCredentials:
+        """Generates temporary credentials for AWS MSK."""
+        with self.lock:
+            current_time = time.time()  # Current time in seconds
+            if (
+                not self.token
+                or current_time >= self.expires_at - self.refresh_margin_seconds
+            ):
+                try:
+                    auth_token, expiry_ms = MSKAuthTokenProvider.generate_auth_token(
+                        self.region
+                    )
+                    structlogger.debug(
+                        "rasa.core.aws_msk_iam_credentials_provider.get_credentials",
+                        event_info="Successfully generated AWS IAM token for "
+                        "Kafka authentication.",
+                    )
+                    self.token = auth_token
+                    self.expires_at = int(expiry_ms) / 1000  # Convert ms to seconds
+                    return TemporaryCredentials(
+                        auth_token=auth_token,
+                        expiration=self.expires_at,
+                    )
+                except Exception as exc:
+                    raise ConnectionException(
+                        f"Failed to generate AWS IAM token "
+                        f"for MSK authentication. Original exception: {exc}"
+                    ) from exc
+            else:
+                structlogger.debug(
+                    "rasa.core.aws_msk_iam_credentials_provider.get_credentials",
+                    event_info="Using cached AWS IAM token for Kafka authentication.",
+                )
+                return TemporaryCredentials(
+                    auth_token=self.token,
+                    expiration=self.expires_at,
+                )
+
+
 def create_aws_iam_credentials_provider(
     provider_input: "IAMCredentialsProviderInput",
 ) -> Optional["IAMCredentialsProvider"]:
@@ -63,4 +135,7 @@ def create_aws_iam_credentials_provider(
             port=provider_input.port,
         )
 
+    if provider_input.service_name == SupportedServiceType.EVENT_BROKER:
+        return AWSMSKafkaIAMCredentialsProvider()
+
     return None

rasa/core/iam_credentials_providers/credentials_provider_protocol.py

@@ -16,7 +16,7 @@ class TemporaryCredentials(BaseModel):
     """Dataclass storing temporary credentials."""
 
     auth_token: Optional[str] = None
-    expiration: Optional[int] = None
+    expiration: Optional[float] = None
     username: Optional[str] = None
     presigned_url: Optional[str] = None