rasa-pro 3.13.12__py3-none-any.whl → 3.14.0__py3-none-any.whl

rasa/core/channels/studio_chat.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import audioop
 import base64
@@ -14,6 +16,7 @@ from typing import (
     Optional,
     Text,
     Tuple,
+    Union,
 )
 
 import structlog
@@ -43,14 +46,13 @@ if TYPE_CHECKING:
     from sanic import Sanic, Websocket  # type: ignore[attr-defined]
     from socketio import AsyncServer
 
-    from rasa.core.channels.channel import InputChannel, UserMessage
     from rasa.shared.core.trackers import DialogueStateTracker
 
 
 structlogger = structlog.get_logger()
 
 
-def tracker_as_dump(tracker: "DialogueStateTracker") -> str:
+def tracker_as_dump(tracker: "DialogueStateTracker") -> Dict[str, Any]:
     """Create a dump of the tracker state."""
     from rasa.shared.core.trackers import get_trackers_for_conversation_sessions
 
@@ -62,7 +64,7 @@ def tracker_as_dump(tracker: "DialogueStateTracker") -> str:
         last_tracker = multiple_tracker_sessions[-1]
 
     state = last_tracker.current_state(EventVerbosity.AFTER_RESTART)
-    return json.dumps(state)
+    return state
 
 
 def does_need_action_prediction(tracker: "DialogueStateTracker") -> bool:
@@ -91,12 +93,12 @@ class StudioTrackerUpdatePlugin:
         """Remove tasks that have already completed."""
         self.tasks = [task for task in self.tasks if not task.done()]
 
-    @hookimpl  # type: ignore[misc]
+    @hookimpl
     def after_new_user_message(self, tracker: "DialogueStateTracker") -> None:
         """Triggers a tracker update notification after a new user message."""
         self.handle_tracker_update(tracker)
 
-    @hookimpl  # type: ignore[misc]
+    @hookimpl
     def after_action_executed(self, tracker: "DialogueStateTracker") -> None:
         """Triggers a tracker update notification after an action is executed."""
         self.handle_tracker_update(tracker)
@@ -116,7 +118,7 @@ class StudioTrackerUpdatePlugin:
         self.tasks.append(task)
         self._cleanup_tasks()
 
-    @hookimpl  # type: ignore[misc]
+    @hookimpl
     def after_server_stop(self) -> None:
         """Cancels all remaining tasks when the server stops."""
         self._cancel_tasks()
@@ -144,6 +146,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         jwt_key: Optional[Text] = None,
         jwt_method: Optional[Text] = "HS256",
         metadata_key: Optional[Text] = "metadata",
+        enable_silence_timeout: bool = False,
     ) -> None:
         """Creates a `StudioChatInput` object."""
         from rasa.core.agent import Agent
@@ -161,6 +164,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
             jwt_key=jwt_key,
             jwt_method=jwt_method,
             metadata_key=metadata_key,
+            enable_silence_timeout=enable_silence_timeout,
         )
 
         # Initialize the Voice Input Channel
@@ -176,11 +180,14 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         # `background_tasks` holds the asyncio tasks for voice streaming
         self.active_connections: Dict[str, SocketIOVoiceWebsocketAdapter] = {}
         self.background_tasks: Dict[str, asyncio.Task] = {}
+        self._turn_start_times: Dict[Text, float] = {}
 
         self._register_tracker_update_hook()
 
     @classmethod
-    def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> "InputChannel":
+    def from_credentials(
+        cls, credentials: Optional[Dict[Text, Any]]
+    ) -> "StudioChatInput":
         """Creates a StudioChatInput channel from credentials."""
         credentials = credentials or {}
 
@@ -198,14 +205,15 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
             jwt_key=credentials.get("jwt_key"),
             jwt_method=credentials.get("jwt_method", "HS256"),
             metadata_key=credentials.get("metadata_key", "metadata"),
+            enable_silence_timeout=credentials.get("enable_silence_timeout", False),
         )
 
-    async def emit(self, event: str, data: Dict, room: str) -> None:
+    async def emit(self, event: str, data: Union[Dict, str], room: str) -> None:
         """Emits an event to the websocket."""
-        if not self.sio:
+        if not self.sio_server:
             structlogger.error("studio_chat.emit.sio_not_initialized")
             return
-        await self.sio.emit(event, data, room=room)
+        await self.sio_server.emit(event, data, room=room)
 
     def _register_tracker_update_hook(self) -> None:
         plugin_manager().register(StudioTrackerUpdatePlugin(self))
@@ -214,20 +222,29 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         """Triggers a tracker update notification after a change to the tracker."""
         await self.publish_tracker_update(tracker.sender_id, tracker_as_dump(tracker))
 
-    async def publish_tracker_update(self, sender_id: str, tracker_dump: Dict) -> None:
+    async def publish_tracker_update(
+        self, sender_id: str, tracker_dump: Dict[str, Any]
+    ) -> None:
         """Publishes a tracker update notification to the websocket."""
         await self.emit("tracker", tracker_dump, room=sender_id)
 
     async def on_message_proxy(
         self,
-        on_new_message: Callable[["UserMessage"], Awaitable[Any]],
-        message: "UserMessage",
+        on_new_message: Callable[[UserMessage], Awaitable[Any]],
+        message: UserMessage,
     ) -> None:
         """Proxies the on_new_message call to the underlying channel.
 
         Triggers a tracker update notification after processing the message.
         """
-        await on_new_message(message)
+        try:
+            await on_new_message(message)
+        except Exception as e:
+            structlogger.exception(
+                "studio_chat.on_new_message.error",
+                error=str(e),
+                sender_id=message.sender_id,
+            )
 
         if not self.agent or not self.agent.is_ready():
             structlogger.error("studio_chat.on_message_proxy.agent_not_initialized")
@@ -335,26 +352,33 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         elif "marker" in message:
             if message["marker"] == call_state.latest_bot_audio_id:
                 # Just finished streaming last audio bytes
-                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
+                call_state.is_bot_speaking = False
                 if call_state.should_hangup:
                     structlogger.debug(
                         "studio_chat.hangup", marker=call_state.latest_bot_audio_id
                     )
                     return EndConversationAction()
             else:
-                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+                call_state.is_bot_speaking = True
         return ContinueConversationAction()
 
     def create_output_channel(
         self, voice_websocket: "Websocket", tts_engine: TTSEngine
     ) -> VoiceOutputChannel:
-        """Create a voice output channel."""
+        """Create a voice output channel. This is used by VoiceInputChannel."""
         return StudioVoiceOutputChannel(
             voice_websocket,
             tts_engine,
             self.tts_cache,
         )
 
+    async def interrupt_playback(
+        self, ws: Websocket, call_parameters: CallParameters
+    ) -> None:
+        """Interrupt the current playback of audio."""
+        structlogger.debug("studio_chat.interrupt_playback")
+        await ws.send(json.dumps({"interruptPlayback": True}))
+
     def _start_voice_session(
         self,
         session_id: str,
@@ -375,7 +399,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
 
         # Create a websocket adapter for this connection
         ws_adapter = SocketIOVoiceWebsocketAdapter(
-            sio=self.sio,
+            sio_server=self.sio_server,
             session_id=session_id,
             sid=sid,
             bot_message_evt=self.bot_message_evt,
@@ -414,7 +438,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         if sid in self.active_connections:
             del self.active_connections[sid]
 
-    @hookimpl  # type: ignore[misc]
+    @hookimpl
     def after_server_stop(self) -> None:
         """Cleanup background tasks and active connections when the server stops."""
         structlogger.info("studio_chat.after_server_stop.cleanup")
@@ -423,13 +447,13 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
             task.cancel()
 
     def blueprint(
-        self, on_new_message: Callable[["UserMessage"], Awaitable[Any]]
+        self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
     ) -> SocketBlueprint:
         socket_blueprint = super().blueprint(
             partial(self.on_message_proxy, on_new_message)
         )
 
-        if not self.sio:
+        if not self.sio_server:
             structlogger.error("studio_chat.blueprint.sio_not_initialized")
             return socket_blueprint
 
@@ -437,14 +461,15 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         async def after_server_start(
             app: "Sanic", _: asyncio.AbstractEventLoop
         ) -> None:
-            self.agent = app.ctx.agent
+            if hasattr(app.ctx, "agent"):
+                self.agent = app.ctx.agent
 
-        @self.sio.on("disconnect", namespace=self.namespace)
+        @self.sio_server.on("disconnect", namespace=self.namespace)
         async def disconnect(sid: Text) -> None:
             structlogger.debug("studio_chat.sio.disconnect", sid=sid)
             self._cleanup_tasks_for_sid(sid)
 
-        @self.sio.on("session_request", namespace=self.namespace)
+        @self.sio_server.on("session_request", namespace=self.namespace)
         async def session_request(sid: Text, data: Optional[Dict]) -> None:
             """Overrides the base SocketIOInput session_request handler.
 
@@ -464,7 +489,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
             if data and data.get("is_voice", False):
                 self._start_voice_session(data["session_id"], sid, on_new_message)
 
-        @self.sio.on(self.user_message_evt, namespace=self.namespace)
+        @self.sio_server.on(self.user_message_evt, namespace=self.namespace)
         async def handle_message(sid: Text, data: Dict) -> None:
             """Overrides the base SocketIOInput handle_message handler."""
             # Handle voice messages
@@ -475,10 +500,18 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
                     ws.put_message(data)
                 return
 
-            # Handle text messages
-            await self.handle_user_message(sid, data, on_new_message)
+            try:
+                # Handle text messages
+                await self.handle_user_message(sid, data, on_new_message)
+            except Exception as e:
+                structlogger.exception(
+                    "studio_chat.sio.handle_message.error",
+                    error=str(e),
+                    sid=sid,
+                )
+                await self.emit("error", str(e), room=sid)
 
-        @self.sio.on("update_tracker", namespace=self.namespace)
+        @self.sio_server.on("update_tracker", namespace=self.namespace)
         async def on_update_tracker(sid: Text, data: Dict) -> None:
             await self.handle_tracker_update(sid, data)
 
@@ -500,16 +533,33 @@ class StudioVoiceOutputChannel(VoiceOutputChannel):
 
     def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
         message_id = uuid.uuid4().hex
-        return json.dumps({"marker": message_id}), message_id
+        marker_data: Dict[str, Any] = {"marker": message_id}
+
+        # Include comprehensive latency information if available
+        latency_data = {
+            "asr_latency_ms": call_state.asr_latency_ms,
+            "rasa_processing_latency_ms": call_state.rasa_processing_latency_ms,
+            "tts_first_byte_latency_ms": call_state.tts_first_byte_latency_ms,
+            "tts_complete_latency_ms": call_state.tts_complete_latency_ms,
+        }
+
+        # Filter out None values from latency data
+        latency_data = {k: v for k, v in latency_data.items() if v is not None}
+
+        # Add latency data to marker if any metrics are available
+        if latency_data:
+            marker_data["latency"] = latency_data
+
+        return json.dumps(marker_data), message_id
 
 
 class SocketIOVoiceWebsocketAdapter:
     """Adapter to make Socket.IO work like a Sanic WebSocket for voice channels."""
 
     def __init__(
-        self, sio: "AsyncServer", session_id: str, sid: str, bot_message_evt: str
+        self, sio_server: "AsyncServer", session_id: str, sid: str, bot_message_evt: str
     ) -> None:
-        self.sio = sio
+        self.sio_server = sio_server
         self.bot_message_evt = bot_message_evt
         self._closed = False
         self._receive_queue: asyncio.Queue[Any] = asyncio.Queue()
@@ -528,7 +578,7 @@ class SocketIOVoiceWebsocketAdapter:
     async def send(self, data: Any) -> None:
         """Send data to the client."""
         if not self.closed:
-            await self.sio.emit(self.bot_message_evt, data, room=self.sid)
+            await self.sio_server.emit(self.bot_message_evt, data, room=self.sid)
 
     async def recv(self) -> Any:
         """Receive data from the client."""

rasa/core/channels/telegram.py

@@ -4,6 +4,9 @@ import typing
 from copy import deepcopy
 from typing import Any, Awaitable, Callable, Dict, List, Optional, Text
 
+# Import aiogram at module level to raise error if not installed
+from aiogram import Bot
+from aiogram.types import Message, Update
 from sanic import Blueprint, response
 from sanic.request import Request
 from sanic.response import HTTPResponse
@@ -28,15 +31,7 @@ class TelegramOutput(OutputChannel):
         return "telegram"
 
     def __init__(self, access_token: Optional[Text]) -> None:
-        try:
-            from aiogram import Bot
-
-            self.bot = Bot(access_token)
-        except ImportError:
-            raise ImportError(
-                "To use the Telegram channel, please install the aiogram package "
-                "with 'pip install aiogram'"
-            )
+        self.bot = Bot(access_token)
 
     async def send_text_message(
         self, recipient_id: Text, text: Text, **kwargs: Any

rasa/core/channels/voice_ready/twilio_voice.py

@@ -30,7 +30,7 @@ TWILIO_VOICE_PATH = "webhooks/twilio_voice/webhook"
 
 
 def map_call_params(form: RequestParameters) -> CallParameters:
-    """Map the Audiocodes parameters to the CallParameters dataclass."""
+    """Map the Twilio Voice parameters to the CallParameters dataclass."""
     return CallParameters(
         call_id=form.get("CallSid"),
         user_phone=form.get("Caller"),

rasa/core/channels/voice_stream/asr/asr_event.py

@@ -15,7 +15,7 @@ class NewTranscript(ASREvent):
 
 @dataclass
 class UserIsSpeaking(ASREvent):
-    pass
+    text: str
 
 
 @dataclass

rasa/core/channels/voice_stream/asr/azure.py

@@ -1,7 +1,7 @@
 import asyncio
 import os
 from dataclasses import dataclass
-from typing import Any, AsyncIterator, Dict, Optional
+from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Optional
 
 import structlog
 
@@ -15,6 +15,9 @@ from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
 from rasa.shared.constants import AZURE_SPEECH_API_KEY_ENV_VAR
 from rasa.shared.exceptions import ConnectionException
 
+if TYPE_CHECKING:
+    from azure.cognitiveservices.speech import SpeechRecognitionEventArgs
+
 logger = structlog.get_logger(__name__)
 
 
@@ -43,9 +46,9 @@ class AzureASR(ASREngine[AzureASRConfig]):
         )
         self.main_loop = asyncio.get_running_loop()
 
-    def signal_user_is_speaking(self, event: Any) -> None:
+    def signal_user_is_speaking(self, event: "SpeechRecognitionEventArgs") -> None:
         """Replace the azure event with a generic is speaking event."""
-        self.fill_queue(UserIsSpeaking())
+        self.fill_queue(UserIsSpeaking(event.result.text))
 
     def fill_queue(self, event: Any) -> None:
         """Either puts the event or a dedicated ASR Event into the queue."""

rasa/core/channels/voice_stream/asr/deepgram.py

@@ -117,7 +117,7 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
                         self.accumulated_transcript, transcript
                     )
             elif transcript:
-                return UserIsSpeaking()
+                return UserIsSpeaking(transcript)
         # event that comes after utterance_end_ms of no new transcript
         elif data_type == "UtteranceEnd":
             if self.accumulated_transcript:

rasa/core/channels/voice_stream/audiocodes.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import base64
 import hmac
@@ -21,6 +23,7 @@ from rasa.core.channels.voice_stream.call_state import (
     call_state,
 )
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
+from rasa.core.channels.voice_stream.util import repack_voice_credentials
 from rasa.core.channels.voice_stream.voice_channel import (
     ContinueConversationAction,
     EndConversationAction,
@@ -85,7 +88,7 @@ class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
         # however, Audiocodes does not have an event to indicate that.
         # This is an approximation, as the bot will be sent the audio chunks next
         # which are played to the user immediately.
-        call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+        call_state.is_bot_speaking = True
         VoiceInputChannel._cancel_silence_timeout_watcher()
 
     async def send_intermediate_marker(self, recipient_id: str) -> None:
@@ -114,6 +117,7 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
         server_url: str,
         asr_config: Dict,
         tts_config: Dict,
+        interruptions: Optional[Dict[str, int]] = None,
         token: Optional[Text] = None,
     ):
         mark_as_beta_feature("Audiocodes (audiocodes_stream) Channel")
@@ -121,6 +125,7 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
             server_url=server_url,
             asr_config=asr_config,
             tts_config=tts_config,
+            interruptions=interruptions,
         )
         self.token = token
 
@@ -128,10 +133,10 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
     def from_credentials(
         cls,
         credentials: Optional[Dict[str, Any]],
-    ) -> "AudiocodesVoiceInputChannel":
-        channel = super().from_credentials(credentials)
-        channel.token = credentials.get("token")  # type: ignore[attr-defined, union-attr]
-        return channel  # type: ignore[return-value]
+    ) -> AudiocodesVoiceInputChannel:
+        cls.validate_basic_credentials(credentials)
+        new_creds = repack_voice_credentials(credentials)
+        return cls(**new_creds)
 
     def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
         return RasaAudioBytes(base64.b64decode(input_bytes))
@@ -185,7 +190,7 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
                     pass
                 elif activity["name"] == "playFinished":
                     logger.debug("audiocodes_stream.playFinished", data=activity)
-                    call_state.is_bot_speaking = False  # type: ignore[attr-defined]
+                    call_state.is_bot_speaking = False
                     if call_state.should_hangup:
                         logger.info("audiocodes_stream.hangup")
                         self._send_hangup(ws, data)

rasa/core/channels/voice_stream/browser_audio.py

@@ -1,8 +1,12 @@
+from __future__ import annotations
+
 import audioop
 import base64
 import json
+import os
 import uuid
-from typing import Any, Awaitable, Callable, Optional, Tuple
+import wave
+from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
 
 import structlog
 from sanic import (  # type: ignore[attr-defined]
@@ -18,6 +22,7 @@ from rasa.core.channels.voice_ready.utils import CallParameters
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.call_state import call_state
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
+from rasa.core.channels.voice_stream.util import repack_voice_credentials
 from rasa.core.channels.voice_stream.voice_channel import (
     ContinueConversationAction,
     EndConversationAction,
@@ -45,10 +50,70 @@ class BrowserAudioOutputChannel(VoiceOutputChannel):
 
     def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
         message_id = uuid.uuid4().hex
-        return json.dumps({"marker": message_id}), message_id
+        marker_data = {"marker": message_id}
+
+        # Include comprehensive latency information if available
+        latency_data = {
+            "asr_latency_ms": call_state.asr_latency_ms,
+            "rasa_processing_latency_ms": call_state.rasa_processing_latency_ms,
+            "tts_first_byte_latency_ms": call_state.tts_first_byte_latency_ms,
+            "tts_complete_latency_ms": call_state.tts_complete_latency_ms,
+        }
+
+        # Filter out None values from latency data
+        latency_data = {k: v for k, v in latency_data.items() if v is not None}
+
+        # Add latency data to marker if any metrics are available
+        if latency_data:
+            marker_data["latency"] = latency_data  # type: ignore[assignment]
+
+        return json.dumps(marker_data), message_id
 
 
 class BrowserAudioInputChannel(VoiceInputChannel):
+    requires_voice_license = False
+
+    def __init__(
+        self,
+        server_url: str,
+        asr_config: Dict[str, Any],
+        tts_config: Dict[str, Any],
+        recording: bool = False,
+        interruptions: Optional[Dict[str, int]] = None,
+    ) -> None:
+        """Initializes the browser audio input channel."""
+        super().__init__(server_url, asr_config, tts_config, interruptions)
+
+        # For debugging, recording of user audio might be useful
+        # to identify audio quality issues or transcription errors
+        self._recording_enabled = recording
+        self._wav_file: Optional[wave.Wave_write] = None
+
+    def _start_recording(self, call_id: str, user_id: str) -> None:
+        os.makedirs("recordings", exist_ok=True)
+        filename = f"{user_id}_{call_id}.wav"
+        file_path = os.path.join("recordings", filename)
+
+        if not self._recording_enabled:
+            return
+
+        self._wav_file = wave.open(file_path, "wb")
+        self._wav_file.setnchannels(1)  # Mono audio
+        self._wav_file.setsampwidth(4)  # 32-bit audio (4 bytes)
+        self._wav_file.setframerate(8000)  # 8kHz sample rate
+        logger.info("voice_channel.user_audio_recording.started", file_path=file_path)
+
+    def _append_audio_to_recording(self, audio_bytes: bytes) -> None:
+        if self._wav_file and self._recording_enabled:
+            self._wav_file.writeframes(audio_bytes)
+
+    def _stop_recording(self) -> None:
+        """Close the recording file if it's open."""
+        if self._wav_file:
+            self._wav_file.close()
+            self._wav_file = None
+            logger.debug("voice_channel.user_audio_recording.stopped")
+
     @classmethod
     def name(cls) -> str:
         return "browser_audio"
@@ -62,6 +127,15 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         call_id = f"inspect-{uuid.uuid4()}"
         return CallParameters(call_id, "local", "local", stream_id=call_id)
 
+    @classmethod
+    def from_credentials(
+        cls,
+        credentials: Optional[Dict[str, Any]],
+    ) -> BrowserAudioInputChannel:
+        cls.validate_basic_credentials(credentials)
+        new_creds = repack_voice_credentials(credentials or {})
+        return cls(**new_creds)
+
     def map_input_message(
         self,
         message: Any,
@@ -70,21 +144,29 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         data = json.loads(message)
         if "audio" in data:
             channel_bytes = base64.b64decode(data["audio"])
+            self._append_audio_to_recording(channel_bytes)
             audio_bytes = self.channel_bytes_to_rasa_audio_bytes(channel_bytes)
             return NewAudioAction(audio_bytes)
         elif "marker" in data:
             if data["marker"] == call_state.latest_bot_audio_id:
                 # Just finished streaming last audio bytes
-                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
+                call_state.is_bot_speaking = False
                 if call_state.should_hangup:
                     logger.debug(
                         "browser_audio.hangup", marker=call_state.latest_bot_audio_id
                     )
                     return EndConversationAction()
             else:
-                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+                call_state.is_bot_speaking = True
         return ContinueConversationAction()
 
+    async def interrupt_playback(
+        self, ws: Websocket, call_parameters: CallParameters
+    ) -> None:
+        """Interrupt the current playback of audio."""
+        logger.debug("browser_audio.interrupt_playback")
+        await ws.send(json.dumps({"interruptPlayback": True}))
+
     def create_output_channel(
         self, voice_websocket: Websocket, tts_engine: TTSEngine
     ) -> VoiceOutputChannel:
@@ -107,8 +189,13 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         @blueprint.websocket("/websocket")  # type: ignore
         async def handle_message(request: Request, ws: Websocket) -> None:
             try:
+                call_parameters = await self.collect_call_parameters(ws)
+                if call_parameters and call_parameters.call_id:
+                    self._start_recording(call_parameters.call_id, "local")
                 await self.run_audio_streaming(on_new_message, ws)
             except Exception as e:
                 logger.error("browser_audio.handle_message.error", error=e)
+            finally:
+                self._stop_recording()
 
         return blueprint

rasa/core/channels/voice_stream/call_state.py

@@ -1,7 +1,7 @@
 import asyncio
 from contextvars import ContextVar
 from dataclasses import dataclass, field
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, cast
 
 from werkzeug.local import LocalProxy
 
@@ -19,9 +19,20 @@ class CallState:
     should_hangup: bool = False
     connection_failed: bool = False
 
+    # Latency tracking - start times only
+    user_speech_start_time: Optional[float] = None
+    rasa_processing_start_time: Optional[float] = None
+    tts_start_time: Optional[float] = None
+
+    # Calculated latencies (used by channels like browser_audio)
+    asr_latency_ms: Optional[float] = None
+    rasa_processing_latency_ms: Optional[float] = None
+    tts_first_byte_latency_ms: Optional[float] = None
+    tts_complete_latency_ms: Optional[float] = None
+
     # Generic field for channel-specific state data
     channel_data: Dict[str, Any] = field(default_factory=dict)
 
 
 _call_state: ContextVar[CallState] = ContextVar("call_state")
-call_state = LocalProxy(_call_state)
+call_state: CallState = cast(CallState, LocalProxy(_call_state))