rasa-pro 3.12.0rc2__py3-none-any.whl → 3.12.0rc3__py3-none-any.whl

rasa/cli/dialogue_understanding_test.py

@@ -3,7 +3,7 @@ import asyncio
 import datetime
 import importlib
 import sys
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Type, cast
 
 import structlog
 
@@ -20,9 +20,7 @@ from rasa.core.exceptions import AgentNotReady
 from rasa.core.processor import MessageProcessor
 from rasa.core.utils import AvailableEndpoints
 from rasa.dialogue_understanding.commands import Command
-from rasa.dialogue_understanding.generator import (
-    LLMBasedCommandGenerator,
-)
+from rasa.dialogue_understanding.generator import LLMBasedCommandGenerator
 from rasa.dialogue_understanding.generator.command_parser import DEFAULT_COMMANDS
 from rasa.dialogue_understanding_test.command_metric_calculation import (
     calculate_command_metrics,
@@ -372,18 +370,17 @@ def split_test_results(
 def _get_llm_command_generator_config(
     processor: MessageProcessor,
 ) -> Optional[Dict[str, Any]]:
-    from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
-
     train_schema = processor.model_metadata.train_schema
 
     for node_name, node in train_schema.nodes.items():
         if node.matches_type(LLMBasedCommandGenerator, include_subtypes=True):
             # Configurations can reference model groups defined in the endpoints.yml
-            resolved_config = resolve_model_client_config(
+            resolved_llm_config = resolve_model_client_config(
                 node.config.get(LLM_CONFIG_KEY, {}), node_name
             )
+            llm_command_generator = cast(Type[LLMBasedCommandGenerator], node.uses)
             return combine_custom_and_default_config(
-                resolved_config, DEFAULT_LLM_CONFIG
+                resolved_llm_config, llm_command_generator.get_default_llm_config()
             )
 
     return None

rasa/cli/llm_fine_tuning.py

@@ -1,7 +1,7 @@
 import argparse
 import asyncio
 import sys
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Type, cast
 
 import structlog
 
@@ -22,7 +22,12 @@ from rasa.cli.e2e_test import (
 )
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.utils import AvailableEndpoints
-from rasa.dialogue_understanding.generator import SingleStepLLMCommandGenerator
+from rasa.dialogue_understanding.generator.llm_based_command_generator import (
+    LLMBasedCommandGenerator,
+)
+from rasa.dialogue_understanding.generator.multi_step.multi_step_llm_command_generator import (  # noqa: E501
+    MultiStepLLMCommandGenerator,
+)
 from rasa.e2e_test.e2e_test_runner import E2ETestRunner
 from rasa.llm_fine_tuning.annotation_module import annotate_e2e_tests
 from rasa.llm_fine_tuning.llm_data_preparation_module import convert_to_fine_tuning_data
@@ -112,7 +117,6 @@ def create_llm_finetune_data_preparation_subparser(
         help_text="Configuration file for the model server and the connectors as a "
         "yml file.",
     )
-
     return data_preparation_subparser
 
 
@@ -205,6 +209,9 @@ def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
 
     flows = asyncio.run(e2e_test_runner.agent.processor.get_flows())
     llm_command_generator_config = _get_llm_command_generator_config(e2e_test_runner)
+    llm_command_generator: Type[LLMBasedCommandGenerator] = _get_llm_command_generator(
+        e2e_test_runner
+    )
 
     # set up storage context
     storage_context = create_storage_context(StorageType.FILE, output_dir)
@@ -235,6 +242,7 @@ def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
             rephrase_config,
             args.num_rephrases,
             flows,
+            llm_command_generator,
             llm_command_generator_config,
             storage_context,
         )
@@ -271,30 +279,57 @@ def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
     write_statistics(statistics, output_dir)
 
     rasa.shared.utils.cli.print_success(
-        f"Data and intermediate results are written " f"to '{output_dir}'."
+        f"Data and intermediate results are written to '{output_dir}'."
     )
 
 
 def _get_llm_command_generator_config(e2e_test_runner: E2ETestRunner) -> Dict[str, Any]:
-    from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
-
     train_schema = e2e_test_runner.agent.processor.model_metadata.train_schema  # type: ignore
 
     for node_name, node in train_schema.nodes.items():
-        if node.matches_type(SingleStepLLMCommandGenerator, include_subtypes=True):
+        if node.matches_type(
+            LLMBasedCommandGenerator, include_subtypes=True
+        ) and not node.matches_type(
+            MultiStepLLMCommandGenerator, include_subtypes=True
+        ):
             # Configurations can reference model groups defined in the endpoints.yml
-            resolved_config = resolve_model_client_config(
+            resolved_llm_config = resolve_model_client_config(
                 node.config.get(LLM_CONFIG_KEY, {}), node_name
             )
+            llm_command_generator = cast(Type[LLMBasedCommandGenerator], node.uses)
             return combine_custom_and_default_config(
-                resolved_config, DEFAULT_LLM_CONFIG
+                resolved_llm_config, llm_command_generator.get_default_llm_config()
             )
 
     rasa.shared.utils.cli.print_error(
         "The provided model is not trained using 'SingleStepLLMCommandGenerator' or "
-        "its subclasses. Without it, no data for fine-tuning can be generated. To "
-        "resolve this, please include 'SingleStepLLMCommandGenerator' or its subclass "
-        "in your config and train your model."
+        "'CompactLLMCommandGenerator' or its subclasses. Without it, no data for "
+        "fine-tuning can be generated. To resolve this, please include "
+        "'SingleStepLLMCommandGenerator' or 'CompactLLMCommandGenerator' or its "
+        "subclasses in your config and train your model."
+    )
+    sys.exit(1)
+
+
+def _get_llm_command_generator(
+    e2e_test_runner: E2ETestRunner,
+) -> Type[LLMBasedCommandGenerator]:
+    train_schema = e2e_test_runner.agent.processor.model_metadata.train_schema  # type: ignore
+
+    for _, node in train_schema.nodes.items():
+        if node.matches_type(
+            LLMBasedCommandGenerator, include_subtypes=True
+        ) and not node.matches_type(
+            MultiStepLLMCommandGenerator, include_subtypes=True
+        ):
+            return cast(Type[LLMBasedCommandGenerator], node.uses)
+
+    rasa.shared.utils.cli.print_error(
+        "The provided model is not trained using 'SingleStepLLMCommandGenerator' or "
+        "'CompactLLMCommandGenerator' or its subclasses. Without it, no data for "
+        "fine-tuning can be generated. To resolve this, please include "
+        "'SingleStepLLMCommandGenerator' or 'CompactLLMCommandGenerator' or its "
+        "subclasses in your config and train your model."
     )
     sys.exit(1)
 

rasa/core/channels/voice_stream/asr/asr_event.py

@@ -16,3 +16,8 @@ class NewTranscript(ASREvent):
 @dataclass
 class UserIsSpeaking(ASREvent):
     pass
+
+
+@dataclass
+class UserSilence(ASREvent):
+    pass

rasa/core/channels/voice_stream/audiocodes.py

@@ -46,6 +46,19 @@ class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
     def name(cls) -> str:
         return "ac_voice"
 
+    def _ensure_stream_id(self) -> None:
+        """Audiocodes requires a stream ID with playStream messages."""
+        if "stream_id" not in call_state.channel_data:
+            call_state.channel_data["stream_id"] = 0
+
+    def _increment_stream_id(self) -> None:
+        self._ensure_stream_id()
+        call_state.channel_data["stream_id"] += 1
+
+    def _get_stream_id(self) -> str:
+        self._ensure_stream_id()
+        return str(call_state.channel_data["stream_id"])
+
     def rasa_audio_bytes_to_channel_bytes(
         self, rasa_audio_bytes: RasaAudioBytes
     ) -> bytes:
@@ -55,7 +68,7 @@ class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
         media_message = json.dumps(
             {
                 "type": "playStream.chunk",
-                "streamId": str(call_state.stream_id),
+                "streamId": self._get_stream_id(),
                 "audioChunk": channel_bytes.decode("utf-8"),
             }
         )
@@ -63,14 +76,14 @@ class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
 
     async def send_start_marker(self, recipient_id: str) -> None:
         """Send playStream.start before first audio chunk."""
-        call_state.stream_id += 1  # type: ignore[attr-defined]
+        self._increment_stream_id()
         media_message = json.dumps(
             {
                 "type": "playStream.start",
-                "streamId": str(call_state.stream_id),
+                "streamId": self._get_stream_id(),
             }
         )
-        logger.debug("Sending start marker", stream_id=call_state.stream_id)
+        logger.debug("Sending start marker", stream_id=self._get_stream_id())
         await self.voice_websocket.send(media_message)
 
     async def send_intermediate_marker(self, recipient_id: str) -> None:
@@ -82,10 +95,10 @@ class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
         media_message = json.dumps(
             {
                 "type": "playStream.stop",
-                "streamId": str(call_state.stream_id),
+                "streamId": self._get_stream_id(),
             }
         )
-        logger.debug("Sending end marker", stream_id=call_state.stream_id)
+        logger.debug("Sending end marker", stream_id=self._get_stream_id())
         await self.voice_websocket.send(media_message)
 
 

rasa/core/channels/voice_stream/call_state.py

@@ -1,7 +1,7 @@
 import asyncio
 from contextvars import ContextVar
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Any, Dict, Optional
 
 from werkzeug.local import LocalProxy
 
@@ -19,14 +19,8 @@ class CallState:
     should_hangup: bool = False
     connection_failed: bool = False
 
-    # Genesys requires the server and client each maintain a
-    # monotonically increasing message sequence number.
-    client_sequence_number: int = 0
-    server_sequence_number: int = 0
-    audio_buffer: bytearray = field(default_factory=bytearray)
-
-    # Audiocodes requires a stream ID at start and end of stream
-    stream_id: int = 0
+    # Generic field for channel-specific state data
+    channel_data: Dict[str, Any] = field(default_factory=dict)
 
 
 _call_state: ContextVar[CallState] = ContextVar("call_state")

rasa/core/channels/voice_stream/genesys.py

@@ -27,8 +27,23 @@ from rasa.core.channels.voice_stream.voice_channel import (
     VoiceOutputChannel,
 )
 
-# Not mentioned in the documentation but observed in Geneys's example
-# https://github.com/GenesysCloudBlueprints/audioconnector-server-reference-implementation
+"""
+Genesys throws a rate limit error with too many audio messages.
+To avoid this, we buffer the audio messages and send them in chunks.
+
+- global.inbound.binary.average.rate.per.second: 5
+The allowed average rate per second of inbound binary data
+
+- global.inbound.binary.max: 25
+The maximum number of inbound binary data messages
+that can be sent instantaneously
+
+https://developer.genesys.cloud/organization/organization/limits#audiohook
+
+The maximum binary message size is not mentioned
+in the documentation but observed in their example app
+https://github.com/GenesysCloudBlueprints/audioconnector-server-reference-implementation
+"""
 MAXIMUM_BINARY_MESSAGE_SIZE = 64000  # 64KB
 logger = structlog.get_logger(__name__)
 
@@ -56,52 +71,7 @@ class GenesysOutputChannel(VoiceOutputChannel):
     async def send_audio_bytes(
         self, recipient_id: str, audio_bytes: RasaAudioBytes
     ) -> None:
-        """
-        Send audio bytes to the recipient with buffering.
-
-        Genesys throws a rate limit error with too many audio messages.
-        To avoid this, we buffer the audio messages and send them in chunks.
-
-        - global.inbound.binary.average.rate.per.second: 5
-        The allowed average rate per second of inbound binary data
-
-        - global.inbound.binary.max: 25
-        The maximum number of inbound binary data messages
-        that can be sent instantaneously
-
-        https://developer.genesys.cloud/organization/organization/limits#audiohook
-        """
-        call_state.audio_buffer.extend(audio_bytes)
-
-        # If we receive a non-standard chunk size, assume it's the end of a sequence
-        # or buffer is more than 32KB (this is half of genesys's max audio message size)
-        if len(audio_bytes) != 1024 or len(call_state.audio_buffer) >= (
-            MAXIMUM_BINARY_MESSAGE_SIZE / 2
-        ):
-            # TODO: we should send the buffer when we receive a synthesis complete event
-            # from TTS. This will ensure that the last audio chunk is always sent.
-            await self._send_audio_buffer(self.voice_websocket)
-
-    async def _send_audio_buffer(self, ws: Websocket) -> None:
-        """Send the audio buffer to the recipient if it's not empty."""
-        if call_state.audio_buffer:
-            buffer_bytes = bytes(call_state.audio_buffer)
-            await self._send_bytes_to_ws(ws, buffer_bytes)
-            call_state.audio_buffer.clear()
-
-    async def _send_bytes_to_ws(self, ws: Websocket, data: bytes) -> None:
-        """Send audio bytes to the recipient as a binary websocket message."""
-        if len(data) <= MAXIMUM_BINARY_MESSAGE_SIZE:
-            await self.voice_websocket.send(data)
-        else:
-            # split the audio into chunks
-            current_position = 0
-            while current_position < len(data):
-                end_position = min(
-                    current_position + MAXIMUM_BINARY_MESSAGE_SIZE, len(data)
-                )
-                await self.voice_websocket.send(data[current_position:end_position])
-                current_position = end_position
+        await self.voice_websocket.send(audio_bytes)
 
     async def send_marker_message(self, recipient_id: str) -> None:
         """
@@ -119,6 +89,17 @@ class GenesysInputChannel(VoiceInputChannel):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
+    def _ensure_channel_data_initialized(self) -> None:
+        """Initialize Genesys-specific channel data if not already present.
+
+        Genesys requires the server and client each maintain a
+        monotonically increasing message sequence number.
+        """
+        if "server_sequence_number" not in call_state.channel_data:
+            call_state.channel_data["server_sequence_number"] = 0
+        if "client_sequence_number" not in call_state.channel_data:
+            call_state.channel_data["client_sequence_number"] = 0
+
     def _get_next_sequence(self) -> int:
         """
         Get the next message sequence number
@@ -128,23 +109,26 @@ class GenesysInputChannel(VoiceInputChannel):
         Genesys requires the server and client each maintain a
         monotonically increasing message sequence number.
         """
-        cs = call_state
-        cs.server_sequence_number += 1  # type: ignore[attr-defined]
-        return cs.server_sequence_number
+        self._ensure_channel_data_initialized()
+        call_state.channel_data["server_sequence_number"] += 1
+        return call_state.channel_data["server_sequence_number"]
 
     def _get_last_client_sequence(self) -> int:
         """Get the last client(Genesys) sequence number."""
-        return call_state.client_sequence_number
+        self._ensure_channel_data_initialized()
+        return call_state.channel_data["client_sequence_number"]
 
     def _update_client_sequence(self, seq: int) -> None:
         """Update the client(Genesys) sequence number."""
-        if seq - call_state.client_sequence_number != 1:
+        self._ensure_channel_data_initialized()
+
+        if seq - call_state.channel_data["client_sequence_number"] != 1:
             logger.warning(
                 "genesys.update_client_sequence.sequence_gap",
                 received_seq=seq,
-                last_seq=call_state.client_sequence_number,
+                last_seq=call_state.channel_data["client_sequence_number"],
             )
-        call_state.client_sequence_number = seq  # type: ignore[attr-defined]
+        call_state.channel_data["client_sequence_number"] = seq
 
     def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
         return RasaAudioBytes(input_bytes)
@@ -211,6 +195,7 @@ class GenesysInputChannel(VoiceInputChannel):
             voice_websocket,
             tts_engine,
             self.tts_cache,
+            min_buffer_size=MAXIMUM_BINARY_MESSAGE_SIZE // 2,
         )
 
     async def handle_open(self, ws: Websocket, message: dict) -> CallParameters:

rasa/core/channels/voice_stream/voice_channel.py

@@ -17,6 +17,7 @@ from rasa.core.channels.voice_stream.asr.asr_event import (
     ASREvent,
     NewTranscript,
     UserIsSpeaking,
+    UserSilence,
 )
 from rasa.core.channels.voice_stream.asr.azure import AzureASR
 from rasa.core.channels.voice_stream.asr.deepgram import DeepgramASR
@@ -120,13 +121,14 @@ class VoiceOutputChannel(OutputChannel):
         voice_websocket: Websocket,
         tts_engine: TTSEngine,
         tts_cache: TTSCache,
+        min_buffer_size: int = 0,
     ):
         super().__init__()
         self.voice_websocket = voice_websocket
         self.tts_engine = tts_engine
         self.tts_cache = tts_cache
-
         self.latest_message_id: Optional[str] = None
+        self.min_buffer_size = min_buffer_size
 
     def rasa_audio_bytes_to_channel_bytes(
         self, rasa_audio_bytes: RasaAudioBytes
@@ -186,6 +188,7 @@ class VoiceOutputChannel(OutputChannel):
         cached_audio_bytes = self.tts_cache.get(text)
         collected_audio_bytes = RasaAudioBytes(b"")
         seconds_marker = -1
+        last_sent_offset = 0
 
         # Send start marker before first chunk
         try:
@@ -205,17 +208,37 @@ class VoiceOutputChannel(OutputChannel):
                 audio_stream = self.chunk_audio(generate_silence())
 
         async for audio_bytes in audio_stream:
-            try:
-                await self.send_audio_bytes(recipient_id, audio_bytes)
-                full_seconds_of_audio = len(collected_audio_bytes) // HERTZ
-                if full_seconds_of_audio > seconds_marker:
-                    await self.send_intermediate_marker(recipient_id)
-                    seconds_marker = full_seconds_of_audio
+            collected_audio_bytes = RasaAudioBytes(collected_audio_bytes + audio_bytes)
 
+            # Check if we have enough new bytes to send
+            current_buffer_size = len(collected_audio_bytes) - last_sent_offset
+            should_send = current_buffer_size >= self.min_buffer_size
+
+            if should_send:
+                try:
+                    # Send only the new bytes since last send
+                    new_bytes = RasaAudioBytes(collected_audio_bytes[last_sent_offset:])
+                    await self.send_audio_bytes(recipient_id, new_bytes)
+                    last_sent_offset = len(collected_audio_bytes)
+
+                    full_seconds_of_audio = len(collected_audio_bytes) // HERTZ
+                    if full_seconds_of_audio > seconds_marker:
+                        await self.send_intermediate_marker(recipient_id)
+                        seconds_marker = full_seconds_of_audio
+
+                except (WebsocketClosed, ServerError):
+                    # ignore sending error, and keep collecting and caching audio bytes
+                    call_state.connection_failed = True  # type: ignore[attr-defined]
+
+        # Send any remaining audio not yet sent
+        remaining_bytes = len(collected_audio_bytes) - last_sent_offset
+        if remaining_bytes > 0:
+            try:
+                new_bytes = RasaAudioBytes(collected_audio_bytes[last_sent_offset:])
+                await self.send_audio_bytes(recipient_id, new_bytes)
             except (WebsocketClosed, ServerError):
-                # ignore sending error, and keep collecting and caching audio bytes
+                # ignore sending error
                 call_state.connection_failed = True  # type: ignore[attr-defined]
-            collected_audio_bytes = RasaAudioBytes(collected_audio_bytes + audio_bytes)
 
         try:
             await self.send_end_marker(recipient_id)
@@ -265,13 +288,7 @@ class VoiceInputChannel(InputChannel):
         self.monitor_silence = monitor_silence
         self.tts_cache = TTSCache(tts_config.get("cache_size", 1000))
 
-    async def handle_silence_timeout(
-        self,
-        voice_websocket: Websocket,
-        on_new_message: Callable[[UserMessage], Awaitable[Any]],
-        tts_engine: TTSEngine,
-        call_parameters: CallParameters,
-    ) -> None:
+    async def monitor_silence_timeout(self, asr_event_queue: asyncio.Queue) -> None:
         timeout = call_state.silence_timeout
         if not timeout:
             return
@@ -279,16 +296,8 @@ class VoiceInputChannel(InputChannel):
             return
         logger.debug("voice_channel.silence_timeout_watch_started", timeout=timeout)
         await asyncio.sleep(timeout)
+        await asr_event_queue.put(UserSilence())
         logger.debug("voice_channel.silence_timeout_tripped")
-        output_channel = self.create_output_channel(voice_websocket, tts_engine)
-        message = UserMessage(
-            "/silence_timeout",
-            output_channel,
-            call_parameters.stream_id,
-            input_channel=self.name(),
-            metadata=asdict(call_parameters),
-        )
-        await on_new_message(message)
 
     @staticmethod
     def _cancel_silence_timeout_watcher() -> None:
@@ -350,6 +359,7 @@ class VoiceInputChannel(InputChannel):
         _call_state.set(CallState())
         asr_engine = asr_engine_from_config(self.asr_config)
         tts_engine = tts_engine_from_config(self.tts_config)
+        asr_event_queue: asyncio.Queue = asyncio.Queue()
         await asr_engine.connect()
 
         call_parameters = await self.collect_call_parameters(channel_websocket)
@@ -376,12 +386,7 @@ class VoiceInputChannel(InputChannel):
                     self._cancel_silence_timeout_watcher()
                     call_state.silence_timeout_watcher = (  # type: ignore[attr-defined]
                         asyncio.create_task(
-                            self.handle_silence_timeout(
-                                channel_websocket,
-                                on_new_message,
-                                tts_engine,
-                                call_parameters,
-                            )
+                            self.monitor_silence_timeout(asr_event_queue)
                         )
                     )
                 if isinstance(channel_action, NewAudioAction):
@@ -390,8 +395,13 @@ class VoiceInputChannel(InputChannel):
                     # end stream event came from the other side
                     break
 
-        async def consume_asr_events() -> None:
+        async def receive_asr_events() -> None:
             async for event in asr_engine.stream_asr_events():
+                await asr_event_queue.put(event)
+
+        async def handle_asr_events() -> None:
+            while True:
+                event = await asr_event_queue.get()
                 await self.handle_asr_event(
                     event,
                     channel_websocket,
@@ -400,16 +410,18 @@ class VoiceInputChannel(InputChannel):
                     call_parameters,
                 )
 
-        audio_forwarding_task = asyncio.create_task(consume_audio_bytes())
-        asr_event_task = asyncio.create_task(consume_asr_events())
+        tasks = [
+            asyncio.create_task(consume_audio_bytes()),
+            asyncio.create_task(receive_asr_events()),
+            asyncio.create_task(handle_asr_events()),
+        ]
         await asyncio.wait(
-            [audio_forwarding_task, asr_event_task],
+            tasks,
             return_when=asyncio.FIRST_COMPLETED,
         )
-        if not audio_forwarding_task.done():
-            audio_forwarding_task.cancel()
-        if not asr_event_task.done():
-            asr_event_task.cancel()
+        for task in tasks:
+            if not task.done():
+                task.cancel()
         await tts_engine.close_connection()
         await asr_engine.close_connection()
         await channel_websocket.close()
@@ -447,3 +459,13 @@ class VoiceInputChannel(InputChannel):
         elif isinstance(e, UserIsSpeaking):
             self._cancel_silence_timeout_watcher()
             call_state.is_user_speaking = True  # type: ignore[attr-defined]
+        elif isinstance(e, UserSilence):
+            output_channel = self.create_output_channel(voice_websocket, tts_engine)
+            message = UserMessage(
+                "/silence_timeout",
+                output_channel,
+                call_parameters.stream_id,
+                input_channel=self.name(),
+                metadata=asdict(call_parameters),
+            )
+            await on_new_message(message)