rasa-pro 3.11.4__py3-none-any.whl → 3.12.0__py3-none-any.whl

rasa/llm_fine_tuning/paraphrasing/rephrase_validator.py

@@ -1,18 +1,18 @@
-from typing import Dict, Any, List
+from typing import Any, Dict, List, Type
 
 import structlog
 
 from rasa.dialogue_understanding.commands import Command, SetSlotCommand
-from rasa.dialogue_understanding.generator import SingleStepLLMCommandGenerator
+from rasa.dialogue_understanding.generator.llm_based_command_generator import (
+    LLMBasedCommandGenerator,
+)
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.paraphrasing.rephrased_user_message import (
     RephrasedUserMessage,
 )
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.exceptions import ProviderClientAPIException
-from rasa.shared.utils.llm import (
-    llm_factory,
-)
+from rasa.shared.utils.llm import llm_factory
 
 structlogger = structlog.get_logger()
 
@@ -26,6 +26,7 @@ class RephraseValidator:
         self,
         rephrasings: List[RephrasedUserMessage],
         conversation: Conversation,
+        llm_command_generator: Type[LLMBasedCommandGenerator],
     ) -> List[RephrasedUserMessage]:
         """Split rephrased user messages into passing and failing.
 
@@ -38,6 +39,7 @@ class RephraseValidator:
         Args:
             rephrasings: The rephrased user messages.
             conversation: The conversation.
+            llm_command_generator: A LLM based command generator class.
 
         Returns:
             A list of rephrased user messages including the passing and failing
@@ -49,7 +51,9 @@ class RephraseValidator:
             current_rephrasings = rephrasings[i]
 
             for rephrase in current_rephrasings.rephrasings:
-                if await self._validate_rephrase_is_passing(rephrase, step):
+                if await self._validate_rephrase_is_passing(
+                    rephrase, step, llm_command_generator
+                ):
                     current_rephrasings.passed_rephrasings.append(rephrase)
                 else:
                     current_rephrasings.failed_rephrasings.append(rephrase)
@@ -60,25 +64,26 @@ class RephraseValidator:
         self,
         rephrase: str,
         step: ConversationStep,
+        llm_command_generator: Type[LLMBasedCommandGenerator],
     ) -> bool:
         prompt = self._update_prompt(
             rephrase, step.original_test_step.text, step.llm_prompt
         )
 
-        action_list = await self._invoke_llm(prompt)
+        action_list = await self._invoke_llm(
+            prompt, llm_command_generator.get_default_llm_config()
+        )
 
         commands_from_original_utterance = step.llm_commands
-        commands_from_rephrased_utterance = (
-            SingleStepLLMCommandGenerator.parse_commands(action_list, None, self.flows)
+        commands_from_rephrased_utterance = llm_command_generator.parse_commands(  # type: ignore
+            action_list, None, self.flows
         )
         return self._check_commands_match(
             commands_from_original_utterance, commands_from_rephrased_utterance
         )
 
-    async def _invoke_llm(self, prompt: str) -> str:
-        from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
-
-        llm = llm_factory(self.llm_config, DEFAULT_LLM_CONFIG)
+    async def _invoke_llm(self, prompt: str, default_llm_config: Dict[str, Any]) -> str:
+        llm = llm_factory(self.llm_config, default_llm_config)
 
         try:
             llm_response = await llm.acompletion(prompt)

rasa/llm_fine_tuning/paraphrasing_module.py

@@ -1,8 +1,11 @@
-from typing import List, Dict, Any, Tuple
+from typing import Any, Dict, List, Tuple, Type
 
 import structlog
 from tqdm import tqdm
 
+from rasa.dialogue_understanding.generator.llm_based_command_generator import (
+    LLMBasedCommandGenerator,
+)
 from rasa.llm_fine_tuning.conversations import Conversation
 from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
     ConversationRephraser,
@@ -25,6 +28,7 @@ async def create_paraphrased_conversations(
     rephrase_config: Dict[str, Any],
     num_rephrases: int,
     flows: FlowsList,
+    llm_command_generator: Type[LLMBasedCommandGenerator],
     llm_command_generator_config: Dict[str, Any],
     storage_context: StorageContext,
 ) -> Tuple[List[Conversation], Dict[str, Any]]:
@@ -71,7 +75,7 @@ async def create_paraphrased_conversations(
             rephrasings = _filter_rephrasings(rephrasings, conversations[i])
             # check if the rephrasings are still producing the same commands
             rephrasings = await validator.validate_rephrasings(
-                rephrasings, current_conversation
+                rephrasings, current_conversation, llm_command_generator
             )
         except ProviderClientAPIException as e:
             structlogger.error(

rasa/llm_fine_tuning/storage.py

@@ -1,9 +1,9 @@
 import json
 import os
-from abc import abstractmethod, ABC
+from abc import ABC, abstractmethod
 from enum import Enum
 from pathlib import Path
-from typing import List, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, List, Optional
 
 import rasa.shared.utils.io
 import rasa.utils.io
@@ -11,9 +11,9 @@ from rasa.llm_fine_tuning.conversations import Conversation
 from rasa.shared.utils.yaml import write_yaml
 
 if TYPE_CHECKING:
+    from rasa.e2e_test.e2e_test_case import TestSuite
     from rasa.llm_fine_tuning.llm_data_preparation_module import LLMDataExample
     from rasa.llm_fine_tuning.train_test_split_module import DataExampleFormat
-    from rasa.e2e_test.e2e_test_case import TestSuite
 
 
 class StorageType(Enum):

rasa/llm_fine_tuning/train_test_split_module.py

@@ -1,27 +1,18 @@
 import random
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Dict, List, Protocol, Set, Tuple
+from typing import Any, Dict, List, Protocol, Set, Tuple, Type
 
 import structlog
 
+from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.e2e_test.e2e_test_case import TestSuite
 from rasa.llm_fine_tuning.llm_data_preparation_module import LLMDataExample
 from rasa.llm_fine_tuning.storage import StorageContext
+from rasa.llm_fine_tuning.utils import commands_as_string
 
 TRAIN_TEST_MODULE_STORAGE_LOCATION = "4_train_test_split"
 
-SUPPORTED_COMMANDS = [
-    "SetSlot",
-    "StartFlow",
-    "CancelFlow",
-    "ChitChat",
-    "SkipQuestion",
-    "SearchAndReply",
-    "HumanHandoff",
-    "Clarify",
-]
-
 INSTRUCTION_DATA_FORMAT = "instruction"
 CONVERSATIONAL_DATA_FORMAT = "conversational"
 
@@ -77,17 +68,19 @@ class ConversationalDataFormat(DataExampleFormat):
         }
 
 
-def _get_command_types_covered_by_llm_data_point(commands: LLMDataExample) -> Set[str]:
+def _get_command_types_covered_by_llm_data_point(
+    data_point: LLMDataExample,
+) -> Set[Type[PromptCommand]]:
     """Get the command types covered by the LLM data point.
 
     This function returns the set of command types from the output present in a
-    LLMDataExample object. Eg: The function returns {'SetSlot', 'StartFlow'} when the
-    LLMDataExample.output is 'SetSlot(slot, abc), SetSlot(slot, cde), StartFlow(xyz)'.
+    LLMDataExample object. Eg: The function returns {'SetSlotCommand',
+    'StartFlowCommand'} when the LLMDataExample.output is 'SetSlotCommand(slot, abc),
+    SetSlotCommand(slot, cde), StartFlowCommand(xyz)'.
     """
     commands_covered = set()
-    for command in SUPPORTED_COMMANDS:
-        if command in commands.output:
-            commands_covered.add(command)
+    for command in data_point.output:
+        commands_covered.add(command.__class__)
     return commands_covered
 
 
@@ -146,14 +139,18 @@ def _get_minimum_test_case_groups_to_cover_all_commands(
         {
             "test_case_name": "t1",
             "data_examples": [],
-            "commands": {"SetSlot", "CancelFlow"}
+            "commands": {"SetSlotCommand", "CancelFlowCommand"}
         },
-        {"test_case_name": "t2", "data_examples": [], "commands": {"CancelFlow"}},
-        {"test_case_name": "t3", "data_examples": [], "commands": {"StartFlow"}},
+        {
+            "test_case_name": "t2",
+            "data_examples": [],
+            "commands": {"CancelFlowCommand"}
+        },
+        {"test_case_name": "t3", "data_examples": [], "commands": {"StartFlowCommand"}},
         {
             "test_case_name": "t4",
             "data_examples": [],
-            "commands": {"SetSlot", "StartFlow"}
+            "commands": {"SetSlotCommand", "StartFlowCommand"}
         },
     ]
 
@@ -166,7 +163,7 @@ def _get_minimum_test_case_groups_to_cover_all_commands(
         command for test_group in grouped_data for command in test_group[KEY_COMMANDS]
     )
     selected_test_cases = []
-    covered_commands: Set[str] = set()
+    covered_commands: Set[Type[PromptCommand]] = set()
 
     while covered_commands != all_commands:
         # Find the test case group that covers the most number of uncovered commands
@@ -187,7 +184,7 @@ def _get_minimum_test_case_groups_to_cover_all_commands(
 
     structlogger.info(
         "llm_fine_tuning.train_test_split_module.command_coverage_in_train_dataset",
-        covered_commands=covered_commands,
+        covered_commands=[command.__name__ for command in covered_commands],
     )
     return selected_test_cases
 
@@ -205,7 +202,10 @@ def _get_finetuning_data_in_instruction_data_format(
         data: List[Dict[str, Any]],
     ) -> List[DataExampleFormat]:
         return [
-            InstructionDataFormat(llm_data_example.prompt, llm_data_example.output)
+            InstructionDataFormat(
+                llm_data_example.prompt,
+                commands_as_string(llm_data_example.output),
+            )
             for test_group in data
             for llm_data_example in test_group[KEY_DATA_EXAMPLES]
         ]
@@ -232,7 +232,7 @@ def _get_finetuning_data_in_conversational_data_format(
                 [
                     ConversationalMessageDataFormat("user", llm_data_example.prompt),
                     ConversationalMessageDataFormat(
-                        "assistant", llm_data_example.output
+                        "assistant", commands_as_string(llm_data_example.output)
                     ),
                 ]
             )
@@ -271,7 +271,7 @@ def _check_and_log_missing_validation_dataset_command_coverage(
         structlogger.warning(
             "llm_fine_tuning.train_test_split_module.missing_commands_in_validation_dat"
             "aset",
-            missing_commands=missing_commands,
+            missing_commands=[command.__name__ for command in missing_commands],
         )
 
 

rasa/llm_fine_tuning/utils.py

@@ -0,0 +1,7 @@
+from typing import List
+
+from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
+
+
+def commands_as_string(commands: List[PromptCommand], delimiter: str = "\n") -> str:
+    return delimiter.join([command.to_dsl() for command in commands])

rasa/markers/marker.py

@@ -1,15 +1,14 @@
 import logging
 from typing import List, Optional, Text
 
-from rasa.shared.core.domain import Domain
-from rasa.shared.core.events import ActionExecuted, Event, SlotSet, UserUttered
-
 from rasa.markers.marker_base import (
     ConditionMarker,
     Marker,
     MarkerRegistry,
     OperatorMarker,
 )
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.events import ActionExecuted, Event, SlotSet, UserUttered
 
 logger = logging.getLogger(__name__)
 

rasa/markers/marker_base.py

@@ -6,9 +6,9 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from pathlib import Path
 from typing import (
-    ClassVar,
     TYPE_CHECKING,
     Any,
+    ClassVar,
     Dict,
     Iterator,
     List,
@@ -30,7 +30,6 @@ from rasa.shared.core.domain import Domain
 from rasa.shared.core.events import ActionExecuted, Event, UserUttered
 from rasa.shared.exceptions import InvalidConfigException, RasaException
 from rasa.utils.io import WriteRow
-
 from rasa.utils.yaml import (
     collect_configs_from_yaml_files,
     collect_yaml_files_from_path,

rasa/markers/upload.py

@@ -3,10 +3,10 @@ from pathlib import Path
 from typing import Any, Dict, List, Text
 
 import requests
-from rasa.shared.core.domain import Domain
-from rasa.shared.utils.cli import print_error_and_exit
 
 from rasa.markers.validate import validate_marker_file
+from rasa.shared.core.domain import Domain
+from rasa.shared.utils.cli import print_error_and_exit
 from rasa.utils.yaml import (
     YAML_CONFIGS,
     collect_configs_from_yaml_files,

rasa/markers/validate.py

@@ -1,10 +1,9 @@
 from pathlib import Path
 
-from rasa.shared.core.domain import Domain
-from rasa.shared.utils.cli import print_error_and_exit
-
 from rasa.markers.marker import OrMarker
 from rasa.markers.marker_base import Marker
+from rasa.shared.core.domain import Domain
+from rasa.shared.utils.cli import print_error_and_exit
 
 
 def validate_marker_file(domain: Domain, markers_path: Path) -> None:

rasa/model.py

@@ -3,13 +3,11 @@ import hashlib
 import logging
 import os
 from pathlib import Path
-from subprocess import check_output, DEVNULL, CalledProcessError
-from typing import Text, Optional, Union
-
-from rasa.shared.constants import DEFAULT_MODELS_PATH
+from subprocess import DEVNULL, CalledProcessError, check_output
+from typing import Optional, Text, Union
 
 from rasa.exceptions import ModelNotFound
-
+from rasa.shared.constants import DEFAULT_MODELS_PATH
 
 logger = logging.getLogger(__name__)
 

rasa/model_manager/config.py

@@ -1,5 +1,5 @@
-import sys
 import os
+import sys
 
 DEFAULT_SERVER_BASE_WORKING_DIRECTORY = "working-data"
 

rasa/model_manager/model_api.py

@@ -1,21 +1,22 @@
 import asyncio
-from functools import wraps
 import os
+from functools import wraps
 from http import HTTPStatus
 from typing import Any, Callable, Dict, Optional, Union
+
 import dotenv
 import psutil
+import structlog
 from sanic import Blueprint, Sanic, response
-from sanic.response import json
 from sanic.exceptions import NotFound
 from sanic.request import Request
-import structlog
+from sanic.response import json
 from socketio import AsyncServer
 
+from rasa.constants import MODEL_ARCHIVE_EXTENSION
 from rasa.exceptions import ModelNotFound
 from rasa.model_manager import config
 from rasa.model_manager.config import SERVER_BASE_URL
-from rasa.constants import MODEL_ARCHIVE_EXTENSION
 from rasa.model_manager.runner_service import (
     BotSession,
     BotSessionStatus,

rasa/model_manager/runner_service.py

@@ -1,22 +1,22 @@
 import os
 import shutil
+import subprocess
+from enum import Enum
 from typing import Dict, Optional
+
 import aiohttp
 import structlog
-import subprocess
 from pydantic import BaseModel, ConfigDict
-from enum import Enum
 
+from rasa.constants import MODEL_ARCHIVE_EXTENSION
 from rasa.exceptions import ModelNotFound
+from rasa.model_manager import config
 from rasa.model_manager.utils import (
+    logs_path,
     models_base_path,
     subpath,
     write_encoded_data_to_file,
 )
-from rasa.constants import MODEL_ARCHIVE_EXTENSION
-
-from rasa.model_manager import config
-from rasa.model_manager.utils import logs_path
 from rasa.model_manager.warm_rasa_process import start_rasa_process
 
 structlogger = structlog.get_logger()
@@ -258,10 +258,13 @@ def run_bot(
 
 async def update_bot_status(bot: BotSession) -> None:
     """Update the status of a bot based on the process return code."""
-    if bot.has_died_recently():
-        set_bot_status_to_stopped(bot)
-    elif await bot.completed_startup_recently():
-        set_bot_status_to_running(bot)
+    try:
+        if bot.has_died_recently():
+            set_bot_status_to_stopped(bot)
+        elif await bot.completed_startup_recently():
+            set_bot_status_to_running(bot)
+    except Exception as e:
+        structlogger.error("model_runner.update_bot_status.error", error=str(e))
 
 
 def terminate_bot(bot: BotSession) -> None:

rasa/model_manager/socket_bridge.py

@@ -1,8 +1,10 @@
+import json
 from typing import Any, Dict, Optional
 
-from socketio import AsyncServer
 import structlog
+from socketio import AsyncServer
 from socketio.asyncio_client import AsyncClient
+from socketio.exceptions import ConnectionRefusedError
 
 from rasa.model_manager.runner_service import BotSession
 from rasa.model_manager.studio_jwt_auth import (
@@ -28,7 +30,7 @@ async def socketio_websocket_traffic_wrapper(
 
     if auth_token is None:
         structlogger.error("model_runner.user_no_token", sid=sid)
-        return False
+        raise ConnectionRefusedError("model_runner.user_no_token")
 
     try:
         authenticate_user_to_service(auth_token)
@@ -37,22 +39,22 @@ async def socketio_websocket_traffic_wrapper(
         structlogger.error(
             "model_runner.user_authentication_failed", sid=sid, error=str(error)
         )
-        return False
+        raise ConnectionRefusedError("model_runner.user_authentication_failed")
 
     deployment_id = auth.get("deployment_id") if auth else None
 
     if deployment_id is None:
         structlogger.error("model_runner.bot_no_deployment_id", sid=sid)
-        return False
+        raise ConnectionRefusedError("model_runner.bot_no_deployment_id")
 
     bot = running_bots.get(deployment_id)
     if bot is None:
         structlogger.error("model_runner.bot_not_found", deployment_id=deployment_id)
-        return False
+        raise ConnectionRefusedError("model_runner.bot_not_found")
 
     if not bot.is_alive():
         structlogger.error("model_runner.bot_not_alive", deployment_id=deployment_id)
-        return False
+        raise ConnectionRefusedError("model_runner.bot_not_alive")
 
     client = await create_bridge_client(sio, bot.internal_url, sid, deployment_id)
 
@@ -66,7 +68,7 @@ async def socketio_websocket_traffic_wrapper(
         structlogger.error(
             "model_runner.bot_connection_failed", deployment_id=deployment_id
         )
-        return False
+        raise ConnectionRefusedError("model_runner.bot_connection_failed")
 
 
 def create_bridge_server(sio: AsyncServer, running_bots: Dict[str, BotSession]) -> None:
@@ -92,7 +94,7 @@ def create_bridge_server(sio: AsyncServer, running_bots: Dict[str, BotSession])
 
     @sio.on("*")
     async def handle_message(event: str, sid: str, data: Dict[str, Any]) -> None:
-        """ "Bridge messages between user and bot.
+        """Bridge messages between user and bot.
 
         Both incoming user messages to the bot_url and
         bot responses sent back to the client need to
@@ -111,7 +113,7 @@ async def create_bridge_client(
 ) -> AsyncClient:
     """Create a new socket bridge client.
 
-    Forwards messages comming from the bot to the user.
+    Forwards messages coming from the bot to the user.
     """
     client = AsyncClient()
 
@@ -129,6 +131,10 @@ async def create_bridge_client(
         structlogger.debug("model_runner.bot_message", deployment_id=deployment_id)
         await sio.emit("bot_message", data, room=sid)
 
+    @client.event  # type: ignore[misc]
+    async def tracker(data: Dict[str, Any]) -> None:
+        await sio.emit("tracker", json.loads(data), room=sid)
+
     @client.event  # type: ignore[misc]
     async def disconnect() -> None:
         structlogger.debug(

rasa/model_manager/studio_jwt_auth.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, Optional
 import jwt
 import requests
 import structlog
+
 from rasa.shared.exceptions import RasaException
 
 structlogger = structlog.get_logger()

rasa/model_manager/trainer_service.py

@@ -1,23 +1,25 @@
 import os
-from typing import Any, Dict, Optional
 import shutil
-import structlog
 import subprocess
+from enum import Enum
+from typing import Any, Dict, Optional
+
+import structlog
+from pydantic import BaseModel, ConfigDict
+
 from rasa.constants import MODEL_ARCHIVE_EXTENSION
+from rasa.model_manager import config
 from rasa.model_manager.utils import (
+    ensure_base_directory_exists,
+    logs_path,
     models_base_path,
     subpath,
     write_encoded_data_to_file,
 )
-from pydantic import BaseModel, ConfigDict
-from enum import Enum
-
-from rasa.model_manager import config
 from rasa.model_manager.warm_rasa_process import (
     start_rasa_process,
 )
 from rasa.model_training import generate_random_model_name
-from rasa.model_manager.utils import ensure_base_directory_exists, logs_path
 
 structlogger = structlog.get_logger()
 

rasa/model_manager/utils.py

@@ -1,5 +1,5 @@
-import os
 import base64
+import os
 from typing import Optional
 
 import structlog

rasa/model_manager/warm_rasa_process.py

@@ -1,12 +1,16 @@
+import os
 import shlex
 import subprocess
-from rasa.__main__ import main
-import os
+import uuid
+from dataclasses import dataclass
 from typing import List
+
 import structlog
-from dataclasses import dataclass
-import uuid
 
+from rasa.__main__ import main
+from rasa.dialogue_understanding.constants import (
+    RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME,
+)
 from rasa.model_manager import config
 from rasa.model_manager.utils import ensure_base_directory_exists, logs_path
 
@@ -42,7 +46,7 @@ def _create_warm_rasa_process() -> WarmRasaProcess:
     ]
 
     envs = os.environ.copy()
-    envs["RASA_TELEMETRY_ENABLED"] = "false"
+    envs[RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME] = "true"
 
     log_id = uuid.uuid4().hex
     log_path = logs_path(log_id)
@@ -146,14 +150,15 @@ def warmup() -> None:
     directly start.
     """
     try:
-        import presidio_analyzer  # noqa: F401
-        import litellm  # noqa: F401
         import langchain  # noqa: F401
-        import tensorflow  # noqa: F401
+        import litellm  # noqa: F401
         import matplotlib  # noqa: F401
-        import pandas  # noqa: F401
         import numpy  # noqa: F401
+        import pandas  # noqa: F401
+        import presidio_analyzer  # noqa: F401
         import spacy  # noqa: F401
+        import tensorflow  # noqa: F401
+
         import rasa.validator  # noqa: F401
     except ImportError:
         pass

rasa/model_service.py

@@ -1,18 +1,15 @@
 import logging
+from urllib.parse import urlparse
 
-from sanic import Sanic
 import structlog
+from sanic import Sanic
 
 from rasa.cli.scaffold import print_error_and_exit
 from rasa.core.persistor import RemoteStorageType, get_persistor
 from rasa.core.utils import list_routes
-from rasa.model_manager import model_api
-from rasa.model_manager import config
+from rasa.model_manager import config, model_api
 from rasa.model_manager.config import SERVER_BASE_URL, SERVER_PORT
 from rasa.utils.common import configure_logging_and_warnings
-import rasa.utils.licensing
-from urllib.parse import urlparse
-
 from rasa.utils.log_utils import configure_structlog
 from rasa.utils.sanic_error_handler import register_custom_sanic_error_handler
 
@@ -64,6 +61,8 @@ def main() -> None:
     The API server can receive requests to train models, run bots, and manage
     the lifecycle of models and bots.
     """
+    import rasa.utils.licensing
+
     log_level = logging.DEBUG
     configure_logging_and_warnings(
         log_level=log_level,