rasa-pro 3.12.0.dev2__py3-none-any.whl → 3.12.0.dev4__py3-none-any.whl

rasa/__main__.py

@@ -1,13 +1,11 @@
 import argparse
-from typing import Optional, List
-import structlog
 import os
 import platform
 import sys
+from typing import List, Optional
 
+import structlog
 from rasa_sdk import __version__ as rasa_sdk_version
-from rasa.constants import MINIMUM_COMPATIBLE_VERSION
-from rasa.utils.log_utils import configure_structlog
 
 import rasa.telemetry
 import rasa.utils.io
@@ -15,8 +13,10 @@ import rasa.utils.tensorflow.environment as tf_env
 from rasa import version
 from rasa.cli import (
     data,
+    evaluate,
     export,
     interactive,
+    llm_fine_tuning,
     run,
     scaffold,
     shell,
@@ -25,18 +25,18 @@ from rasa.cli import (
     train,
     visualize,
     x,
-    evaluate,
-    llm_fine_tuning,
 )
 from rasa.cli.arguments.default_arguments import add_logging_options
 from rasa.cli.utils import (
+    check_if_studio_command,
     parse_last_positional_argument_as_model_path,
     warn_if_rasa_plus_package_installed,
-    check_if_studio_command,
 )
+from rasa.constants import MINIMUM_COMPATIBLE_VERSION
 from rasa.plugin import plugin_manager
 from rasa.shared.exceptions import RasaException
 from rasa.utils.common import configure_logging_and_warnings
+from rasa.utils.log_utils import configure_structlog
 
 structlogger = structlog.get_logger()
 

rasa/anonymization/anonymisation_rule_yaml_reader.py

@@ -1,7 +1,6 @@
 from typing import List, Optional, Text
 
 import pycountry
-from rasa.shared.exceptions import RasaException
 
 from rasa.anonymization.anonymization_rule_executor import (
     AnonymizationRule,
@@ -11,6 +10,7 @@ from rasa.anonymization.utils import (
     read_endpoint_config,
     validate_anonymization_yaml,
 )
+from rasa.shared.exceptions import RasaException
 
 KEY_ANONYMIZATION_RULES = "anonymization"
 

rasa/anonymization/anonymization_pipeline.py

@@ -7,9 +7,6 @@ import queue
 from typing import Any, Dict, List, Optional, Text
 
 from apscheduler.schedulers.background import BackgroundScheduler
-from rasa.core.brokers.kafka import KafkaEventBroker
-from rasa.shared.core.events import Event
-from rasa.utils.endpoints import EndpointConfig, read_endpoint_config
 
 from rasa.anonymization.anonymisation_rule_yaml_reader import (
     AnonymizationRulesYamlReader,
@@ -18,6 +15,9 @@ from rasa.anonymization.anonymization_rule_executor import AnonymizationRuleList
 from rasa.anonymization.anonymization_rule_orchestrator import (
     AnonymizationRuleOrchestrator,
 )
+from rasa.core.brokers.kafka import KafkaEventBroker
+from rasa.shared.core.events import Event
+from rasa.utils.endpoints import EndpointConfig, read_endpoint_config
 from rasa.utils.singleton import Singleton
 
 logger = logging.getLogger(__name__)

rasa/anonymization/anonymization_rule_executor.py

@@ -12,8 +12,8 @@ from presidio_analyzer.nlp_engine import (
 )
 from presidio_anonymizer import AnonymizerEngine
 from presidio_anonymizer.entities import OperatorConfig
-from rasa.shared.exceptions import RasaException
 
+from rasa.shared.exceptions import RasaException
 from rasa.utils.singleton import Singleton
 
 if typing.TYPE_CHECKING:

rasa/anonymization/anonymization_rule_orchestrator.py

@@ -3,13 +3,12 @@ import json
 import logging
 from typing import Any, Dict, List, Optional, Text
 
-from rasa.core.brokers.broker import EventBroker
-from rasa.core.brokers.kafka import KafkaEventBroker
-
 from rasa.anonymization.anonymization_rule_executor import (
     AnonymizationRuleExecutor,
     AnonymizationRuleList,
 )
+from rasa.core.brokers.broker import EventBroker
+from rasa.core.brokers.kafka import KafkaEventBroker
 
 logger = logging.getLogger(__name__)
 

rasa/cli/arguments/data.py

@@ -2,10 +2,10 @@ import argparse
 from typing import Text
 
 from rasa.cli.arguments.default_arguments import (
-    add_nlu_data_param,
-    add_out_param,
     add_data_param,
     add_domain_param,
+    add_nlu_data_param,
+    add_out_param,
 )
 from rasa.shared.constants import DEFAULT_CONVERTED_DATA_PATH
 

rasa/cli/arguments/evaluate.py

@@ -1,6 +1,7 @@
 import argparse
 from pathlib import Path
-from rasa.cli.arguments.default_arguments import add_endpoint_param, add_domain_param
+
+from rasa.cli.arguments.default_arguments import add_domain_param, add_endpoint_param
 
 
 def set_markers_arguments(parser: argparse.ArgumentParser) -> None:

rasa/cli/arguments/interactive.py

@@ -2,8 +2,8 @@ import argparse
 import uuid
 
 import rasa.cli.arguments.default_arguments
-import rasa.cli.arguments.train
 import rasa.cli.arguments.run
+import rasa.cli.arguments.train
 
 
 def set_interactive_arguments(parser: argparse.ArgumentParser) -> None:

rasa/cli/arguments/run.py

@@ -5,8 +5,8 @@ from typing import Union
 from rasa.cli.arguments.default_arguments import (
     add_endpoint_param,
     add_model_param,
-    add_skip_validation_flag,
     add_remote_storage_param,
+    add_skip_validation_flag,
 )
 from rasa.core import constants
 from rasa.env import (

rasa/cli/arguments/test.py

@@ -1,17 +1,19 @@
 import argparse
 from typing import Union
 
-from rasa.shared.constants import DEFAULT_MODELS_PATH, DEFAULT_RESULTS_PATH
-
 from rasa.cli.arguments.default_arguments import (
-    add_stories_param,
+    add_endpoint_param,
     add_model_param,
     add_nlu_data_param,
-    add_endpoint_param,
     add_out_param,
+    add_stories_param,
 )
 from rasa.model import get_latest_model
-from rasa.shared.constants import DEFAULT_DOMAIN_PATH
+from rasa.shared.constants import (
+    DEFAULT_DOMAIN_PATH,
+    DEFAULT_MODELS_PATH,
+    DEFAULT_RESULTS_PATH,
+)
 
 
 def set_test_arguments(parser: argparse.ArgumentParser) -> None:

rasa/cli/arguments/train.py

@@ -3,12 +3,12 @@ from typing import Union
 
 from rasa.cli.arguments.default_arguments import (
     add_config_param,
-    add_stories_param,
-    add_nlu_data_param,
-    add_out_param,
     add_domain_param,
     add_endpoint_param,
+    add_nlu_data_param,
+    add_out_param,
     add_remote_storage_param,
+    add_stories_param,
 )
 from rasa.graph_components.providers.training_tracker_provider import (
     TrainingTrackerProvider,

rasa/cli/arguments/visualize.py

@@ -2,9 +2,9 @@ import argparse
 
 from rasa.cli.arguments.default_arguments import (
     add_domain_param,
-    add_stories_param,
-    add_out_param,
     add_nlu_data_param,
+    add_out_param,
+    add_stories_param,
 )
 
 

rasa/cli/arguments/x.py

@@ -1,4 +1,5 @@
 import argparse
+
 from rasa.cli.arguments import default_arguments
 from rasa.cli.arguments.run import add_server_arguments
 

rasa/cli/data.py

@@ -19,8 +19,8 @@ from rasa.e2e_test.e2e_config import create_llm_e2e_test_converter_config
 from rasa.e2e_test.e2e_test_converter import E2ETestConverter
 from rasa.e2e_test.utils.e2e_yaml_utils import E2ETestYAMLWriter
 from rasa.shared.constants import (
-    DEFAULT_DATA_PATH,
     DEFAULT_CONFIG_PATH,
+    DEFAULT_DATA_PATH,
     DEFAULT_DOMAIN_PATHS,
 )
 from rasa.shared.exceptions import RasaException
@@ -230,11 +230,12 @@ def split_stories_data(args: argparse.Namespace) -> None:
     Args:
         args: Commandline arguments
     """
+    from sklearn.model_selection import train_test_split
+
     from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
-        YAMLStoryReader,
         KEY_STORIES,
+        YAMLStoryReader,
     )
-    from sklearn.model_selection import train_test_split
 
     data_path = rasa.cli.utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
     data_files = rasa.shared.data.get_data_files(

rasa/cli/dialogue_understanding_test.py

@@ -1,20 +1,25 @@
 import argparse
 import asyncio
 import datetime
+import importlib
 import sys
-from typing import List
+from typing import List, Optional
 
 import structlog
 
 import rasa.cli.utils
+import rasa.shared.utils.cli
 from rasa.cli import SubParsersAction
 from rasa.cli.arguments.default_arguments import (
     add_endpoint_param,
     add_model_param,
     add_remote_storage_param,
 )
+from rasa.core.agent import Agent
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.utils import AvailableEndpoints
+from rasa.dialogue_understanding.commands import Command
+from rasa.dialogue_understanding.generator.command_parser import DEFAULT_COMMANDS
 from rasa.dialogue_understanding_test.command_metric_calculation import (
     calculate_command_metrics,
 )
@@ -24,6 +29,7 @@ from rasa.dialogue_understanding_test.constants import (
 )
 from rasa.dialogue_understanding_test.du_test_result import (
     DialogueUnderstandingTestResult,
+    DialogueUnderstandingTestSuiteResult,
 )
 from rasa.dialogue_understanding_test.du_test_runner import (
     DialogueUnderstandingTestRunner,
@@ -31,7 +37,6 @@ from rasa.dialogue_understanding_test.du_test_runner import (
 from rasa.dialogue_understanding_test.io import (
     read_test_suite,
     write_test_results_to_file,
-    print_test_results,
 )
 from rasa.dialogue_understanding_test.validation import (
     validate_cli_arguments,
@@ -39,7 +44,9 @@ from rasa.dialogue_understanding_test.validation import (
 )
 from rasa.e2e_test.e2e_test_case import TestSuite
 from rasa.exceptions import RasaException
-from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH
+from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH, ROUTE_TO_CALM_SLOT
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.flows import FlowsList
 from rasa.utils.beta import ensure_beta_feature_is_enabled
 from rasa.utils.endpoints import EndpointConfig
 
@@ -136,6 +143,45 @@ def add_du_test_arguments(parser: argparse.ArgumentParser) -> None:
         help="If set, the dialogue understanding test output will contain "
         "prompts for each failure.",
     )
+    du_arguments.add_argument(
+        "--additional-commands",
+        type=str,
+        nargs="*",
+        help=(
+            "List of additional custom command classes to add, separated by spaces. "
+            "For example: --additional-commands my_module.MyCustomCommand"
+        ),
+    )
+    du_arguments.add_argument(
+        "--remove-default-commands",
+        type=str,
+        nargs="*",
+        help=(
+            f"List of default commands to remove, separated by spaces. "
+            f"Default commands include: "
+            f"{', '.join([command.__name__ for command in DEFAULT_COMMANDS])}. "
+            f"For example: --remove-default-commands ClarifyCommand HumanHandoffCommand"
+        ),
+    )
+
+
+def ensure_calm_only_bot(agent: Agent) -> None:
+    if agent.domain is None or agent.processor is None:
+        return
+
+    if ROUTE_TO_CALM_SLOT in [slot.name for slot in agent.domain.slots]:
+        rasa.shared.utils.cli.print_error(
+            "You are using coexistence. Dialogue Understanding Tests do only work for "
+            "CALM only assistants."
+        )
+        sys.exit(0)
+
+    if not agent.processor.is_calm_assistant:
+        rasa.shared.utils.cli.print_error(
+            "Dialogue Understanding Tests do only work for CALM assistants. "
+            "Your assistant does not use CALM."
+        )
+        sys.exit(0)
 
 
 def execute_dialogue_understanding_tests(args: argparse.Namespace) -> None:
@@ -155,12 +201,6 @@ def execute_dialogue_understanding_tests(args: argparse.Namespace) -> None:
     # initialization of endpoints
     endpoints = set_up_available_endpoints(args)
 
-    # read test cases from the given path
-    test_suite = get_valid_test_suite(args)
-
-    # setup stub custom actions if they are used
-    set_up_stub_custom_actions(test_suite, endpoints)
-
     # set up the test runner, e.g. start the agent
     try:
         test_runner = DialogueUnderstandingTestRunner(
@@ -175,34 +215,97 @@ def execute_dialogue_understanding_tests(args: argparse.Namespace) -> None:
         )
         sys.exit(1)
 
+    # Exit if the bot is not calm only
+    ensure_calm_only_bot(test_runner.agent)
+
+    # Get flows from the agent
+    # we need them to parse the commands when reading the test cases
+    if test_runner.agent.processor is None:
+        rasa.shared.utils.cli.print_error(
+            "No processor: Not able to retrieve flows and config from trained model."
+        )
+        sys.exit(0)
+    flows = asyncio.run(test_runner.agent.processor.get_flows())
+
+    # read test cases from the given path
+    test_suite: TestSuite = get_valid_test_suite(args, flows, test_runner.agent.domain)
+
+    # setup stub custom actions if they are used
+    set_up_stub_custom_actions(test_suite, endpoints)
+
     # run the actual test cases
     test_results = asyncio.run(
-        test_runner.run_tests(
+        test_runner.run_test_cases(
             test_suite.test_cases, test_suite.fixtures, test_suite.metadata
         )
     )
 
     # evaluate test results
-    failed_tests, passed_tests = split_test_results(test_results)
+    passing_test_results, failing_test_results = split_test_results(test_results)
     command_metrics = calculate_command_metrics(test_results)
 
+    test_suite_result = DialogueUnderstandingTestSuiteResult.from_results(
+        failing_test_results, passing_test_results, command_metrics
+    )
+
+    # Do not move this import to the top of the file as it will break the
+    # instrumentation of this function: the CLI module is initialized before the
+    # instrumentation is set up, and we won't be able to "replace" the function
+    # with the instrumented wrapper
+    from rasa.dialogue_understanding_test.io import print_test_results
+
     # write results to console and file
-    print_test_results(failed_tests, passed_tests, command_metrics, args.output_prompt)
+    print_test_results(test_suite_result, output_prompt=args.output_prompt)
     if not args.no_output:
         write_test_results_to_file(
-            failed_tests,
-            passed_tests,
-            command_metrics,
+            test_suite_result,
             args.output_file,
             args.output_prompt,
         )
 
 
-def get_valid_test_suite(args: argparse.Namespace) -> TestSuite:
+def _import_custom_command_class(class_path: str) -> Command:
+    """Dynamically import a command class from a string path."""
+    try:
+        module_name, class_name = class_path.rsplit(".", 1)
+        module = importlib.import_module(module_name)
+        clz = getattr(module, class_name)
+    except (ImportError, AttributeError, ValueError) as e:
+        raise ValueError(f"Failed to import class '{class_path}': {e}")
+    if not issubclass(clz, Command):
+        structlogger.error(
+            "rasa.dialogue_understanding_test.invalid_additional_command",
+            event_info="The custom command class must be a subclass of Command.",
+            class_path=class_path,
+        )
+        sys.exit(1)
+    return clz
+
+
+def _extract_additional_command_classes_from_cli_args(
+    args: argparse.Namespace,
+) -> List[Command]:
+    """Extract additional command classes from the CLI arguments."""
+    additional_commands = getattr(args, "additional_commands", [])
+    if not additional_commands:
+        return []
+    return [
+        _import_custom_command_class(command_module)
+        for command_module in additional_commands
+    ]
+
+
+def get_valid_test_suite(
+    args: argparse.Namespace, flows: FlowsList, domain: Optional[Domain]
+) -> TestSuite:
     """Read the test cases from the given test case path and validate them."""
     path_to_test_cases = getattr(args, "path-to-test-cases", DEFAULT_INPUT_TESTS_PATH)
-    test_suite = read_test_suite(path_to_test_cases)
-    validate_test_cases(test_suite.test_cases)
+    remove_default_commands = getattr(args, "remove_default_commands", [])
+    custom_command_classes = _extract_additional_command_classes_from_cli_args(args)
+    test_suite = read_test_suite(
+        path_to_test_cases, flows, custom_command_classes, remove_default_commands
+    )
+    validate_test_cases(test_suite.test_cases, domain)
     return test_suite
 
 

rasa/cli/evaluate.py

@@ -1,6 +1,6 @@
 import argparse
 from pathlib import Path
-from typing import List, Text, Optional
+from typing import List, Optional, Text
 
 import rasa.cli.arguments.evaluate as arguments
 import rasa.shared.utils.cli

rasa/cli/export.py

@@ -2,23 +2,23 @@ import argparse
 import asyncio
 import logging
 import typing
-from typing import List, Text, Optional
+from typing import List, Optional, Text
 
-from rasa import telemetry
-from rasa.cli import SubParsersAction
 import rasa.core.utils
 import rasa.shared.utils.cli
 import rasa.utils.common
+from rasa import telemetry
+from rasa.cli import SubParsersAction
 from rasa.cli.arguments import export as arguments
-from rasa.shared.constants import DOCS_URL_EVENT_BROKERS, DOCS_URL_TRACKER_STORES
+from rasa.core.brokers.pika import PikaEventBroker
 from rasa.exceptions import PublishingError
+from rasa.shared.constants import DOCS_URL_EVENT_BROKERS, DOCS_URL_TRACKER_STORES
 from rasa.shared.exceptions import RasaException
-from rasa.core.brokers.pika import PikaEventBroker
 
 if typing.TYPE_CHECKING:
     from rasa.core.brokers.broker import EventBroker
-    from rasa.core.tracker_store import TrackerStore
     from rasa.core.exporter import Exporter
+    from rasa.core.tracker_store import TrackerStore
     from rasa.core.utils import AvailableEndpoints
 
 logger = logging.getLogger(__name__)

rasa/cli/interactive.py

@@ -4,11 +4,13 @@ import os
 from pathlib import Path
 from typing import List, Optional, Text, Union
 
+import rasa.cli.train as train
+import rasa.cli.utils
+import rasa.shared.utils.cli
+import rasa.utils.common
 from rasa import model
 from rasa.cli import SubParsersAction
 from rasa.cli.arguments import interactive as arguments
-import rasa.cli.train as train
-import rasa.cli.utils
 from rasa.engine.storage.local_model_storage import LocalModelStorage
 from rasa.shared.constants import (
     ASSISTANT_ID_DEFAULT_VALUE,
@@ -18,9 +20,6 @@ from rasa.shared.constants import (
 )
 from rasa.shared.data import TrainingType
 from rasa.shared.importers.importer import TrainingDataImporter
-import rasa.shared.utils.cli
-import rasa.utils.common
-
 
 logger = logging.getLogger(__name__)
 

rasa/cli/llm_fine_tuning.py

@@ -1,7 +1,7 @@
 import argparse
 import asyncio
 import sys
-from typing import List, Any, Dict
+from typing import Any, Dict, List
 
 import structlog
 
@@ -16,9 +16,9 @@ from rasa.cli.arguments.default_arguments import (
     add_remote_storage_param,
 )
 from rasa.cli.e2e_test import (
+    RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
     read_test_cases,
     validate_model_path,
-    RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
 )
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.utils import AvailableEndpoints
@@ -31,14 +31,14 @@ from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
 )
 from rasa.llm_fine_tuning.paraphrasing_module import create_paraphrased_conversations
 from rasa.llm_fine_tuning.storage import (
+    FileStorageStrategy,
     StorageContext,
     StorageType,
-    FileStorageStrategy,
 )
 from rasa.llm_fine_tuning.train_test_split_module import (
-    split_llm_fine_tuning_data,
-    INSTRUCTION_DATA_FORMAT,
     CONVERSATIONAL_DATA_FORMAT,
+    INSTRUCTION_DATA_FORMAT,
+    split_llm_fine_tuning_data,
 )
 from rasa.shared.constants import (
     DEFAULT_ENDPOINTS_PATH,

rasa/cli/markers.py

@@ -4,11 +4,10 @@ from typing import List, Text
 
 from rasa.cli import SubParsersAction
 from rasa.cli.arguments.default_arguments import add_domain_param
+from rasa.markers.upload import upload
 from rasa.shared.core.domain import Domain
 from rasa.shared.utils.cli import print_error_and_exit
 
-from rasa.markers.upload import upload
-
 
 def add_subparser(
     subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]

rasa/cli/project_templates/calm/actions/add_contact.py

@@ -4,7 +4,7 @@ from rasa_sdk import Action, Tracker
 from rasa_sdk.events import SlotSet
 from rasa_sdk.executor import CollectingDispatcher
 
-from actions.db import add_contact, get_contacts, Contact
+from actions.db import Contact, add_contact, get_contacts
 
 
 class AddContact(Action):

rasa/cli/project_templates/tutorial/actions/actions.py

@@ -1,7 +1,8 @@
-from typing import Any, Text, Dict, List
+from typing import Any, Dict, List, Text
+
 from rasa_sdk import Action, Tracker
-from rasa_sdk.executor import CollectingDispatcher
 from rasa_sdk.events import SlotSet
+from rasa_sdk.executor import CollectingDispatcher
 
 
 class ActionCheckSufficientFunds(Action):

rasa/cli/shell.py

@@ -1,18 +1,17 @@
 import argparse
 import logging
 import uuid
-
 from typing import List
 
 from rasa import telemetry
 from rasa.cli import SubParsersAction
 from rasa.cli.arguments import shell as arguments
 from rasa.engine.storage.local_model_storage import LocalModelStorage
+from rasa.exceptions import ModelNotFound
 from rasa.model import get_local_model
 from rasa.shared.constants import ASSISTANT_ID_KEY
 from rasa.shared.data import TrainingType
 from rasa.shared.utils.cli import print_error
-from rasa.exceptions import ModelNotFound
 
 logger = logging.getLogger(__name__)
 
@@ -63,9 +62,9 @@ def add_subparser(
 
 def shell_nlu(args: argparse.Namespace) -> None:
     """Talk with an NLU only bot though the command line."""
+    import rasa.nlu.run
     from rasa.cli.utils import get_validated_path
     from rasa.shared.constants import DEFAULT_MODELS_PATH
-    import rasa.nlu.run
 
     args.connector = "cmdline"
 

rasa/cli/studio/download.py

@@ -7,8 +7,7 @@ from rasa.cli.arguments.default_arguments import (
     add_endpoint_param,
 )
 from rasa.cli.arguments.train import add_data_param, add_domain_param
-from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH, DEFAULT_CONFIG_PATH
-
+from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_ENDPOINTS_PATH
 from rasa.studio.download import handle_download
 
 

rasa/cli/studio/studio.py

@@ -3,16 +3,15 @@ from typing import List, Optional, Tuple
 from urllib.parse import ParseResult, urlparse
 
 import questionary
-from rasa.cli import SubParsersAction
 
-import rasa.shared.utils.cli
 import rasa.cli.studio.download
 import rasa.cli.studio.train
 import rasa.cli.studio.upload
+import rasa.shared.utils.cli
+from rasa.cli import SubParsersAction
 from rasa.studio.auth import StudioAuth
 from rasa.studio.config import StudioConfig
 
-
 DEFAULT_REALM_NAME = "rasa-studio"
 
 DEFAULT_CLIENT_ID = "admin-cli"

rasa/cli/studio/train.py

@@ -3,7 +3,6 @@ from typing import List
 
 from rasa.cli import SubParsersAction
 from rasa.cli.arguments.train import set_train_arguments
-
 from rasa.studio.train import handle_train