rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/cli/arguments/train.py

@@ -0,0 +1,279 @@
+import argparse
+from typing import Union
+
+from rasa.cli.arguments.default_arguments import (
+    add_config_param,
+    add_stories_param,
+    add_nlu_data_param,
+    add_out_param,
+    add_domain_param,
+    add_endpoint_param,
+    add_remote_storage_param,
+)
+from rasa.graph_components.providers.training_tracker_provider import (
+    TrainingTrackerProvider,
+)
+from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH
+
+USE_LATEST_MODEL_FOR_FINE_TUNING = True
+
+
+def set_train_arguments(parser: argparse.ArgumentParser) -> None:
+    """Specifies CLI arguments for `rasa train`."""
+    add_data_param(parser)
+    add_config_param(parser)
+    add_domain_param(parser)
+    add_out_param(parser, help_text="Directory where your models should be stored.")
+
+    add_dry_run_param(parser)
+    add_validate_before_train(parser)
+    add_augmentation_param(parser)
+    add_debug_plots_param(parser)
+
+    _add_num_threads_param(parser)
+
+    _add_model_name_param(parser)
+    add_persist_nlu_data_param(parser)
+    add_keep_local_model_copy_param(parser)
+    add_force_param(parser)
+    add_finetune_params(parser)
+    add_endpoint_param(
+        parser, help_text="Configuration file for the connectors as a yml file."
+    )
+    add_remote_storage_param(parser)
+
+
+def set_train_core_arguments(parser: argparse.ArgumentParser) -> None:
+    """Specifies CLI arguments for `rasa train core`."""
+    add_stories_param(parser)
+    add_domain_param(parser)
+    _add_core_config_param(parser)
+    add_out_param(parser, help_text="Directory where your models should be stored.")
+
+    add_augmentation_param(parser)
+    add_debug_plots_param(parser)
+
+    add_force_param(parser)
+
+    _add_model_name_param(parser)
+
+    compare_arguments = parser.add_argument_group("Comparison Arguments")
+    _add_compare_params(compare_arguments)
+    add_finetune_params(parser)
+
+
+def set_train_nlu_arguments(parser: argparse.ArgumentParser) -> None:
+    """Specifies CLI arguments for `rasa train nlu`."""
+    add_config_param(parser)
+    add_domain_param(parser, default=None)
+    add_out_param(parser, help_text="Directory where your models should be stored.")
+
+    add_nlu_data_param(parser, help_text="File or folder containing your NLU data.")
+
+    _add_num_threads_param(parser)
+
+    _add_model_name_param(parser)
+    add_persist_nlu_data_param(parser)
+    add_finetune_params(parser)
+
+
+def add_force_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Specifies if the model should be trained from scratch."""
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Force a model training even if the data has not changed.",
+    )
+
+
+def add_data_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Specifies path to training data."""
+    parser.add_argument(
+        "--data",
+        default=[DEFAULT_DATA_PATH],
+        nargs="+",
+        help="Paths to the Core and NLU data files.",
+    )
+
+
+def _add_core_config_param(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "-c",
+        "--config",
+        nargs="+",
+        default=[DEFAULT_CONFIG_PATH],
+        help="The policy and NLU pipeline configuration of your bot. "
+        "If multiple configuration files are provided, multiple Rasa Core "
+        "models are trained to compare policies.",
+    )
+
+
+def _add_compare_params(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    parser.add_argument(
+        "--percentages",
+        nargs="*",
+        type=int,
+        default=[0, 25, 50, 75],
+        help="Range of exclusion percentages.",
+    )
+    parser.add_argument(
+        "--runs", type=int, default=3, help="Number of runs for experiments."
+    )
+
+
+def add_dry_run_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Adds `--dry-run` argument to a specified `parser`.
+
+    Args:
+        parser: An instance of `ArgumentParser` or `_ActionsContainer`.
+    """
+    parser.add_argument(
+        "--dry-run",
+        default=False,
+        action="store_true",
+        help="If enabled, no actual training will be performed. Instead, "
+        "it will be determined whether a model should be re-trained "
+        "and this information will be printed as the output. The return "
+        "code is a 4-bit bitmask that can also be used to determine what exactly needs "
+        "to be retrained:\n"
+        "- 0 means that no extensive training is required (note that the responses "
+        "still might require updating by running 'rasa train').\n"
+        "- 1 means the model needs to be retrained\n"
+        "- 8 means the training was forced (--force argument is specified)",
+    )
+
+
+def add_validate_before_train(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Adds parameters for validating the domain and data files before training.
+
+    Args:
+        parser: An instance of `ArgumentParser` or `_ActionsContainer`.
+    """
+    parser.add_argument(
+        "--skip-validation",
+        default=False,
+        action="store_true",
+        help="Skip validation step before training.",
+    )
+
+    parser.add_argument(
+        "--fail-on-validation-warnings",
+        default=False,
+        action="store_true",
+        help="Fail on validation warnings. "
+        "If omitted only errors will exit with a non zero status code",
+    )
+
+    parser.add_argument(
+        "--validation-max-history",
+        type=int,
+        default=None,
+        help="Number of turns taken into account for story structure validation.",
+    )
+
+
+def add_augmentation_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Sets the augmentation factor for the Core training.
+
+    Args:
+        parser: An instance of `ArgumentParser` or `_ActionsContainer`.
+    """
+    parser.add_argument(
+        "--augmentation",
+        type=int,
+        default=TrainingTrackerProvider.get_default_config()["augmentation_factor"],
+        help="How much data augmentation to use during training.",
+    )
+
+
+def add_debug_plots_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Specifies if conversation flow should be visualized."""
+    parser.add_argument(
+        "--debug-plots",
+        default=TrainingTrackerProvider.get_default_config()["debug_plots"],
+        action="store_true",
+        help="If enabled, will create plots showing checkpoints "
+        "and their connections between story blocks in a  "
+        "file called `story_blocks_connections.html`.",
+    )
+
+
+def _add_num_threads_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    parser.add_argument(
+        "--num-threads",
+        type=int,
+        help="Maximum amount of threads to use when training.",
+    )
+
+
+def _add_model_name_param(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--fixed-model-name",
+        type=str,
+        help="If set, the name of the model file/directory will be set to the given "
+        "name.",
+    )
+
+
+def add_persist_nlu_data_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Adds parameters for persisting the NLU training data with the model."""
+    parser.add_argument(
+        "--persist-nlu-data",
+        action="store_true",
+        help="Persist the NLU training data in the saved model.",
+    )
+
+
+def add_keep_local_model_copy_param(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Adds parameters for keeping a local copy of the model."""
+    parser.add_argument(
+        "--keep-local-model-copy",
+        action="store_true",
+        help="Keep a copy of the model in the model directory if remote "
+        "model upload is configured. Defaults to `false`, which "
+        "deletes the local copy of the model after upload.",
+    )
+
+
+def add_finetune_params(
+    parser: Union[argparse.ArgumentParser, argparse._ActionsContainer],
+) -> None:
+    """Adds parameters for model finetuning."""
+    parser.add_argument(
+        "--finetune",
+        nargs="?",
+        # If the user doesn't specify `--finetune` at all
+        default=None,
+        # If the user only specifies `--finetune` without an additional path
+        const=USE_LATEST_MODEL_FOR_FINE_TUNING,
+        help="Fine-tune a previously trained model. If no model path is provided, Rasa "
+        "Open Source will try to finetune the latest trained model from the "
+        "model directory specified via '--out'.",
+    )
+
+    parser.add_argument(
+        "--epoch-fraction",
+        type=float,
+        help="Fraction of epochs which are currently specified in the model "
+        "configuration which should be used when finetuning a model.",
+    )

rasa/cli/arguments/visualize.py

@@ -0,0 +1,34 @@
+import argparse
+
+from rasa.cli.arguments.default_arguments import (
+    add_domain_param,
+    add_stories_param,
+    add_out_param,
+    add_nlu_data_param,
+)
+
+
+def set_visualize_stories_arguments(parser: argparse.ArgumentParser) -> None:
+    """Sets the CLI arguments for `rasa data visualize."""
+    add_domain_param(parser)
+    add_stories_param(parser)
+
+    add_out_param(
+        parser,
+        default="graph.html",
+        help_text="Filename of the output path, e.g. 'graph.html'.",
+    )
+
+    parser.add_argument(
+        "--max-history",
+        default=2,
+        type=int,
+        help="Max history to consider when merging paths in the output graph.",
+    )
+
+    add_nlu_data_param(
+        parser,
+        default=None,
+        help_text="File or folder containing your NLU data, "
+        "used to insert example messages into the graph.",
+    )

rasa/cli/arguments/x.py

@@ -0,0 +1,30 @@
+import argparse
+from rasa.cli.arguments import default_arguments
+from rasa.cli.arguments.run import add_server_arguments
+
+
+def set_x_arguments(parser: argparse.ArgumentParser) -> None:
+    """Add CLI arguments for running rasa x --production."""
+    parser.add_argument(
+        "--production",
+        action="store_true",
+        help="Run a Rasa server in a mode that allows connecting "
+        "to Rasa Enterprise as the config endpoint",
+    )
+
+    parser.add_argument(
+        "--config-endpoint",
+        type=str,
+        help="Rasa X endpoint URL from which to pull the runtime config. This URL "
+        "typically contains the Rasa X token for authentication. Example: "
+        "https://example.com/api/config?token=my_rasa_x_token",
+    )
+
+    parser.add_argument(
+        "--no-prompt",
+        action="store_true",
+        help="Automatic yes or default options to prompts and oppressed warnings.",
+    )
+
+    default_arguments.add_model_param(parser, add_positional_arg=False)
+    add_server_arguments(parser)

rasa/cli/data.py

@@ -0,0 +1,354 @@
+import argparse
+import logging
+import pathlib
+from typing import List
+
+import rasa.cli.utils
+import rasa.shared.core.domain
+import rasa.shared.data
+import rasa.shared.nlu.training_data.loading
+import rasa.shared.nlu.training_data.util
+import rasa.shared.utils.cli
+import rasa.shared.utils.io
+import rasa.utils.common
+from rasa import telemetry
+from rasa.cli import SubParsersAction
+from rasa.cli.arguments import data as arguments
+from rasa.cli.arguments import default_arguments
+from rasa.e2e_test.e2e_config import create_llm_e2e_test_converter_config
+from rasa.e2e_test.e2e_test_converter import E2ETestConverter
+from rasa.e2e_test.utils.e2e_yaml_utils import E2ETestYAMLWriter
+from rasa.shared.constants import (
+    DEFAULT_DATA_PATH,
+    DEFAULT_CONFIG_PATH,
+    DEFAULT_DOMAIN_PATHS,
+)
+from rasa.shared.exceptions import RasaException
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.utils.common import minimal_kwargs
+from rasa.shared.utils.yaml import read_yaml_file, write_yaml
+from rasa.utils.beta import ensure_beta_feature_is_enabled
+
+logger = logging.getLogger(__name__)
+
+RASA_PRO_BETA_E2E_CONVERSION_ENV_VAR_NAME = "RASA_PRO_BETA_E2E_CONVERSION"
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add all data parsers.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    data_parser = subparsers.add_parser(
+        "data",
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Utils for the Rasa training files.",
+    )
+    data_parser.set_defaults(func=lambda _: data_parser.print_help(None))
+
+    data_subparsers = data_parser.add_subparsers()
+
+    _add_data_convert_parsers(data_subparsers, parents)
+    _add_data_split_parsers(data_subparsers, parents)
+    _add_data_validate_parsers(data_subparsers, parents)
+    _add_data_migrate_parsers(data_subparsers, parents)
+
+
+def _add_data_convert_parsers(
+    data_subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    convert_parser = data_subparsers.add_parser(
+        "convert",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Converts Rasa data between different formats.",
+    )
+    convert_parser.set_defaults(func=lambda _: convert_parser.print_help(None))
+
+    convert_subparsers = convert_parser.add_subparsers()
+    convert_nlu_parser = convert_subparsers.add_parser(
+        "nlu",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Converts NLU data between formats.",
+    )
+    convert_nlu_parser.set_defaults(func=_convert_nlu_data)
+    arguments.set_convert_nlu_arguments(convert_nlu_parser, data_type="Rasa NLU")
+
+    convert_e2e_parser = convert_subparsers.add_parser(
+        "e2e",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Convert input sample conversations into E2E test cases.",
+    )
+    convert_e2e_parser.set_defaults(func=convert_data_to_e2e_tests)
+    arguments.set_convert_e2e_arguments(convert_e2e_parser)
+
+
+def _add_data_split_parsers(
+    data_subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    split_parser = data_subparsers.add_parser(
+        "split",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Splits Rasa data into training and test data.",
+    )
+    split_parser.set_defaults(func=lambda _: split_parser.print_help(None))
+
+    split_subparsers = split_parser.add_subparsers()
+    nlu_split_parser = split_subparsers.add_parser(
+        "nlu",
+        parents=parents,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Performs a split of your NLU data into training and test data "
+        "according to the specified percentages.",
+    )
+    nlu_split_parser.set_defaults(func=split_nlu_data)
+
+    arguments.set_split_arguments(nlu_split_parser)
+
+    stories_split_parser = split_subparsers.add_parser(
+        "stories",
+        parents=parents,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Performs a split of your stories into training and test data "
+        "according to the specified percentages.",
+    )
+    stories_split_parser.set_defaults(func=split_stories_data)
+
+    arguments.set_split_arguments(stories_split_parser)
+
+
+def _add_data_validate_parsers(
+    data_subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    validate_parser = data_subparsers.add_parser(
+        "validate",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Validates domain and data files to check for possible mistakes.",
+    )
+    _append_story_structure_arguments(validate_parser)
+    validate_parser.set_defaults(
+        func=lambda args: rasa.cli.utils.validate_files(
+            args.fail_on_warnings, args.max_history, _build_training_data_importer(args)
+        )
+    )
+    arguments.set_validator_arguments(validate_parser)
+
+    validate_subparsers = validate_parser.add_subparsers()
+    story_structure_parser = validate_subparsers.add_parser(
+        "stories",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Checks for inconsistencies in the story files.",
+    )
+    _append_story_structure_arguments(story_structure_parser)
+
+    story_structure_parser.set_defaults(
+        func=lambda args: rasa.cli.utils.validate_files(
+            args.fail_on_warnings,
+            args.max_history,
+            _build_training_data_importer(args),
+            stories_only=True,
+        )
+    )
+    arguments.set_validator_arguments(story_structure_parser)
+
+    flows_structure_parser = validate_subparsers.add_parser(
+        "flows",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Checks for inconsistencies in the flows files.",
+    )
+    flows_structure_parser.set_defaults(
+        func=lambda args: rasa.cli.utils.validate_files(
+            args.fail_on_warnings,
+            args.max_history,
+            _build_training_data_importer(args),
+            flows_only=True,
+        )
+    )
+    arguments.set_validator_arguments(flows_structure_parser)
+
+
+def _build_training_data_importer(args: argparse.Namespace) -> "TrainingDataImporter":
+    config = rasa.cli.utils.get_validated_path(
+        args.config, "config", DEFAULT_CONFIG_PATH, none_is_valid=True
+    )
+
+    # Exit the validation if the domain path is invalid
+    domain = rasa.cli.utils.get_validated_path(
+        args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=False
+    )
+
+    return TrainingDataImporter.load_from_config(
+        domain_path=domain, training_data_paths=args.data, config_path=config
+    )
+
+
+def _append_story_structure_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--max-history",
+        type=int,
+        default=None,
+        help="Number of turns taken into account for story structure validation.",
+    )
+    default_arguments.add_config_param(parser)
+
+
+def split_nlu_data(args: argparse.Namespace) -> None:
+    """Load data from a file path and split the NLU data into test and train examples.
+
+    Args:
+        args: Commandline arguments
+    """
+    data_path = rasa.cli.utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
+    data_path = rasa.shared.data.get_nlu_directory(data_path)
+
+    nlu_data = rasa.shared.nlu.training_data.loading.load_data(data_path)
+    extension = rasa.shared.nlu.training_data.util.get_file_format_extension(data_path)
+
+    train, test = nlu_data.train_test_split(args.training_fraction, args.random_seed)
+
+    train.persist(args.out, filename=f"training_data{extension}")
+    test.persist(args.out, filename=f"test_data{extension}")
+
+    telemetry.track_data_split(args.training_fraction, "nlu")
+
+
+def split_stories_data(args: argparse.Namespace) -> None:
+    """Load data from a file path and split stories into test and train examples.
+
+    Args:
+        args: Commandline arguments
+    """
+    from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+        YAMLStoryReader,
+        KEY_STORIES,
+    )
+    from sklearn.model_selection import train_test_split
+
+    data_path = rasa.cli.utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
+    data_files = rasa.shared.data.get_data_files(
+        data_path, YAMLStoryReader.is_stories_file
+    )
+    out_path = pathlib.Path(args.out)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    # load Yaml stories data
+    for file_name in data_files:
+        file_data = read_yaml_file(file_name)
+        assert isinstance(file_data, dict)
+        stories = file_data.get(KEY_STORIES, [])
+        if not stories:
+            logger.info(f"File {file_name} has no stories, skipped")
+            continue
+
+        file_path = pathlib.Path(file_name)
+
+        # everything besides stories are going into the training data
+        train, test = train_test_split(
+            stories, test_size=1 - args.training_fraction, random_state=args.random_seed
+        )
+        out_file_train = out_path / ("train_" + file_path.name)
+        out_file_test = out_path / ("test_" + file_path.name)
+
+        # train file contains everything else from the file + train stories
+        file_data[KEY_STORIES] = train
+        write_yaml(file_data, out_file_train)
+
+        # test file contains just test stories
+        write_yaml({KEY_STORIES: test}, out_file_test)
+        logger.info(
+            f"From {file_name} we produced {out_file_train} "
+            f"with {len(train)} stories and {out_file_test} "
+            f"with {len(test)} stories"
+        )
+
+
+def _convert_nlu_data(args: argparse.Namespace) -> None:
+    import rasa.nlu.convert
+
+    if args.format in ["json", "yaml"]:
+        rasa.nlu.convert.convert_training_data(
+            args.data, args.out, args.format, args.language
+        )
+        telemetry.track_data_convert(args.format, "nlu")
+    else:
+        rasa.shared.utils.cli.print_error_and_exit(
+            "Could not recognize output format. Supported output formats: 'json' "
+            "and 'yaml'. Specify the desired output format with '--format'."
+        )
+
+
+def _add_data_migrate_parsers(
+    data_subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    migrate_parser = data_subparsers.add_parser(
+        "migrate",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=parents,
+        help="Converts Rasa domain 2.0 format to required format for 3.0.",
+    )
+    migrate_parser.set_defaults(func=_migrate_domain)
+
+    arguments.set_migrate_arguments(migrate_parser)
+
+
+def _migrate_domain(args: argparse.Namespace) -> None:
+    import rasa.core.migrate
+
+    rasa.core.migrate.migrate_domain_format(args.domain, args.out)
+
+
+def validate_e2e_test_conversion_output_path(output_path: str) -> None:
+    """Validates that the provided output path is within the project directory.
+
+    Args:
+        output_path (str): The output path to be validated.
+
+    Raises:
+        RasaException: If the provided output path is an absolute path.
+    """
+    if pathlib.Path(output_path).is_absolute():
+        raise RasaException(
+            "Please provide a relative output path within the assistant "
+            "project directory in which the command is running."
+        )
+
+
+def convert_data_to_e2e_tests(args: argparse.Namespace) -> None:
+    """Converts sample conversation data into E2E test cases
+    and stores them in the output YAML file.
+
+    Args:
+        args: The arguments passed in from the CLI.
+    """
+    try:
+        ensure_beta_feature_is_enabled(
+            "conversion of sample conversations into end-to-end tests",
+            RASA_PRO_BETA_E2E_CONVERSION_ENV_VAR_NAME,
+        )
+        validate_e2e_test_conversion_output_path(args.output)
+
+        config_path = pathlib.Path(args.output)
+        llm_config = create_llm_e2e_test_converter_config(config_path)
+
+        kwargs = minimal_kwargs(vars(args), E2ETestConverter)
+        converter = E2ETestConverter(llm_config=llm_config, **kwargs)
+        yaml_tests_string = converter.run()
+
+        writer = E2ETestYAMLWriter(output_path=args.output)
+        writer.write_to_file(yaml_tests_string)
+    except RasaException as exc:
+        rasa.shared.utils.cli.print_error_and_exit(
+            f"Failed to convert the data into E2E tests. Error: {exc}"
+        )