rasa-pro 3.9.18__py3-none-any.whl

rasa/cli/test.py

@@ -0,0 +1,280 @@
+import argparse
+import asyncio
+import logging
+import os
+from typing import List, Optional, Text, Dict, Union, Any
+
+from rasa.cli import SubParsersAction
+import rasa.shared.data
+from rasa.shared.exceptions import YamlException
+import rasa.shared.utils.io
+import rasa.shared.utils.cli
+from rasa.cli.arguments import test as arguments
+from rasa.core.constants import (
+    FAILED_STORIES_FILE,
+    SUCCESSFUL_STORIES_FILE,
+    STORIES_WITH_WARNINGS_FILE,
+)
+from rasa.shared.constants import (
+    CONFIG_SCHEMA_FILE,
+    DEFAULT_E2E_TESTS_PATH,
+    DEFAULT_CONFIG_PATH,
+    DEFAULT_MODELS_PATH,
+    DEFAULT_DATA_PATH,
+    DEFAULT_RESULTS_PATH,
+)
+from rasa.shared.utils.yaml import validate_raw_yaml_using_schema_file
+import rasa.cli.utils
+import rasa.utils.common
+from rasa.shared.importers.importer import TrainingDataImporter
+
+logger = logging.getLogger(__name__)
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add all test parsers.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    test_parser = subparsers.add_parser(
+        "test",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Tests Rasa models using your test NLU data and stories.",
+    )
+
+    arguments.set_test_arguments(test_parser)
+
+    test_subparsers = test_parser.add_subparsers()
+    test_core_parser = test_subparsers.add_parser(
+        "core",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Tests Rasa Core models using your test stories.",
+    )
+    arguments.set_test_core_arguments(test_core_parser)
+
+    test_nlu_parser = test_subparsers.add_parser(
+        "nlu",
+        parents=parents,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Tests Rasa NLU models using your test NLU data.",
+    )
+    arguments.set_test_nlu_arguments(test_nlu_parser)
+
+    test_core_parser.set_defaults(func=run_core_test)
+    test_nlu_parser.set_defaults(func=run_nlu_test)
+    test_parser.set_defaults(func=test, stories=DEFAULT_E2E_TESTS_PATH)
+
+
+def _print_core_test_execution_info(args: argparse.Namespace) -> None:
+    output = args.out or DEFAULT_RESULTS_PATH
+
+    if args.successes:
+        rasa.shared.utils.cli.print_info(
+            f"Successful stories written to "
+            f"'{os.path.join(output, SUCCESSFUL_STORIES_FILE)}'"
+        )
+    if not args.no_errors:
+        rasa.shared.utils.cli.print_info(
+            f"Failed stories written to '{os.path.join(output, FAILED_STORIES_FILE)}'"
+        )
+    if not args.no_warnings:
+        rasa.shared.utils.cli.print_info(
+            f"Stories with prediction warnings written to "
+            f"'{os.path.join(output, STORIES_WITH_WARNINGS_FILE)}'"
+        )
+
+
+async def run_core_test_async(args: argparse.Namespace) -> None:
+    """Run core tests."""
+    from rasa.model_testing import (
+        test_core_models_in_directory,
+        test_core,
+        test_core_models,
+    )
+
+    stories = rasa.cli.utils.get_validated_path(
+        args.stories, "stories", DEFAULT_DATA_PATH
+    )
+
+    output = args.out or DEFAULT_RESULTS_PATH
+    args.errors = not args.no_errors
+    args.warnings = not args.no_warnings
+
+    rasa.shared.utils.io.create_directory(output)
+
+    if isinstance(args.model, list) and len(args.model) == 1:
+        args.model = args.model[0]
+
+    if args.model is None:
+        rasa.shared.utils.cli.print_error(
+            "No model provided. Please make sure to specify "
+            "the model to test with '--model'."
+        )
+        return
+
+    if isinstance(args.model, str):
+        model_path = rasa.cli.utils.get_validated_path(
+            args.model, "model", DEFAULT_MODELS_PATH
+        )
+
+        if args.evaluate_model_directory:
+            await test_core_models_in_directory(
+                args.model, stories, output, use_conversation_test_files=args.e2e
+            )
+        else:
+            await test_core(
+                model=model_path,
+                stories=stories,
+                output=output,
+                additional_arguments=vars(args),
+                use_conversation_test_files=args.e2e,
+            )
+
+    else:
+        await test_core_models(
+            args.model, stories, output, use_conversation_test_files=args.e2e
+        )
+
+    _print_core_test_execution_info(args)
+
+
+async def run_nlu_test_async(
+    config: Optional[Union[Text, List[Text]]],
+    data_path: Text,
+    models_path: Text,
+    output_dir: Text,
+    cross_validation: bool,
+    percentages: List[int],
+    runs: int,
+    no_errors: bool,
+    domain_path: Text,
+    all_args: Dict[Text, Any],
+) -> None:
+    """Runs NLU tests.
+
+    Args:
+        all_args: all arguments gathered in a Dict so we can pass it as one argument
+                  to other functions.
+        config: it refers to the model configuration file. It can be a single file or
+                a list of multiple files or a folder with multiple config files inside.
+        data_path: path for the nlu data.
+        models_path: path to a trained Rasa model.
+        output_dir: output path for any files created during the evaluation.
+        cross_validation: indicates if it should test the model using cross validation
+                          or not.
+        percentages: defines the exclusion percentage of the training data.
+        runs: number of comparison runs to make.
+        domain_path: path to domain.
+        no_errors: indicates if incorrect predictions should be written to a file
+                   or not.
+    """
+    from rasa.model_testing import (
+        compare_nlu_models,
+        perform_nlu_cross_validation,
+        test_nlu,
+    )
+
+    data_path = str(
+        rasa.cli.utils.get_validated_path(data_path, "nlu", DEFAULT_DATA_PATH)
+    )
+    test_data_importer = TrainingDataImporter.load_from_dict(
+        training_data_paths=[data_path], domain_path=domain_path
+    )
+    nlu_data = test_data_importer.get_nlu_data()
+
+    output = output_dir or DEFAULT_RESULTS_PATH
+    all_args["errors"] = not no_errors
+    rasa.shared.utils.io.create_directory(output)
+
+    if config is not None and len(config) == 1:
+        config = os.path.abspath(config[0])
+        if os.path.isdir(config):
+            config = rasa.shared.utils.io.list_files(config)
+
+    if isinstance(config, list):
+        logger.info(
+            "Multiple configuration files specified, running nlu comparison mode."
+        )
+
+        config_files = []
+        for file in config:
+            try:
+                validate_raw_yaml_using_schema_file(
+                    rasa.shared.utils.io.read_file(file), CONFIG_SCHEMA_FILE
+                )
+                config_files.append(file)
+            except YamlException:
+                rasa.shared.utils.io.raise_warning(
+                    f"Ignoring file '{file}' as it is not a valid config file."
+                )
+                continue
+        await compare_nlu_models(
+            configs=config_files,
+            test_data=nlu_data,
+            output=output,
+            runs=runs,
+            exclusion_percentages=percentages,
+        )
+    elif cross_validation:
+        logger.info("Test model using cross validation.")
+        # FIXME: supporting Union[Path, Text] down the chain
+        # is the proper fix and needs more work
+        config = str(
+            rasa.cli.utils.get_validated_path(config, "config", DEFAULT_CONFIG_PATH)
+        )
+        config_importer = TrainingDataImporter.load_from_dict(config_path=config)
+
+        config_dict = config_importer.get_config()
+        await perform_nlu_cross_validation(config_dict, nlu_data, output, all_args)
+    else:
+        model_path = rasa.cli.utils.get_validated_path(
+            models_path, "model", DEFAULT_MODELS_PATH
+        )
+
+        await test_nlu(model_path, data_path, output, all_args, domain_path=domain_path)
+
+
+def run_nlu_test(args: argparse.Namespace) -> None:
+    """Runs NLU tests.
+
+    Args:
+        args: the parsed CLI arguments for 'rasa test nlu'.
+    """
+    asyncio.run(
+        run_nlu_test_async(
+            args.config,
+            args.nlu,
+            args.model,
+            args.out,
+            args.cross_validation,
+            args.percentages,
+            args.runs,
+            args.no_errors,
+            args.domain,
+            vars(args),
+        )
+    )
+
+
+def run_core_test(args: argparse.Namespace) -> None:
+    """Runs Core tests.
+
+    Args:
+        args: the parsed CLI arguments for 'rasa test core'.
+    """
+    asyncio.run(run_core_test_async(args))
+
+
+def test(args: argparse.Namespace) -> None:
+    """Run end-to-end tests."""
+    setattr(args, "e2e", True)
+    run_core_test(args)
+    run_nlu_test(args)

rasa/cli/train.py

@@ -0,0 +1,260 @@
+import argparse
+import structlog
+import sys
+import asyncio
+from pathlib import Path
+from typing import Dict, List, Optional, Text, Union
+
+from rasa.cli import SubParsersAction
+import rasa.cli.arguments.train as train_arguments
+
+import rasa.cli.utils
+import rasa.core.utils
+from rasa.shared.importers.importer import TrainingDataImporter
+import rasa.utils.common
+from rasa.core.nlg.generator import NaturalLanguageGenerator
+from rasa.core.train import do_compare_training
+from rasa.shared.constants import (
+    CONFIG_MANDATORY_KEYS_CORE,
+    CONFIG_MANDATORY_KEYS_NLU,
+    CONFIG_MANDATORY_KEYS,
+    DEFAULT_DATA_PATH,
+    DEFAULT_DOMAIN_PATHS,
+)
+
+structlogger = structlog.getLogger(__name__)
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add all training parsers.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    train_parser = subparsers.add_parser(
+        "train",
+        help="Trains a Rasa model using your NLU data and stories.",
+        parents=parents,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    train_arguments.set_train_arguments(train_parser)
+
+    train_subparsers = train_parser.add_subparsers()
+    train_core_parser = train_subparsers.add_parser(
+        "core",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Trains a Rasa Core model using your stories.",
+    )
+    train_core_parser.set_defaults(func=run_core_training)
+
+    train_nlu_parser = train_subparsers.add_parser(
+        "nlu",
+        parents=parents,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Trains a Rasa NLU model using your NLU data.",
+    )
+    train_nlu_parser.set_defaults(func=run_nlu_training)
+
+    train_parser.set_defaults(func=lambda args: run_training(args, can_exit=True))
+
+    train_arguments.set_train_core_arguments(train_core_parser)
+    train_arguments.set_train_nlu_arguments(train_nlu_parser)
+
+
+def _check_nlg_endpoint_validity(endpoint: Union[Path, str]) -> None:
+    try:
+        endpoints = rasa.core.utils.read_endpoints_from_path(endpoint)
+        NaturalLanguageGenerator.create(endpoints.nlg)
+    except Exception as e:
+        structlogger.error(
+            "cli.train.nlg_failed_to_initialise.validation_error",
+            exception=f"{e}",
+            event_info=(
+                f"The validation failed for NLG configuration defined in "
+                f"{endpoint}. Please make sure the NLG configuration is correct."
+            ),
+        )
+        sys.exit(1)
+
+
+def run_training(args: argparse.Namespace, can_exit: bool = False) -> Optional[Text]:
+    """Trains a model.
+
+    Args:
+        args: Namespace arguments.
+        can_exit: If `True`, the operation can send `sys.exit` in the case
+            training was not successful.
+
+    Returns:
+        Path to a trained model or `None` if training was not successful.
+    """
+    from rasa import train as train_all
+
+    domain = rasa.cli.utils.get_validated_path(
+        args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=True
+    )
+    config = rasa.cli.utils.get_validated_config(args.config, CONFIG_MANDATORY_KEYS)
+
+    _check_nlg_endpoint_validity(args.endpoints)
+
+    training_files = [
+        rasa.cli.utils.get_validated_path(
+            f, "data", DEFAULT_DATA_PATH, none_is_valid=True
+        )
+        for f in args.data
+    ]
+
+    if not args.skip_validation:
+        structlogger.info(
+            "cli.train.run_training",
+            event_info="Started validating domain and training data...",
+        )
+        importer = TrainingDataImporter.load_from_config(
+            domain_path=domain, training_data_paths=args.data, config_path=config
+        )
+        rasa.cli.utils.validate_files(
+            args.fail_on_validation_warnings, args.validation_max_history, importer
+        )
+
+    training_result = train_all(
+        domain=domain,
+        config=config,
+        training_files=training_files,
+        output=args.out,
+        dry_run=args.dry_run,
+        force_training=args.force,
+        fixed_model_name=args.fixed_model_name,
+        persist_nlu_training_data=args.persist_nlu_data,
+        core_additional_arguments={
+            **extract_core_additional_arguments(args),
+        },
+        nlu_additional_arguments=extract_nlu_additional_arguments(args),
+        model_to_finetune=_model_for_finetuning(args),
+        finetuning_epoch_fraction=args.epoch_fraction,
+    )
+    if training_result.code != 0 and can_exit:
+        sys.exit(training_result.code)
+
+    return training_result.model
+
+
+def _model_for_finetuning(args: argparse.Namespace) -> Optional[Text]:
+    if args.finetune == train_arguments.USE_LATEST_MODEL_FOR_FINE_TUNING:
+        # We use this constant to signal that the user specified `--finetune` but
+        # didn't provide a path to a model. In this case we try to load the latest
+        # model from the output directory (that's usually models/).
+        return args.out
+    else:
+        return args.finetune
+
+
+def run_core_training(args: argparse.Namespace) -> Optional[Text]:
+    """Trains a Rasa Core model only.
+
+    Args:
+        args: Command-line arguments to configure training.
+
+    Returns:
+        Path to a trained model or `None` if training was not successful.
+    """
+    from rasa.model_training import train_core
+
+    args.domain = rasa.cli.utils.get_validated_path(
+        args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=True
+    )
+    story_file = rasa.cli.utils.get_validated_path(
+        args.stories, "stories", DEFAULT_DATA_PATH, none_is_valid=True
+    )
+    additional_arguments = {
+        **extract_core_additional_arguments(args),
+    }
+
+    # Policies might be a list for the compare training. Do normal training
+    # if only list item was passed.
+    if not isinstance(args.config, list) or len(args.config) == 1:
+        if isinstance(args.config, list):
+            args.config = args.config[0]
+
+        config = rasa.cli.utils.get_validated_config(
+            args.config, CONFIG_MANDATORY_KEYS_CORE
+        )
+
+        return asyncio.run(
+            train_core(
+                domain=args.domain,
+                config=config,
+                stories=story_file,
+                output=args.out,
+                fixed_model_name=args.fixed_model_name,
+                additional_arguments=additional_arguments,
+                model_to_finetune=_model_for_finetuning(args),
+                finetuning_epoch_fraction=args.epoch_fraction,
+            )
+        )
+    else:
+        asyncio.run(do_compare_training(args, story_file, additional_arguments))
+        return None
+
+
+def run_nlu_training(args: argparse.Namespace) -> Optional[Text]:
+    """Trains an NLU model.
+
+    Args:
+        args: Namespace arguments.
+
+    Returns:
+        Path to a trained model or `None` if training was not successful.
+    """
+    from rasa.model_training import train_nlu
+
+    config = rasa.cli.utils.get_validated_config(args.config, CONFIG_MANDATORY_KEYS_NLU)
+    nlu_data = rasa.cli.utils.get_validated_path(
+        args.nlu, "nlu", DEFAULT_DATA_PATH, none_is_valid=True
+    )
+
+    if args.domain:
+        args.domain = rasa.cli.utils.get_validated_path(
+            args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=True
+        )
+
+    return asyncio.run(
+        train_nlu(
+            config=config,
+            nlu_data=nlu_data,
+            output=args.out,
+            fixed_model_name=args.fixed_model_name,
+            persist_nlu_training_data=args.persist_nlu_data,
+            additional_arguments={
+                **extract_nlu_additional_arguments(args),
+            },
+            domain=args.domain,
+            model_to_finetune=_model_for_finetuning(args),
+            finetuning_epoch_fraction=args.epoch_fraction,
+        )
+    )
+
+
+def extract_core_additional_arguments(args: argparse.Namespace) -> Dict:
+    arguments = {}
+
+    if "augmentation" in args:
+        arguments["augmentation_factor"] = args.augmentation
+    if "debug_plots" in args:
+        arguments["debug_plots"] = args.debug_plots
+
+    return arguments
+
+
+def extract_nlu_additional_arguments(args: argparse.Namespace) -> Dict:
+    arguments = {}
+
+    if "num_threads" in args:
+        arguments["num_threads"] = args.num_threads
+
+    return arguments