rasa-pro 3.9.18__py3-none-any.whl → 3.10.3__py3-none-any.whl

rasa/cli/llm_fine_tuning.py

@@ -0,0 +1,395 @@
+import argparse
+import asyncio
+import sys
+from typing import List, Any, Dict
+
+import structlog
+
+import rasa.cli.utils
+import rasa.shared.utils.cli
+import rasa.shared.utils.io
+import rasa.shared.utils.yaml
+from rasa.cli import SubParsersAction
+from rasa.cli.arguments.default_arguments import (
+    add_endpoint_param,
+    add_model_param,
+    add_remote_storage_param,
+)
+from rasa.cli.e2e_test import (
+    read_test_cases,
+    validate_model_path,
+    RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
+)
+from rasa.core.exceptions import AgentNotReady
+from rasa.core.utils import AvailableEndpoints
+from rasa.e2e_test.e2e_test_runner import E2ETestRunner
+from rasa.llm_fine_tuning.annotation_module import annotate_e2e_tests
+from rasa.llm_fine_tuning.llm_data_preparation_module import convert_to_fine_tuning_data
+from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
+    ConversationRephraser,
+)
+from rasa.llm_fine_tuning.paraphrasing_module import create_paraphrased_conversations
+from rasa.llm_fine_tuning.storage import (
+    StorageContext,
+    StorageType,
+    FileStorageStrategy,
+)
+from rasa.llm_fine_tuning.train_test_split_module import (
+    split_llm_fine_tuning_data,
+    INSTRUCTION_DATA_FORMAT,
+    CONVERSATIONAL_DATA_FORMAT,
+)
+from rasa.shared.constants import (
+    DEFAULT_ENDPOINTS_PATH,
+    DEFAULT_MODELS_PATH,
+    LLM_CONFIG_KEY,
+)
+from rasa.shared.utils.yaml import read_config_file
+from rasa.utils.beta import ensure_beta_feature_is_enabled
+
+DEFAULT_INPUT_E2E_TEST_PATH = "e2e_tests"
+DEFAULT_OUTPUT_FOLDER = "output"
+RESULT_SUMMARY_FILE = "result_summary.yaml"
+PARAMETERS_FILE = "params.yaml"
+
+structlogger = structlog.get_logger()
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add the llm fine-tuning subparser to `rasa test`.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    llm_parser = subparsers.add_parser(
+        "llm",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Commands related to LLMs.",
+    )
+    llm_subparsers = llm_parser.add_subparsers()
+
+    llm_finetune_parser = llm_subparsers.add_parser(
+        "finetune",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Commands related to LLM fine-tuning.",
+    )
+    llm_finetune_subparser = llm_finetune_parser.add_subparsers()
+
+    create_llm_finetune_data_preparation_subparser(llm_finetune_subparser, parents)
+
+
+def create_llm_finetune_data_preparation_subparser(
+    fine_tune_llm_parser: SubParsersAction,
+    parents: List[argparse.ArgumentParser],
+) -> argparse.ArgumentParser:
+    """Create fine-tuning LLM data preparation subparser."""
+    data_preparation_subparser = fine_tune_llm_parser.add_parser(
+        "prepare-data",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Prepares data for LLM fine-tuning.",
+    )
+
+    data_preparation_subparser.set_defaults(func=prepare_llm_fine_tuning_data)
+
+    add_data_preparation_arguments(data_preparation_subparser)
+    add_model_param(data_preparation_subparser, add_positional_arg=False)
+    add_endpoint_param(
+        data_preparation_subparser,
+        help_text="Configuration file for the model server and the connectors as a "
+        "yml file.",
+    )
+
+    return data_preparation_subparser
+
+
+def add_data_preparation_arguments(parser: argparse.ArgumentParser) -> None:
+    """Arguments for preparing LLM fine-tuning data."""
+    parser.add_argument(
+        "-o",
+        "--out",
+        type=str,
+        default=DEFAULT_OUTPUT_FOLDER,
+        help="The output folder to store the data to.",
+    )
+    parser.add_argument(
+        "path-to-e2e-test-cases",
+        nargs="?",
+        type=str,
+        help="Input file or folder containing end-to-end test cases.",
+        default=DEFAULT_INPUT_E2E_TEST_PATH,
+    )
+
+    add_remote_storage_param(parser)
+
+    rephrasing_arguments = parser.add_argument_group("Rephrasing Module")
+    rephrasing_arguments.add_argument(
+        "--num-rephrases",
+        choices=range(0, 50),
+        type=int,
+        default=10,
+        help="Number of rephrases to be generated per user utterance.",
+    )
+    rephrasing_arguments.add_argument(
+        "--rephrase-config",
+        type=str,
+        default=None,
+        help="Path to config file that contains the configuration of the "
+        "rephrasing module.",
+    )
+
+    train_test_split_arguments = parser.add_argument_group("Train/Test Split Module")
+    train_test_split_arguments.add_argument(
+        "--train-frac",
+        type=restricted_float,
+        default=0.8,
+        help="The amount of data that should go into the training dataset. The value "
+        "should be >0.0 and <=1.0.",
+    )
+    train_test_split_arguments.add_argument(
+        "--output-format",
+        choices=[INSTRUCTION_DATA_FORMAT, CONVERSATIONAL_DATA_FORMAT],
+        type=str,
+        nargs="?",
+        default=INSTRUCTION_DATA_FORMAT,
+        help="Format of the output file.",
+    )
+
+
+def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
+    """Prepare LLM fine-tuning data.
+
+    Args:
+        args: Commandline arguments.
+    """
+    ensure_beta_feature_is_enabled(
+        "LLM fine-tuning recipe",
+        env_flag=RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
+    )
+
+    rephrase_config = (
+        read_config_file(args.rephrase_config) if args.rephrase_config else {}
+    )
+    ConversationRephraser.validate_config(rephrase_config)
+
+    # make sure the output directory exists
+    output_dir = args.out
+    rasa.shared.utils.io.create_directory(output_dir)
+
+    # read e2e test cases
+    path_to_test_cases = getattr(
+        args, "path-to-e2e-test-cases", DEFAULT_INPUT_E2E_TEST_PATH
+    )
+    test_suite = read_test_cases(path_to_test_cases)
+    # set up the e2e test runner
+    e2e_test_runner = set_up_e2e_test_runner(args)
+
+    if e2e_test_runner.agent.processor is None:
+        rasa.shared.utils.cli.print_error(
+            "No processor: Not able to retrieve flows and config from trained model."
+        )
+        sys.exit(0)
+
+    flows = asyncio.run(e2e_test_runner.agent.processor.get_flows())
+    llm_command_generator_config = _get_llm_command_generator_config(e2e_test_runner)
+
+    # set up storage context
+    storage_context = create_storage_context(StorageType.FILE, output_dir)
+
+    statistics = {}
+
+    # 1. annotate e2e tests
+    log_start_of_module("Annotation")
+    conversations = annotate_e2e_tests(e2e_test_runner, test_suite, storage_context)
+    statistics["num_input_e2e_tests"] = len(test_suite.test_cases)
+    statistics["num_annotated_conversations"] = len(conversations)
+    statistics["num_user_messages_across_conversations"] = sum(
+        [len(conversation.get_user_messages()) for conversation in conversations]
+    )
+    statistics["num_user_messages_to_rephrase_across_conversations"] = sum(
+        [
+            len(conversation.get_user_messages_to_rephrase())
+            for conversation in conversations
+        ]
+    )
+    log_end_of_module("Annotation", statistics)
+
+    # 2. paraphrase conversations
+    log_start_of_module("Rephrasing")
+    conversations, rephrase_config = asyncio.run(
+        create_paraphrased_conversations(
+            conversations,
+            rephrase_config,
+            args.num_rephrases,
+            flows,
+            llm_command_generator_config,
+            storage_context,
+        )
+    )
+    statistics["num_passing_rephrased_user_messages"] = sum(
+        [conversation.get_number_of_rephrases(True) for conversation in conversations]
+    )
+    statistics["num_failing_rephrased_user_messages"] = sum(
+        [conversation.get_number_of_rephrases(False) for conversation in conversations]
+    )
+    log_end_of_module("Rephrasing", statistics)
+
+    # 3. create fine-tuning dataset
+    log_start_of_module("LLM Data Preparation")
+    llm_fine_tuning_data = convert_to_fine_tuning_data(conversations, storage_context)
+    statistics["num_ft_data_points"] = len(llm_fine_tuning_data)
+    log_end_of_module("LLM Data Preparation", statistics)
+
+    # 4. create train/test split
+    log_start_of_module("Train/Test Split")
+    train_data, val_data = split_llm_fine_tuning_data(
+        llm_fine_tuning_data,
+        args.train_frac,
+        args.output_format,
+        storage_context,
+        test_suite,
+    )
+    statistics["num_train_data_points"] = len(train_data)
+    statistics["num_val_data_points"] = len(val_data)
+    log_end_of_module("Train/Test Split", statistics)
+
+    # write down params and statistics to a file
+    write_params(args, rephrase_config, output_dir)
+    write_statistics(statistics, output_dir)
+
+    rasa.shared.utils.cli.print_success(
+        f"Data and intermediate results are written " f"to '{output_dir}'."
+    )
+
+
+def _get_llm_command_generator_config(e2e_test_runner: E2ETestRunner) -> Dict[str, Any]:
+    from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
+
+    train_schema = e2e_test_runner.agent.processor.model_metadata.train_schema  # type: ignore
+
+    for node in train_schema.nodes:
+        if "SingleStepLLMCommandGenerator" in node:
+            return {
+                **DEFAULT_LLM_CONFIG,
+                **train_schema.nodes[node].config.get(LLM_CONFIG_KEY),
+            }
+
+    rasa.shared.utils.cli.print_error(
+        "The provided model was not trained with the 'SingleStepLLMCommandGenerator'."
+        "Without the 'SingleStepLLMCommandGenerator' no data for fine-tuning can be "
+        "created. Please add the 'SingleStepLLMCommandGenerator' to your config and"
+        "train your model."
+    )
+    sys.exit(0)
+
+
+def log_start_of_module(module_name: str) -> None:
+    log_info = f"Starting {module_name} Module"
+    rasa.shared.utils.cli.print_info(
+        f"{rasa.shared.utils.cli.pad(log_info, char='-')}\n"
+    )
+
+
+def log_end_of_module(module_name: str, statistics: Dict[str, int]) -> None:
+    log_info = f"Finished {module_name} Module"
+    rasa.shared.utils.cli.print_info(
+        f"{rasa.shared.utils.cli.pad(log_info, char='-')}\n"
+    )
+    rasa.shared.utils.cli.print_color(
+        "Current Statistics:", color=rasa.shared.utils.io.bcolors.BOLD
+    )
+    for key, value in statistics.items():
+        rasa.shared.utils.cli.print_color(
+            f"  {key}: {value}", color=rasa.shared.utils.io.bcolors.BOLD
+        )
+
+
+def restricted_float(x: Any) -> float:
+    try:
+        x = float(x)
+    except ValueError:
+        raise argparse.ArgumentTypeError("%r not a floating-point literal" % (x,))
+
+    if x <= 0.0 or x > 1.0:
+        raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0]" % (x,))
+    return x
+
+
+def write_params(
+    args: argparse.Namespace, rephrase_config: Dict[str, Any], output_path: str
+) -> None:
+    yaml_data = {
+        "parameters": {
+            "num_rephrases": args.num_rephrases,
+            "rephrase_config": rephrase_config,
+            "model": args.model,
+            "endpoints": args.endpoints,
+            "remote-storage": args.remote_storage,
+            "train_frac": args.train_frac,
+            "output_format": args.output_format,
+            "out": output_path,
+        }
+    }
+
+    rasa.shared.utils.yaml.write_yaml(yaml_data, f"{output_path}/{PARAMETERS_FILE}")
+
+
+def write_statistics(statistics: Dict[str, Any], output_path: str) -> None:
+    rasa.shared.utils.yaml.write_yaml(
+        statistics, f"{output_path}/{RESULT_SUMMARY_FILE}"
+    )
+
+
+def get_valid_endpoints(endpoints_file: str) -> AvailableEndpoints:
+    validated_endpoints_file = rasa.cli.utils.get_validated_path(
+        endpoints_file, "endpoints", DEFAULT_ENDPOINTS_PATH, True
+    )
+    endpoints = AvailableEndpoints.read_endpoints(validated_endpoints_file)
+
+    # Ignore all endpoints apart from action server, model, nlu and nlg
+    # to ensure InMemoryTrackerStore is being used instead of production
+    # tracker store
+    endpoints.tracker_store = None
+    endpoints.lock_store = None
+    endpoints.event_broker = None
+
+    return endpoints
+
+
+def set_up_e2e_test_runner(args: argparse.Namespace) -> E2ETestRunner:
+    endpoints = get_valid_endpoints(args.endpoints)
+
+    if endpoints.model is None:
+        args.model = validate_model_path(args.model, "model", DEFAULT_MODELS_PATH)
+
+    try:
+        return E2ETestRunner(
+            remote_storage=args.remote_storage,
+            model_path=args.model,
+            model_server=endpoints.model,
+            endpoints=endpoints,
+        )
+    except AgentNotReady as error:
+        structlogger.error(
+            "cli.finetune_llm.prepare_data.set_up_e2e_test_runner", error=error.message
+        )
+        sys.exit(1)
+
+
+def create_storage_context(
+    storage_type: StorageType, output_dir: str
+) -> StorageContext:
+    if storage_type == StorageType.FILE:
+        strategy = FileStorageStrategy(output_dir)
+    else:
+        raise ValueError("Unsupported storage type")
+
+    return StorageContext(strategy)

rasa/cli/project_templates/calm/endpoints.yml

@@ -11,7 +11,7 @@
 # https://rasa.com/docs/rasa-pro/concepts/custom-actions
 
 action_endpoint:
-  url: "http://localhost:5055/webhook"
+  actions_module: "actions"
 
 # Tracker store which is used to store the conversations.
 # By default the conversations are stored in memory.

rasa/cli/project_templates/tutorial/endpoints.yml

@@ -11,7 +11,7 @@
 # https://rasa.com/docs/rasa-pro/concepts/custom-actions
 
 action_endpoint:
-  url: "http://localhost:5055/webhook"
+  actions_module: "actions"
 
 # Tracker store which is used to store the conversations.
 # By default the conversations are stored in memory.

rasa/cli/run.py

@@ -3,16 +3,19 @@ import logging
 import os
 from typing import List, Text
 
+from rasa.api import run as rasa_run
 from rasa.cli import SubParsersAction
 from rasa.cli.arguments import run as arguments
+from rasa.cli.utils import get_validated_path
+from rasa.exceptions import ModelNotFound
 from rasa.shared.constants import (
-    DOCS_BASE_URL,
-    DEFAULT_ENDPOINTS_PATH,
-    DEFAULT_CREDENTIALS_PATH,
     DEFAULT_ACTIONS_PATH,
+    DEFAULT_CREDENTIALS_PATH,
+    DEFAULT_ENDPOINTS_PATH,
     DEFAULT_MODELS_PATH,
+    DOCS_BASE_URL,
 )
-from rasa.exceptions import ModelNotFound
+from rasa.shared.utils.cli import print_error
 
 logger = logging.getLogger(__name__)
 
@@ -77,19 +80,17 @@ def run(args: argparse.Namespace) -> None:
     Args:
         args: The CLI arguments.
     """
-    import rasa
-
-    args.endpoints = rasa.cli.utils.get_validated_path(
+    args.endpoints = get_validated_path(
         args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True
     )
-    args.credentials = rasa.cli.utils.get_validated_path(
+    args.credentials = get_validated_path(
         args.credentials, "credentials", DEFAULT_CREDENTIALS_PATH, True
     )
 
     if args.enable_api:
         if not args.remote_storage:
             args.model = _validate_model_path(args.model, "model", DEFAULT_MODELS_PATH)
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
     # if the API is not enable you cannot start without a model
@@ -101,14 +102,14 @@ def run(args: argparse.Namespace) -> None:
 
     # start server if remote storage is configured
     if args.remote_storage is not None:
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
     # start server if model server is configured
     endpoints = AvailableEndpoints.read_endpoints(args.endpoints)
     model_server = endpoints.model if endpoints and endpoints.model else None
     if model_server is not None:
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
     # start server if local model found
@@ -120,10 +121,10 @@ def run(args: argparse.Namespace) -> None:
         local_model_set = False
 
     if local_model_set:
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
-    rasa.shared.utils.cli.print_error(
+    print_error(
         f"No model found. You have three options to provide a model:\n"
         f"1. Configure a model server in the endpoint configuration and provide "
         f"the configuration via '--endpoints'.\n"

rasa/cli/scaffold.py

@@ -1,21 +1,22 @@
 import argparse
-from collections import defaultdict
-from enum import Enum
 import os
 import sys
+from collections import defaultdict
+from enum import Enum
 from typing import List, Text
 
 from rasa import telemetry
+from rasa.api import train
 from rasa.cli import SubParsersAction
 from rasa.cli.inspect import inspect
-from rasa.shared.utils.cli import print_success, print_error_and_exit
 from rasa.shared.constants import (
-    DOCS_BASE_URL,
     DEFAULT_CONFIG_PATH,
-    DEFAULT_DOMAIN_PATH,
     DEFAULT_DATA_PATH,
+    DEFAULT_DOMAIN_PATH,
     DEFAULT_MODELS_PATH,
+    DOCS_BASE_URL,
 )
+from rasa.shared.utils.cli import print_error_and_exit, print_success
 
 
 class ProjectTemplateName(Enum):
@@ -72,7 +73,6 @@ def add_subparser(
 def print_train_or_instructions(args: argparse.Namespace) -> None:
     """Train a model if the user wants to."""
     import questionary
-    import rasa
 
     print_success("Finished creating project structure.")
 
@@ -84,7 +84,7 @@ def print_train_or_instructions(args: argparse.Namespace) -> None:
 
     if should_train:
         print_success("Training an initial model...")
-        training_result = rasa.train(
+        training_result = train(
             template_domain_path[args.template],
             DEFAULT_CONFIG_PATH,
             DEFAULT_DATA_PATH,
@@ -102,9 +102,10 @@ def print_train_or_instructions(args: argparse.Namespace) -> None:
 
 
 def print_run_or_instructions(args: argparse.Namespace) -> None:
-    from rasa.core import constants
     import questionary
 
+    from rasa.core import constants
+
     should_run = (
         questionary.confirm("Do you want to speak to the trained assistant? 🤖")
         .skip_if(args.no_prompt, default=False)
@@ -164,6 +165,7 @@ def create_initial_project(
 
 def scaffold_path(template: ProjectTemplateName) -> Text:
     import importlib_resources
+
     import rasa.cli.project_templates
 
     template_module = rasa.cli.project_templates.__name__

rasa/cli/train.py

@@ -1,26 +1,26 @@
 import argparse
-import structlog
-import sys
 import asyncio
+import sys
 from pathlib import Path
 from typing import Dict, List, Optional, Text, Union
 
-from rasa.cli import SubParsersAction
-import rasa.cli.arguments.train as train_arguments
+import structlog
 
+import rasa.cli.arguments.train as train_arguments
 import rasa.cli.utils
 import rasa.core.utils
-from rasa.shared.importers.importer import TrainingDataImporter
 import rasa.utils.common
+from rasa.cli import SubParsersAction
 from rasa.core.nlg.generator import NaturalLanguageGenerator
 from rasa.core.train import do_compare_training
 from rasa.shared.constants import (
+    CONFIG_MANDATORY_KEYS,
     CONFIG_MANDATORY_KEYS_CORE,
     CONFIG_MANDATORY_KEYS_NLU,
-    CONFIG_MANDATORY_KEYS,
     DEFAULT_DATA_PATH,
     DEFAULT_DOMAIN_PATHS,
 )
+from rasa.shared.importers.importer import TrainingDataImporter
 
 structlogger = structlog.getLogger(__name__)
 
@@ -94,7 +94,7 @@ def run_training(args: argparse.Namespace, can_exit: bool = False) -> Optional[T
     Returns:
         Path to a trained model or `None` if training was not successful.
     """
-    from rasa import train as train_all
+    from rasa.api import train as train_all
 
     domain = rasa.cli.utils.get_validated_path(
         args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=True
@@ -137,6 +137,7 @@ def run_training(args: argparse.Namespace, can_exit: bool = False) -> Optional[T
         nlu_additional_arguments=extract_nlu_additional_arguments(args),
         model_to_finetune=_model_for_finetuning(args),
         finetuning_epoch_fraction=args.epoch_fraction,
+        remote_storage=args.remote_storage,
     )
     if training_result.code != 0 and can_exit:
         sys.exit(training_result.code)

rasa/cli/utils.py

@@ -462,3 +462,18 @@ def warn_if_rasa_plus_package_installed() -> None:
 def check_if_studio_command() -> bool:
     """Checks if the command is a Rasa Studio command."""
     return len(sys.argv) >= 2 and sys.argv[1] == "studio"
+
+
+def get_e2e_results_file_name(
+    results_output_path: Path,
+    result_type: str,
+) -> str:
+    """Returns the name of the e2e results file."""
+    if results_output_path.is_dir():
+        file_name = str(results_output_path) + f"/e2e_results_{result_type}.yml"
+    else:
+        parent = results_output_path.parent
+        stem = results_output_path.stem
+        file_name = str(parent) + f"/{stem}_{result_type}.yml"
+
+    return file_name

rasa/constants.py

@@ -18,7 +18,7 @@ CONFIG_TELEMETRY_ID = "rasa_user_id"
 CONFIG_TELEMETRY_ENABLED = "enabled"
 CONFIG_TELEMETRY_DATE = "date"
 
-MINIMUM_COMPATIBLE_VERSION = "3.9.16"
+MINIMUM_COMPATIBLE_VERSION = "3.10.0rc1"
 
 GLOBAL_USER_CONFIG_PATH = os.path.expanduser("~/.config/rasa/global.yml")
 
@@ -27,6 +27,7 @@ ENV_LOG_LEVEL_LIBRARIES = "LOG_LEVEL_LIBRARIES"
 ENV_LOG_LEVEL_MATPLOTLIB = "LOG_LEVEL_MATPLOTLIB"
 ENV_LOG_LEVEL_RABBITMQ = "LOG_LEVEL_RABBITMQ"
 ENV_LOG_LEVEL_KAFKA = "LOG_LEVEL_KAFKA"
+ENV_LOG_LEVEL_MLFLOW = "LOG_LEVEL_MLFLOW"
 
 DEFAULT_SANIC_WORKERS = 1
 ENV_SANIC_WORKERS = "SANIC_WORKERS"
@@ -35,3 +36,8 @@ ENV_SANIC_BACKLOG = "SANIC_BACKLOG"
 ENV_GPU_CONFIG = "TF_GPU_MEMORY_ALLOC"
 ENV_CPU_INTER_OP_CONFIG = "TF_INTER_OP_PARALLELISM_THREADS"
 ENV_CPU_INTRA_OP_CONFIG = "TF_INTRA_OP_PARALLELISM_THREADS"
+
+MODEL_ARCHIVE_EXTENSION = "tar.gz"
+
+HTTP_STATUS_FORBIDDEN = 403
+HTTP_STATUS_NOT_FOUND = 404