rasa-pro 3.9.18__py3-none-any.whl → 3.10.16__py3-none-any.whl

rasa/cli/llm_fine_tuning.py

@@ -0,0 +1,398 @@
+import argparse
+import asyncio
+import sys
+from typing import List, Any, Dict
+
+import structlog
+
+import rasa.cli.utils
+import rasa.shared.utils.cli
+import rasa.shared.utils.io
+import rasa.shared.utils.yaml
+from rasa.cli import SubParsersAction
+from rasa.cli.arguments.default_arguments import (
+    add_endpoint_param,
+    add_model_param,
+    add_remote_storage_param,
+)
+from rasa.cli.e2e_test import (
+    read_test_cases,
+    validate_model_path,
+    RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
+)
+from rasa.core.exceptions import AgentNotReady
+from rasa.core.utils import AvailableEndpoints
+from rasa.dialogue_understanding.generator import SingleStepLLMCommandGenerator
+from rasa.e2e_test.e2e_test_runner import E2ETestRunner
+from rasa.llm_fine_tuning.annotation_module import annotate_e2e_tests
+from rasa.llm_fine_tuning.llm_data_preparation_module import convert_to_fine_tuning_data
+from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
+    ConversationRephraser,
+)
+from rasa.llm_fine_tuning.paraphrasing_module import create_paraphrased_conversations
+from rasa.llm_fine_tuning.storage import (
+    StorageContext,
+    StorageType,
+    FileStorageStrategy,
+)
+from rasa.llm_fine_tuning.train_test_split_module import (
+    split_llm_fine_tuning_data,
+    INSTRUCTION_DATA_FORMAT,
+    CONVERSATIONAL_DATA_FORMAT,
+)
+from rasa.shared.constants import (
+    DEFAULT_ENDPOINTS_PATH,
+    DEFAULT_MODELS_PATH,
+    LLM_CONFIG_KEY,
+)
+from rasa.shared.utils.llm import (
+    combine_custom_and_default_config,
+)
+from rasa.shared.utils.yaml import read_config_file
+from rasa.utils.beta import ensure_beta_feature_is_enabled
+
+DEFAULT_INPUT_E2E_TEST_PATH = "e2e_tests"
+DEFAULT_OUTPUT_FOLDER = "output"
+RESULT_SUMMARY_FILE = "result_summary.yaml"
+PARAMETERS_FILE = "params.yaml"
+
+structlogger = structlog.get_logger()
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add the llm fine-tuning subparser to `rasa test`.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    llm_parser = subparsers.add_parser(
+        "llm",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Commands related to LLMs.",
+    )
+    llm_subparsers = llm_parser.add_subparsers()
+
+    llm_finetune_parser = llm_subparsers.add_parser(
+        "finetune",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Commands related to LLM fine-tuning.",
+    )
+    llm_finetune_subparser = llm_finetune_parser.add_subparsers()
+
+    create_llm_finetune_data_preparation_subparser(llm_finetune_subparser, parents)
+
+
+def create_llm_finetune_data_preparation_subparser(
+    fine_tune_llm_parser: SubParsersAction,
+    parents: List[argparse.ArgumentParser],
+) -> argparse.ArgumentParser:
+    """Create fine-tuning LLM data preparation subparser."""
+    data_preparation_subparser = fine_tune_llm_parser.add_parser(
+        "prepare-data",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Prepares data for LLM fine-tuning.",
+    )
+
+    data_preparation_subparser.set_defaults(func=prepare_llm_fine_tuning_data)
+
+    add_data_preparation_arguments(data_preparation_subparser)
+    add_model_param(data_preparation_subparser, add_positional_arg=False)
+    add_endpoint_param(
+        data_preparation_subparser,
+        help_text="Configuration file for the model server and the connectors as a "
+        "yml file.",
+    )
+
+    return data_preparation_subparser
+
+
+def add_data_preparation_arguments(parser: argparse.ArgumentParser) -> None:
+    """Arguments for preparing LLM fine-tuning data."""
+    parser.add_argument(
+        "-o",
+        "--out",
+        type=str,
+        default=DEFAULT_OUTPUT_FOLDER,
+        help="The output folder to store the data to.",
+    )
+    parser.add_argument(
+        "path-to-e2e-test-cases",
+        nargs="?",
+        type=str,
+        help="Input file or folder containing end-to-end test cases.",
+        default=DEFAULT_INPUT_E2E_TEST_PATH,
+    )
+
+    add_remote_storage_param(parser)
+
+    rephrasing_arguments = parser.add_argument_group("Rephrasing Module")
+    rephrasing_arguments.add_argument(
+        "--num-rephrases",
+        choices=range(0, 50),
+        type=int,
+        default=10,
+        help="Number of rephrases to be generated per user utterance.",
+    )
+    rephrasing_arguments.add_argument(
+        "--rephrase-config",
+        type=str,
+        default=None,
+        help="Path to config file that contains the configuration of the "
+        "rephrasing module.",
+    )
+
+    train_test_split_arguments = parser.add_argument_group("Train/Test Split Module")
+    train_test_split_arguments.add_argument(
+        "--train-frac",
+        type=restricted_float,
+        default=0.8,
+        help="The amount of data that should go into the training dataset. The value "
+        "should be >0.0 and <=1.0.",
+    )
+    train_test_split_arguments.add_argument(
+        "--output-format",
+        choices=[INSTRUCTION_DATA_FORMAT, CONVERSATIONAL_DATA_FORMAT],
+        type=str,
+        nargs="?",
+        default=INSTRUCTION_DATA_FORMAT,
+        help="Format of the output file.",
+    )
+
+
+def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
+    """Prepare LLM fine-tuning data.
+
+    Args:
+        args: Commandline arguments.
+    """
+    ensure_beta_feature_is_enabled(
+        "LLM fine-tuning recipe",
+        env_flag=RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
+    )
+
+    rephrase_config = (
+        read_config_file(args.rephrase_config) if args.rephrase_config else {}
+    )
+    ConversationRephraser.validate_config(rephrase_config)
+
+    # make sure the output directory exists
+    output_dir = args.out
+    rasa.shared.utils.io.create_directory(output_dir)
+
+    # read e2e test cases
+    path_to_test_cases = getattr(
+        args, "path-to-e2e-test-cases", DEFAULT_INPUT_E2E_TEST_PATH
+    )
+    test_suite = read_test_cases(path_to_test_cases)
+    # set up the e2e test runner
+    e2e_test_runner = set_up_e2e_test_runner(args)
+
+    if e2e_test_runner.agent.processor is None:
+        rasa.shared.utils.cli.print_error(
+            "No processor: Not able to retrieve flows and config from trained model."
+        )
+        sys.exit(0)
+
+    flows = asyncio.run(e2e_test_runner.agent.processor.get_flows())
+    llm_command_generator_config = _get_llm_command_generator_config(e2e_test_runner)
+
+    # set up storage context
+    storage_context = create_storage_context(StorageType.FILE, output_dir)
+
+    statistics = {}
+
+    # 1. annotate e2e tests
+    log_start_of_module("Annotation")
+    conversations = annotate_e2e_tests(e2e_test_runner, test_suite, storage_context)
+    statistics["num_input_e2e_tests"] = len(test_suite.test_cases)
+    statistics["num_annotated_conversations"] = len(conversations)
+    statistics["num_user_messages_across_conversations"] = sum(
+        [len(conversation.get_user_messages()) for conversation in conversations]
+    )
+    statistics["num_user_messages_to_rephrase_across_conversations"] = sum(
+        [
+            len(conversation.get_user_messages_to_rephrase())
+            for conversation in conversations
+        ]
+    )
+    log_end_of_module("Annotation", statistics)
+
+    # 2. paraphrase conversations
+    log_start_of_module("Rephrasing")
+    conversations, rephrase_config = asyncio.run(
+        create_paraphrased_conversations(
+            conversations,
+            rephrase_config,
+            args.num_rephrases,
+            flows,
+            llm_command_generator_config,
+            storage_context,
+        )
+    )
+    statistics["num_passing_rephrased_user_messages"] = sum(
+        [conversation.get_number_of_rephrases(True) for conversation in conversations]
+    )
+    statistics["num_failing_rephrased_user_messages"] = sum(
+        [conversation.get_number_of_rephrases(False) for conversation in conversations]
+    )
+    log_end_of_module("Rephrasing", statistics)
+
+    # 3. create fine-tuning dataset
+    log_start_of_module("LLM Data Preparation")
+    llm_fine_tuning_data = convert_to_fine_tuning_data(conversations, storage_context)
+    statistics["num_ft_data_points"] = len(llm_fine_tuning_data)
+    log_end_of_module("LLM Data Preparation", statistics)
+
+    # 4. create train/test split
+    log_start_of_module("Train/Test Split")
+    train_data, val_data = split_llm_fine_tuning_data(
+        llm_fine_tuning_data,
+        args.train_frac,
+        args.output_format,
+        storage_context,
+        test_suite,
+    )
+    statistics["num_train_data_points"] = len(train_data)
+    statistics["num_val_data_points"] = len(val_data)
+    log_end_of_module("Train/Test Split", statistics)
+
+    # write down params and statistics to a file
+    write_params(args, rephrase_config, output_dir)
+    write_statistics(statistics, output_dir)
+
+    rasa.shared.utils.cli.print_success(
+        f"Data and intermediate results are written " f"to '{output_dir}'."
+    )
+
+
+def _get_llm_command_generator_config(e2e_test_runner: E2ETestRunner) -> Dict[str, Any]:
+    from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
+
+    train_schema = e2e_test_runner.agent.processor.model_metadata.train_schema  # type: ignore
+
+    for node_name, node in train_schema.nodes.items():
+        if node.matches_type(SingleStepLLMCommandGenerator, include_subtypes=True):
+            return combine_custom_and_default_config(
+                node.config.get(LLM_CONFIG_KEY, {}), DEFAULT_LLM_CONFIG
+            )
+
+    rasa.shared.utils.cli.print_error(
+        "The provided model is not trained using 'SingleStepLLMCommandGenerator' or "
+        "its subclasses. Without it, no data for fine-tuning can be generated. To "
+        "resolve this, please include 'SingleStepLLMCommandGenerator' or its subclass "
+        "in your config and train your model."
+    )
+    sys.exit(1)
+
+
+def log_start_of_module(module_name: str) -> None:
+    log_info = f"Starting {module_name} Module"
+    rasa.shared.utils.cli.print_info(
+        f"{rasa.shared.utils.cli.pad(log_info, char='-')}\n"
+    )
+
+
+def log_end_of_module(module_name: str, statistics: Dict[str, int]) -> None:
+    log_info = f"Finished {module_name} Module"
+    rasa.shared.utils.cli.print_info(
+        f"{rasa.shared.utils.cli.pad(log_info, char='-')}\n"
+    )
+    rasa.shared.utils.cli.print_color(
+        "Current Statistics:", color=rasa.shared.utils.io.bcolors.BOLD
+    )
+    for key, value in statistics.items():
+        rasa.shared.utils.cli.print_color(
+            f"  {key}: {value}", color=rasa.shared.utils.io.bcolors.BOLD
+        )
+
+
+def restricted_float(x: Any) -> float:
+    try:
+        x = float(x)
+    except ValueError:
+        raise argparse.ArgumentTypeError("%r not a floating-point literal" % (x,))
+
+    if x <= 0.0 or x > 1.0:
+        raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0]" % (x,))
+    return x
+
+
+def write_params(
+    args: argparse.Namespace, rephrase_config: Dict[str, Any], output_path: str
+) -> None:
+    yaml_data = {
+        "parameters": {
+            "num_rephrases": args.num_rephrases,
+            "rephrase_config": rephrase_config,
+            "model": args.model,
+            "endpoints": args.endpoints,
+            "remote-storage": args.remote_storage,
+            "train_frac": args.train_frac,
+            "output_format": args.output_format,
+            "out": output_path,
+        }
+    }
+
+    rasa.shared.utils.yaml.write_yaml(yaml_data, f"{output_path}/{PARAMETERS_FILE}")
+
+
+def write_statistics(statistics: Dict[str, Any], output_path: str) -> None:
+    rasa.shared.utils.yaml.write_yaml(
+        statistics, f"{output_path}/{RESULT_SUMMARY_FILE}"
+    )
+
+
+def get_valid_endpoints(endpoints_file: str) -> AvailableEndpoints:
+    validated_endpoints_file = rasa.cli.utils.get_validated_path(
+        endpoints_file, "endpoints", DEFAULT_ENDPOINTS_PATH, True
+    )
+    endpoints = AvailableEndpoints.get_instance(validated_endpoints_file)
+
+    # Ignore all endpoints apart from action server, model, nlu and nlg
+    # to ensure InMemoryTrackerStore is being used instead of production
+    # tracker store
+    endpoints.tracker_store = None
+    endpoints.lock_store = None
+    endpoints.event_broker = None
+
+    return endpoints
+
+
+def set_up_e2e_test_runner(args: argparse.Namespace) -> E2ETestRunner:
+    endpoints = get_valid_endpoints(args.endpoints)
+
+    if endpoints.model is None:
+        args.model = validate_model_path(args.model, "model", DEFAULT_MODELS_PATH)
+
+    try:
+        return E2ETestRunner(
+            remote_storage=args.remote_storage,
+            model_path=args.model,
+            model_server=endpoints.model,
+            endpoints=endpoints,
+        )
+    except AgentNotReady as error:
+        structlogger.error(
+            "cli.finetune_llm.prepare_data.set_up_e2e_test_runner", error=error.message
+        )
+        sys.exit(1)
+
+
+def create_storage_context(
+    storage_type: StorageType, output_dir: str
+) -> StorageContext:
+    if storage_type == StorageType.FILE:
+        strategy = FileStorageStrategy(output_dir)
+    else:
+        raise ValueError("Unsupported storage type")
+
+    return StorageContext(strategy)

rasa/cli/project_templates/calm/endpoints.yml

@@ -11,7 +11,7 @@
 # https://rasa.com/docs/rasa-pro/concepts/custom-actions
 
 action_endpoint:
-  url: "http://localhost:5055/webhook"
+  actions_module: "actions"
 
 # Tracker store which is used to store the conversations.
 # By default the conversations are stored in memory.

rasa/cli/project_templates/tutorial/endpoints.yml

@@ -11,7 +11,7 @@
 # https://rasa.com/docs/rasa-pro/concepts/custom-actions
 
 action_endpoint:
-  url: "http://localhost:5055/webhook"
+  actions_module: "actions"
 
 # Tracker store which is used to store the conversations.
 # By default the conversations are stored in memory.

rasa/cli/run.py

@@ -3,16 +3,19 @@ import logging
 import os
 from typing import List, Text
 
+from rasa.api import run as rasa_run
 from rasa.cli import SubParsersAction
 from rasa.cli.arguments import run as arguments
+from rasa.cli.utils import get_validated_path
+from rasa.exceptions import ModelNotFound
 from rasa.shared.constants import (
-    DOCS_BASE_URL,
-    DEFAULT_ENDPOINTS_PATH,
-    DEFAULT_CREDENTIALS_PATH,
     DEFAULT_ACTIONS_PATH,
+    DEFAULT_CREDENTIALS_PATH,
+    DEFAULT_ENDPOINTS_PATH,
     DEFAULT_MODELS_PATH,
+    DOCS_BASE_URL,
 )
-from rasa.exceptions import ModelNotFound
+from rasa.shared.utils.cli import print_error
 
 logger = logging.getLogger(__name__)
 
@@ -77,19 +80,17 @@ def run(args: argparse.Namespace) -> None:
     Args:
         args: The CLI arguments.
     """
-    import rasa
-
-    args.endpoints = rasa.cli.utils.get_validated_path(
+    args.endpoints = get_validated_path(
         args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True
     )
-    args.credentials = rasa.cli.utils.get_validated_path(
+    args.credentials = get_validated_path(
         args.credentials, "credentials", DEFAULT_CREDENTIALS_PATH, True
     )
 
     if args.enable_api:
         if not args.remote_storage:
             args.model = _validate_model_path(args.model, "model", DEFAULT_MODELS_PATH)
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
     # if the API is not enable you cannot start without a model
@@ -101,14 +102,14 @@ def run(args: argparse.Namespace) -> None:
 
     # start server if remote storage is configured
     if args.remote_storage is not None:
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
     # start server if model server is configured
-    endpoints = AvailableEndpoints.read_endpoints(args.endpoints)
+    endpoints = AvailableEndpoints.get_instance(args.endpoints)
     model_server = endpoints.model if endpoints and endpoints.model else None
     if model_server is not None:
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
     # start server if local model found
@@ -120,10 +121,10 @@ def run(args: argparse.Namespace) -> None:
         local_model_set = False
 
     if local_model_set:
-        rasa.run(**vars(args))
+        rasa_run(**vars(args))
         return
 
-    rasa.shared.utils.cli.print_error(
+    print_error(
         f"No model found. You have three options to provide a model:\n"
         f"1. Configure a model server in the endpoint configuration and provide "
         f"the configuration via '--endpoints'.\n"

rasa/cli/scaffold.py

@@ -1,21 +1,22 @@
 import argparse
-from collections import defaultdict
-from enum import Enum
 import os
 import sys
+from collections import defaultdict
+from enum import Enum
 from typing import List, Text
 
 from rasa import telemetry
+from rasa.api import train
 from rasa.cli import SubParsersAction
 from rasa.cli.inspect import inspect
-from rasa.shared.utils.cli import print_success, print_error_and_exit
 from rasa.shared.constants import (
-    DOCS_BASE_URL,
     DEFAULT_CONFIG_PATH,
-    DEFAULT_DOMAIN_PATH,
     DEFAULT_DATA_PATH,
+    DEFAULT_DOMAIN_PATH,
     DEFAULT_MODELS_PATH,
+    DOCS_BASE_URL,
 )
+from rasa.shared.utils.cli import print_error_and_exit, print_success
 
 
 class ProjectTemplateName(Enum):
@@ -72,7 +73,6 @@ def add_subparser(
 def print_train_or_instructions(args: argparse.Namespace) -> None:
     """Train a model if the user wants to."""
     import questionary
-    import rasa
 
     print_success("Finished creating project structure.")
 
@@ -84,7 +84,7 @@ def print_train_or_instructions(args: argparse.Namespace) -> None:
 
     if should_train:
         print_success("Training an initial model...")
-        training_result = rasa.train(
+        training_result = train(
             template_domain_path[args.template],
             DEFAULT_CONFIG_PATH,
             DEFAULT_DATA_PATH,
@@ -102,9 +102,10 @@ def print_train_or_instructions(args: argparse.Namespace) -> None:
 
 
 def print_run_or_instructions(args: argparse.Namespace) -> None:
-    from rasa.core import constants
     import questionary
 
+    from rasa.core import constants
+
     should_run = (
         questionary.confirm("Do you want to speak to the trained assistant? 🤖")
         .skip_if(args.no_prompt, default=False)
@@ -164,6 +165,7 @@ def create_initial_project(
 
 def scaffold_path(template: ProjectTemplateName) -> Text:
     import importlib_resources
+
     import rasa.cli.project_templates
 
     template_module = rasa.cli.project_templates.__name__

rasa/cli/studio/studio.py

@@ -1,10 +1,11 @@
 import argparse
-from typing import List, Optional
+from typing import List, Optional, Tuple
 from urllib.parse import ParseResult, urlparse
 
 import questionary
 from rasa.cli import SubParsersAction
 
+import rasa.shared.utils.cli
 import rasa.cli.studio.download
 import rasa.cli.studio.train
 import rasa.cli.studio.upload
@@ -50,6 +51,15 @@ def _add_config_subparser(
 
     studio_config_parser.set_defaults(func=create_and_store_studio_config)
 
+    studio_config_parser.add_argument(
+        "--disable-verify",
+        "-x",
+        action="store_true",
+        default=False,
+        help="Disable strict SSL verification for the "
+        "Rasa Studio authentication server.",
+    )
+
     # add advanced configuration flag to trigger
     # advanced configuration setup for authentication settings
     studio_config_parser.add_argument(
@@ -139,7 +149,7 @@ def _configure_studio_url() -> Optional[str]:
     return studio_url
 
 
-def _get_advanced_config(studio_url: str) -> tuple:
+def _get_advanced_config(studio_url: str) -> Tuple:
     """Get the advanced configuration values for Rasa Studio."""
     keycloak_url = questionary.text(
         "Please provide your Rasa Studio Keycloak URL",
@@ -157,7 +167,7 @@ def _get_advanced_config(studio_url: str) -> tuple:
     return keycloak_url, realm_name, client_id
 
 
-def _get_default_config(studio_url: str) -> tuple:
+def _get_default_config(studio_url: str) -> Tuple:
     """Get the default configuration values for Rasa Studio."""
     keycloak_url = studio_url + "auth/"
     realm_name = DEFAULT_REALM_NAME
@@ -168,6 +178,7 @@ def _get_default_config(studio_url: str) -> tuple:
         f"Keycloak URL: {keycloak_url}, "
         f"Realm Name: '{realm_name}', "
         f"Client ID: '{client_id}'. "
+        f"SSL verification is enabled."
         f"You can use '--advanced' to configure these settings."
     )
 
@@ -175,7 +186,11 @@ def _get_default_config(studio_url: str) -> tuple:
 
 
 def _create_studio_config(
-    studio_url: str, keycloak_url: str, realm_name: str, client_id: str
+    studio_url: str,
+    keycloak_url: str,
+    realm_name: str,
+    client_id: str,
+    disable_verify: bool = False,
 ) -> StudioConfig:
     """Create a StudioConfig object with the provided parameters."""
     return StudioConfig(
@@ -183,6 +198,7 @@ def _create_studio_config(
         studio_url=studio_url + "api/graphql/",
         client_id=client_id,
         realm_name=realm_name,
+        disable_verify=disable_verify,
     )
 
 
@@ -217,8 +233,22 @@ def _configure_studio_config(args: argparse.Namespace) -> StudioConfig:
 
     # create a configuration and auth object to try to reach the studio
     studio_config = _create_studio_config(
-        studio_url, keycloak_url, realm_name, client_id
+        studio_url,
+        keycloak_url,
+        realm_name,
+        client_id,
+        disable_verify=args.disable_verify,
     )
+
+    if studio_config.disable_verify:
+        rasa.shared.utils.cli.print_info(
+            "Disabling SSL verification for the Rasa Studio authentication server."
+        )
+    else:
+        rasa.shared.utils.cli.print_info(
+            "Enabling SSL verification for the Rasa Studio authentication server."
+        )
+
     studio_auth = StudioAuth(studio_config)
 
     if _check_studio_auth(studio_auth):