rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/cli/interactive.py

@@ -0,0 +1,166 @@
+import argparse
+import logging
+import os
+from pathlib import Path
+from typing import List, Optional, Text, Union
+
+from rasa import model
+from rasa.cli import SubParsersAction
+from rasa.cli.arguments import interactive as arguments
+import rasa.cli.train as train
+import rasa.cli.utils
+from rasa.engine.storage.local_model_storage import LocalModelStorage
+from rasa.shared.constants import (
+    ASSISTANT_ID_DEFAULT_VALUE,
+    ASSISTANT_ID_KEY,
+    DEFAULT_ENDPOINTS_PATH,
+    DEFAULT_MODELS_PATH,
+)
+from rasa.shared.data import TrainingType
+from rasa.shared.importers.importer import TrainingDataImporter
+import rasa.shared.utils.cli
+import rasa.utils.common
+
+
+logger = logging.getLogger(__name__)
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add all interactive cli parsers.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    interactive_parser = subparsers.add_parser(
+        "interactive",
+        conflict_handler="resolve",
+        parents=parents,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Starts an interactive learning session to create new training data for a "
+        "Rasa model by chatting.",
+    )
+    interactive_parser.set_defaults(func=interactive, core_only=False)
+
+    interactive_subparsers = interactive_parser.add_subparsers()
+    interactive_core_parser = interactive_subparsers.add_parser(
+        "core",
+        conflict_handler="resolve",
+        parents=parents,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Starts an interactive learning session model to create new training data "
+        "for a Rasa Core model by chatting. Uses the 'RegexMessageHandler', i.e. "
+        "`/<intent>` input format.",
+    )
+    interactive_core_parser.set_defaults(func=interactive, core_only=True)
+
+    arguments.set_interactive_arguments(interactive_parser)
+    arguments.set_interactive_core_arguments(interactive_core_parser)
+
+
+def interactive(args: argparse.Namespace) -> None:
+    _set_not_required_args(args)
+    file_importer = TrainingDataImporter.load_from_config(
+        args.config, args.domain, args.data if not args.core_only else [args.stories]
+    )
+
+    if args.model is None:
+        story_graph = file_importer.get_stories()
+        if not story_graph or story_graph.is_empty():
+            rasa.shared.utils.cli.print_error_and_exit(
+                "Could not run interactive learning without either core "
+                "data or a model containing core data."
+            )
+
+        zipped_model: Optional[Union[Text, Path]] = (
+            train.run_core_training(args)
+            if args.core_only
+            else train.run_training(args)
+        )
+        if not zipped_model:
+            rasa.shared.utils.cli.print_error_and_exit(
+                "Could not train an initial model. Either pass paths "
+                "to the relevant training files (`--data`, `--config`, `--domain`), "
+                "or use 'rasa train' to train a model."
+            )
+    else:
+        validate_assistant_id_key_in_config(file_importer)
+
+        zipped_model = get_provided_model(args.model)
+        if not (zipped_model and os.path.exists(zipped_model)):
+            rasa.shared.utils.cli.print_error_and_exit(
+                f"Interactive learning process cannot be started as no "
+                f"initial model was found at path '{args.model}'.  "
+                f"Use 'rasa train' to train a model."
+            )
+        if not args.skip_visualization:
+            logger.info(f"Loading visualization data from {args.data}.")
+
+    perform_interactive_learning(args, zipped_model, file_importer)
+
+
+def _set_not_required_args(args: argparse.Namespace) -> None:
+    args.fixed_model_name = None
+    args.store_uncompressed = False
+    args.dry_run = False
+    args.skip_validation = True
+    args.fail_on_validation_warnings = False
+    args.validation_max_history = None
+    args.remote_storage = None
+    args.keep_local_model_copy = False
+
+
+def perform_interactive_learning(
+    args: argparse.Namespace,
+    zipped_model: Union[Text, "Path"],
+    file_importer: TrainingDataImporter,
+) -> None:
+    """Performs interactive learning.
+
+    Args:
+        args: Namespace arguments.
+        zipped_model: Path to zipped model.
+        file_importer: File importer which provides the training data and model config.
+    """
+    from rasa.core.train import do_interactive_learning
+
+    args.model = str(zipped_model)
+
+    metadata = LocalModelStorage.metadata_from_archive(zipped_model)
+    if metadata.training_type == TrainingType.NLU:
+        rasa.shared.utils.cli.print_error_and_exit(
+            "Can not run interactive learning on an NLU-only model."
+        )
+
+    args.endpoints = rasa.cli.utils.get_validated_path(
+        args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True
+    )
+
+    do_interactive_learning(args, file_importer)
+
+
+def get_provided_model(arg_model: Text) -> Optional[Union[Text, Path]]:
+    """Checks model path input and selects model from it."""
+    model_path = rasa.cli.utils.get_validated_path(
+        arg_model, "model", DEFAULT_MODELS_PATH
+    )
+
+    return (
+        model.get_latest_model(model_path) if os.path.isdir(model_path) else model_path
+    )
+
+
+def validate_assistant_id_key_in_config(file_importer: TrainingDataImporter) -> None:
+    """Verifies that config contains a unique value for assistant identifier."""
+    config_data = file_importer.get_config()
+    assistant_id = config_data.get(ASSISTANT_ID_KEY)
+
+    if assistant_id is None or assistant_id == ASSISTANT_ID_DEFAULT_VALUE:
+        rasa.shared.utils.cli.print_error_and_exit(
+            f"The '{ASSISTANT_ID_KEY}' key in the config file is either missing or "
+            f"is set to the default value. Please replace the placeholder default "
+            f"value and re-train the model."
+        )
+    return None

rasa/cli/license.py

@@ -0,0 +1,65 @@
+import argparse
+from typing import List
+
+import rasa.shared.utils.cli
+from rasa.cli import SubParsersAction
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add license parser.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    license_parser = subparsers.add_parser(
+        "license",
+        parents=parents,
+        help="Displays licensing information.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Display licensing information.",
+    )
+    license_parser.set_defaults(func=display_license_information)
+
+
+def display_license_information(_: argparse.Namespace) -> None:
+    """Display licensing information to stdout."""
+    rasa.shared.utils.cli.print_info(
+        "By installing and using this software, you agree to be "
+        "bound by the terms and conditions of the Developer Terms "
+        "available at https://rasa.com/developer-terms. "
+        "Please review the Developer Terms carefully before proceeding.\n\n"
+        "Rasa Pro relies on several 3rd-party dependencies. "
+        "The ones below require a license disclorure:\n",
+        PSYCOPG2_LICENSE_DISCLOSURE,
+    )
+
+
+PSYCOPG2_LICENSE_DISCLOSURE = """
+psycopg2 & psycopg2-binary
+--------------------------
+
+psycopg2 and the LGPL
+
+psycopg2 is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+psycopg2 is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission to link this program with the OpenSSL library (or with modified versions of OpenSSL that use the same license as OpenSSL), and distribute linked combinations including the two.
+
+You must obey the GNU Lesser General Public License in all respects for all of the code used other than OpenSSL. If you modify file(s) with this exception, you may extend this exception to your version of the file(s), but you are not obligated to do so. If you do not wish to do so, delete this exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here.
+
+You should have received a copy of the GNU Lesser General Public License along with psycopg2 (see the doc/ directory.) If not, see https://www.gnu.org/licenses/. Alternative licenses
+
+The following BSD-like license applies (at your option) to the files following the pattern psycopg/adapter*.{h,c} and psycopg/microprotocol*.{h,c}:
+
+Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:
+
+The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
+
+Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
+
+This notice may not be removed or altered from any source distribution.
+"""  # noqa: E501

rasa/cli/llm_fine_tuning.py

@@ -0,0 +1,403 @@
+import argparse
+import asyncio
+import sys
+from typing import List, Any, Dict
+
+import structlog
+
+import rasa.cli.utils
+import rasa.shared.utils.cli
+import rasa.shared.utils.io
+import rasa.shared.utils.yaml
+from rasa.cli import SubParsersAction
+from rasa.cli.arguments.default_arguments import (
+    add_endpoint_param,
+    add_model_param,
+    add_remote_storage_param,
+)
+from rasa.cli.e2e_test import (
+    read_test_cases,
+    validate_model_path,
+    RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
+)
+from rasa.core.exceptions import AgentNotReady
+from rasa.core.utils import AvailableEndpoints
+from rasa.dialogue_understanding.generator import SingleStepLLMCommandGenerator
+from rasa.e2e_test.e2e_test_runner import E2ETestRunner
+from rasa.llm_fine_tuning.annotation_module import annotate_e2e_tests
+from rasa.llm_fine_tuning.llm_data_preparation_module import convert_to_fine_tuning_data
+from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
+    ConversationRephraser,
+)
+from rasa.llm_fine_tuning.paraphrasing_module import create_paraphrased_conversations
+from rasa.llm_fine_tuning.storage import (
+    StorageContext,
+    StorageType,
+    FileStorageStrategy,
+)
+from rasa.llm_fine_tuning.train_test_split_module import (
+    split_llm_fine_tuning_data,
+    INSTRUCTION_DATA_FORMAT,
+    CONVERSATIONAL_DATA_FORMAT,
+)
+from rasa.shared.constants import (
+    DEFAULT_ENDPOINTS_PATH,
+    DEFAULT_MODELS_PATH,
+    LLM_CONFIG_KEY,
+)
+from rasa.shared.utils.llm import (
+    combine_custom_and_default_config,
+    resolve_model_client_config,
+)
+from rasa.shared.utils.yaml import read_config_file
+from rasa.utils.beta import ensure_beta_feature_is_enabled
+
+DEFAULT_INPUT_E2E_TEST_PATH = "e2e_tests"
+DEFAULT_OUTPUT_FOLDER = "output"
+RESULT_SUMMARY_FILE = "result_summary.yaml"
+PARAMETERS_FILE = "params.yaml"
+
+structlogger = structlog.get_logger()
+
+
+def add_subparser(
+    subparsers: SubParsersAction, parents: List[argparse.ArgumentParser]
+) -> None:
+    """Add the llm fine-tuning subparser to `rasa test`.
+
+    Args:
+        subparsers: subparser we are going to attach to
+        parents: Parent parsers, needed to ensure tree structure in argparse
+    """
+    llm_parser = subparsers.add_parser(
+        "llm",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Commands related to LLMs.",
+    )
+    llm_subparsers = llm_parser.add_subparsers()
+
+    llm_finetune_parser = llm_subparsers.add_parser(
+        "finetune",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Commands related to LLM fine-tuning.",
+    )
+    llm_finetune_subparser = llm_finetune_parser.add_subparsers()
+
+    create_llm_finetune_data_preparation_subparser(llm_finetune_subparser, parents)
+
+
+def create_llm_finetune_data_preparation_subparser(
+    fine_tune_llm_parser: SubParsersAction,
+    parents: List[argparse.ArgumentParser],
+) -> argparse.ArgumentParser:
+    """Create fine-tuning LLM data preparation subparser."""
+    data_preparation_subparser = fine_tune_llm_parser.add_parser(
+        "prepare-data",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Prepares data for LLM fine-tuning.",
+    )
+
+    data_preparation_subparser.set_defaults(func=prepare_llm_fine_tuning_data)
+
+    add_data_preparation_arguments(data_preparation_subparser)
+    add_model_param(data_preparation_subparser, add_positional_arg=False)
+    add_endpoint_param(
+        data_preparation_subparser,
+        help_text="Configuration file for the model server and the connectors as a "
+        "yml file.",
+    )
+
+    return data_preparation_subparser
+
+
+def add_data_preparation_arguments(parser: argparse.ArgumentParser) -> None:
+    """Arguments for preparing LLM fine-tuning data."""
+    parser.add_argument(
+        "-o",
+        "--out",
+        type=str,
+        default=DEFAULT_OUTPUT_FOLDER,
+        help="The output folder to store the data to.",
+    )
+    parser.add_argument(
+        "path-to-e2e-test-cases",
+        nargs="?",
+        type=str,
+        help="Input file or folder containing end-to-end test cases.",
+        default=DEFAULT_INPUT_E2E_TEST_PATH,
+    )
+
+    add_remote_storage_param(parser)
+
+    rephrasing_arguments = parser.add_argument_group("Rephrasing Module")
+    rephrasing_arguments.add_argument(
+        "--num-rephrases",
+        choices=range(0, 50),
+        type=int,
+        default=10,
+        help="Number of rephrases to be generated per user utterance.",
+    )
+    rephrasing_arguments.add_argument(
+        "--rephrase-config",
+        type=str,
+        default=None,
+        help="Path to config file that contains the configuration of the "
+        "rephrasing module.",
+    )
+
+    train_test_split_arguments = parser.add_argument_group("Train/Test Split Module")
+    train_test_split_arguments.add_argument(
+        "--train-frac",
+        type=restricted_float,
+        default=0.8,
+        help="The amount of data that should go into the training dataset. The value "
+        "should be >0.0 and <=1.0.",
+    )
+    train_test_split_arguments.add_argument(
+        "--output-format",
+        choices=[INSTRUCTION_DATA_FORMAT, CONVERSATIONAL_DATA_FORMAT],
+        type=str,
+        nargs="?",
+        default=INSTRUCTION_DATA_FORMAT,
+        help="Format of the output file.",
+    )
+
+
+def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
+    """Prepare LLM fine-tuning data.
+
+    Args:
+        args: Commandline arguments.
+    """
+    ensure_beta_feature_is_enabled(
+        "LLM fine-tuning recipe",
+        env_flag=RASA_PRO_BETA_FINE_TUNING_RECIPE_ENV_VAR_NAME,
+    )
+
+    rephrase_config = (
+        read_config_file(args.rephrase_config) if args.rephrase_config else {}
+    )
+    ConversationRephraser.validate_config(rephrase_config)
+
+    # make sure the output directory exists
+    output_dir = args.out
+    rasa.shared.utils.io.create_directory(output_dir)
+
+    # read e2e test cases
+    path_to_test_cases = getattr(
+        args, "path-to-e2e-test-cases", DEFAULT_INPUT_E2E_TEST_PATH
+    )
+    test_suite = read_test_cases(path_to_test_cases)
+    # set up the e2e test runner
+    e2e_test_runner = set_up_e2e_test_runner(args)
+
+    if e2e_test_runner.agent.processor is None:
+        rasa.shared.utils.cli.print_error(
+            "No processor: Not able to retrieve flows and config from trained model."
+        )
+        sys.exit(0)
+
+    flows = asyncio.run(e2e_test_runner.agent.processor.get_flows())
+    llm_command_generator_config = _get_llm_command_generator_config(e2e_test_runner)
+
+    # set up storage context
+    storage_context = create_storage_context(StorageType.FILE, output_dir)
+
+    statistics = {}
+
+    # 1. annotate e2e tests
+    log_start_of_module("Annotation")
+    conversations = annotate_e2e_tests(e2e_test_runner, test_suite, storage_context)
+    statistics["num_input_e2e_tests"] = len(test_suite.test_cases)
+    statistics["num_annotated_conversations"] = len(conversations)
+    statistics["num_user_messages_across_conversations"] = sum(
+        [len(conversation.get_user_messages()) for conversation in conversations]
+    )
+    statistics["num_user_messages_to_rephrase_across_conversations"] = sum(
+        [
+            len(conversation.get_user_messages_to_rephrase())
+            for conversation in conversations
+        ]
+    )
+    log_end_of_module("Annotation", statistics)
+
+    # 2. paraphrase conversations
+    log_start_of_module("Rephrasing")
+    conversations, rephrase_config = asyncio.run(
+        create_paraphrased_conversations(
+            conversations,
+            rephrase_config,
+            args.num_rephrases,
+            flows,
+            llm_command_generator_config,
+            storage_context,
+        )
+    )
+    statistics["num_passing_rephrased_user_messages"] = sum(
+        [conversation.get_number_of_rephrases(True) for conversation in conversations]
+    )
+    statistics["num_failing_rephrased_user_messages"] = sum(
+        [conversation.get_number_of_rephrases(False) for conversation in conversations]
+    )
+    log_end_of_module("Rephrasing", statistics)
+
+    # 3. create fine-tuning dataset
+    log_start_of_module("LLM Data Preparation")
+    llm_fine_tuning_data = convert_to_fine_tuning_data(conversations, storage_context)
+    statistics["num_ft_data_points"] = len(llm_fine_tuning_data)
+    log_end_of_module("LLM Data Preparation", statistics)
+
+    # 4. create train/test split
+    log_start_of_module("Train/Test Split")
+    train_data, val_data = split_llm_fine_tuning_data(
+        llm_fine_tuning_data,
+        args.train_frac,
+        args.output_format,
+        storage_context,
+        test_suite,
+    )
+    statistics["num_train_data_points"] = len(train_data)
+    statistics["num_val_data_points"] = len(val_data)
+    log_end_of_module("Train/Test Split", statistics)
+
+    # write down params and statistics to a file
+    write_params(args, rephrase_config, output_dir)
+    write_statistics(statistics, output_dir)
+
+    rasa.shared.utils.cli.print_success(
+        f"Data and intermediate results are written " f"to '{output_dir}'."
+    )
+
+
+def _get_llm_command_generator_config(e2e_test_runner: E2ETestRunner) -> Dict[str, Any]:
+    from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
+
+    train_schema = e2e_test_runner.agent.processor.model_metadata.train_schema  # type: ignore
+
+    for node_name, node in train_schema.nodes.items():
+        if node.matches_type(SingleStepLLMCommandGenerator, include_subtypes=True):
+            # Configurations can reference model groups defined in the endpoints.yml
+            resolved_config = resolve_model_client_config(
+                node.config.get(LLM_CONFIG_KEY, {}), node_name
+            )
+            return combine_custom_and_default_config(
+                resolved_config, DEFAULT_LLM_CONFIG
+            )
+
+    rasa.shared.utils.cli.print_error(
+        "The provided model is not trained using 'SingleStepLLMCommandGenerator' or "
+        "its subclasses. Without it, no data for fine-tuning can be generated. To "
+        "resolve this, please include 'SingleStepLLMCommandGenerator' or its subclass "
+        "in your config and train your model."
+    )
+    sys.exit(1)
+
+
+def log_start_of_module(module_name: str) -> None:
+    log_info = f"Starting {module_name} Module"
+    rasa.shared.utils.cli.print_info(
+        f"{rasa.shared.utils.cli.pad(log_info, char='-')}\n"
+    )
+
+
+def log_end_of_module(module_name: str, statistics: Dict[str, int]) -> None:
+    log_info = f"Finished {module_name} Module"
+    rasa.shared.utils.cli.print_info(
+        f"{rasa.shared.utils.cli.pad(log_info, char='-')}\n"
+    )
+    rasa.shared.utils.cli.print_color(
+        "Current Statistics:", color=rasa.shared.utils.io.bcolors.BOLD
+    )
+    for key, value in statistics.items():
+        rasa.shared.utils.cli.print_color(
+            f"  {key}: {value}", color=rasa.shared.utils.io.bcolors.BOLD
+        )
+
+
+def restricted_float(x: Any) -> float:
+    try:
+        x = float(x)
+    except ValueError:
+        raise argparse.ArgumentTypeError("%r not a floating-point literal" % (x,))
+
+    if x <= 0.0 or x > 1.0:
+        raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0]" % (x,))
+    return x
+
+
+def write_params(
+    args: argparse.Namespace, rephrase_config: Dict[str, Any], output_path: str
+) -> None:
+    yaml_data = {
+        "parameters": {
+            "num_rephrases": args.num_rephrases,
+            "rephrase_config": rephrase_config,
+            "model": args.model,
+            "endpoints": args.endpoints,
+            "remote-storage": args.remote_storage,
+            "train_frac": args.train_frac,
+            "output_format": args.output_format,
+            "out": output_path,
+        }
+    }
+
+    rasa.shared.utils.yaml.write_yaml(yaml_data, f"{output_path}/{PARAMETERS_FILE}")
+
+
+def write_statistics(statistics: Dict[str, Any], output_path: str) -> None:
+    rasa.shared.utils.yaml.write_yaml(
+        statistics, f"{output_path}/{RESULT_SUMMARY_FILE}"
+    )
+
+
+def get_valid_endpoints(endpoints_file: str) -> AvailableEndpoints:
+    validated_endpoints_file = rasa.cli.utils.get_validated_path(
+        endpoints_file, "endpoints", DEFAULT_ENDPOINTS_PATH, True
+    )
+    endpoints = AvailableEndpoints.get_instance(validated_endpoints_file)
+
+    # Ignore all endpoints apart from action server, model, nlu and nlg
+    # to ensure InMemoryTrackerStore is being used instead of production
+    # tracker store
+    endpoints.tracker_store = None
+    endpoints.lock_store = None
+    endpoints.event_broker = None
+
+    return endpoints
+
+
+def set_up_e2e_test_runner(args: argparse.Namespace) -> E2ETestRunner:
+    endpoints = get_valid_endpoints(args.endpoints)
+
+    if endpoints.model is None:
+        args.model = validate_model_path(args.model, "model", DEFAULT_MODELS_PATH)
+
+    try:
+        return E2ETestRunner(
+            remote_storage=args.remote_storage,
+            model_path=args.model,
+            model_server=endpoints.model,
+            endpoints=endpoints,
+        )
+    except AgentNotReady as error:
+        structlogger.error(
+            "cli.finetune_llm.prepare_data.set_up_e2e_test_runner", error=error.message
+        )
+        sys.exit(1)
+
+
+def create_storage_context(
+    storage_type: StorageType, output_dir: str
+) -> StorageContext:
+    if storage_type == StorageType.FILE:
+        strategy = FileStorageStrategy(output_dir)
+    else:
+        raise ValueError("Unsupported storage type")
+
+    return StorageContext(strategy)