rasa-pro 3.13.0.dev2__py3-none-any.whl → 3.13.0.dev5__py3-none-any.whl

rasa/shared/utils/llm.py

@@ -1,5 +1,6 @@
 import importlib.resources
 import json
+import logging
 from copy import deepcopy
 from functools import wraps
 from typing import (
@@ -7,6 +8,7 @@ from typing import (
     Any,
     Callable,
     Dict,
+    Literal,
     Optional,
     Text,
     Type,
@@ -59,6 +61,7 @@ from rasa.shared.providers.mappings import (
     get_embedding_client_from_provider,
     get_llm_client_from_provider,
 )
+from rasa.shared.utils.constants import LOG_COMPONENT_SOURCE_METHOD_INIT
 
 if TYPE_CHECKING:
     from rasa.shared.core.trackers import DialogueStateTracker
@@ -654,35 +657,59 @@ def embedder_client_factory(
 
 
 def get_prompt_template(
-    jinja_file_path: Optional[Text], default_prompt_template: Text
+    jinja_file_path: Optional[Text],
+    default_prompt_template: Text,
+    *,
+    log_source_component: Optional[Text] = None,
+    log_source_method: Optional[Literal["init", "fingerprint_addon"]] = None,
 ) -> Text:
     """Returns the jinja template.
 
     Args:
-        jinja_file_path: the path to the jinja file
-        default_prompt_template: the default prompt template
+        jinja_file_path: The path to the jinja template file. If not provided, the
+            default template will be used.
+        default_prompt_template: The fallback prompt template to use if no file is
+            found or specified.
+        log_source_component: The name of the component emitting the log, used to
+            identify the source in structured logging.
+        log_source_method: The name of the method or function emitting the log for
+            better traceability.
 
     Returns:
         The prompt template.
     """
+
     try:
         if jinja_file_path is not None:
             prompt_template = rasa.shared.utils.io.read_file(jinja_file_path)
-            structlogger.info(
-                "utils.llm.get_prompt_template.custom_prompt_template_read_successfull",
+
+            log_level = (
+                logging.INFO
+                if log_source_method == LOG_COMPONENT_SOURCE_METHOD_INIT
+                else logging.DEBUG
+            )
+
+            structlogger.log(
+                log_level,
+                "utils.llm.get_prompt_template"
+                ".custom_prompt_template_read_successfully",
                 event_info=(
                     f"Custom prompt template read successfully from "
                     f"`{jinja_file_path}`."
                 ),
                 prompt_file_path=jinja_file_path,
+                log_source_component=log_source_component,
+                log_source_method=log_source_method,
             )
             return prompt_template
     except (FileIOException, FileNotFoundException):
         structlogger.warning(
-            "utils.llm.get_prompt_template.failed_to_read_custom_prompt_template",
+            "utils.llm.get_prompt_template" ".failed_to_read_custom_prompt_template",
             event_info=(
                 "Failed to read custom prompt template. Using default template instead."
             ),
+            log_source_component=log_source_component,
+            log_source_method=log_source_method,
         )
     return default_prompt_template
 
@@ -692,50 +719,66 @@ def get_default_prompt_template_based_on_model(
     model_prompt_mapping: Dict[str, Any],
     default_prompt_path: str,
     fallback_prompt_path: str,
+    *,
+    log_source_component: Optional[Text] = None,
+    log_source_method: Optional[Literal["init", "fingerprint_addon"]] = None,
 ) -> Text:
     """Returns the default prompt template based on the model name.
 
     Args:
         llm_config: The model config.
-        model_prompt_mapping: The mapping of model name to prompt template.
-        default_prompt_path: The default prompt path of the component.
-        fallback_prompt_path: The fallback prompt path for all other models
-            that do not have a mapping in the model_prompt_mapping.
+        model_prompt_mapping: The model name -> prompt template mapping.
+        default_prompt_path: The path to the default prompt template for the component.
+        fallback_prompt_path: The fallback prompt path for all other models that do not
+            have a mapping in the model_prompt_mapping.
+        log_source_component: The name of the component emitting the log, used to
+            identify the source in structured logging.
+        log_source_method: The name of the method or function emitting the log for
+            better traceability.
 
     Returns:
         The default prompt template.
     """
+    # Extract the provider and model name information from the configuration
     _llm_config = deepcopy(llm_config)
     if MODELS_CONFIG_KEY in _llm_config:
         _llm_config = _llm_config[MODELS_CONFIG_KEY][0]
     provider = _llm_config.get(PROVIDER_CONFIG_KEY)
     model = _llm_config.get(MODEL_CONFIG_KEY)
+
+    # If the model is not defined, we default to the default prompt template.
     if not model:
-        # If the model is not defined, we default to the default prompt template.
-        structlogger.info(
-            "utils.llm.get_default_prompt_template_based_on_model.using_default_prompt_template",
+        structlogger.debug(
+            "utils.llm.get_default_prompt_template_based_on_model"
+            ".using_default_prompt_template",
             event_info=(
                 f"Model not defined in the config. Default prompt template read from"
                 f" - `{default_prompt_path}`."
             ),
             default_prompt_path=default_prompt_path,
+            log_source_component=log_source_component,
+            log_source_method=log_source_method,
         )
         return importlib.resources.read_text(
             DEFAULT_PROMPT_PACKAGE_NAME, default_prompt_path
         )
 
-    model_name = model if provider and provider in model else f"{provider}/{model}"
-    if prompt_file_path := model_prompt_mapping.get(model_name):
-        # If the model is found in the mapping, we use the model-specific prompt
-        # template.
-        structlogger.info(
-            "utils.llm.get_default_prompt_template_based_on_model.using_model_specific_prompt_template",
+    full_model_name = model if provider and provider in model else f"{provider}/{model}"
+
+    # If the model is found in the mapping, we use the model-specific prompt
+    # template.
+    if prompt_file_path := model_prompt_mapping.get(full_model_name):
+        structlogger.debug(
+            "utils.llm.get_default_prompt_template_based_on_model"
+            ".using_model_specific_prompt_template",
             event_info=(
                 f"Using model-specific default prompt template. Default prompt "
                 f"template read from - `{prompt_file_path}`."
             ),
             default_prompt_path=prompt_file_path,
-            model_name=model_name,
+            model_name=full_model_name,
+            log_source_component=log_source_component,
+            log_source_method=log_source_method,
         )
         return importlib.resources.read_text(
             DEFAULT_PROMPT_PACKAGE_NAME, prompt_file_path
@@ -743,14 +786,17 @@ def get_default_prompt_template_based_on_model(
 
     # If the model is not found in the mapping, we default to the fallback prompt
     # template.
-    structlogger.info(
-        "utils.llm.get_default_prompt_template_based_on_model.using_fallback_prompt_template",
+    structlogger.debug(
+        "utils.llm.get_default_prompt_template_based_on_model"
+        ".using_fallback_prompt_template",
         event_info=(
             f"Model not found in the model prompt mapping. Fallback prompt template "
             f"read from - `{fallback_prompt_path}`."
         ),
         fallback_prompt_path=fallback_prompt_path,
-        model_name=model_name,
+        model_name=full_model_name,
+        log_source_component=log_source_component,
+        log_source_method=log_source_method,
     )
     return importlib.resources.read_text(
         DEFAULT_PROMPT_PACKAGE_NAME, fallback_prompt_path
@@ -762,7 +808,7 @@ def allowed_values_for_slot(slot: Slot) -> Union[str, None]:
     if isinstance(slot, BooleanSlot):
         return str([True, False])
     if isinstance(slot, CategoricalSlot):
-        return str([v for v in slot.values if v != "__other__"] + ["other"])
+        return str([v for v in slot.values if v != "__other__"])
     else:
         return None
 

rasa/studio/download/domains.py

@@ -0,0 +1,49 @@
+import logging
+from pathlib import Path
+
+from rasa.shared.core.domain import (
+    Domain,
+)
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.studio.constants import STUDIO_DOMAIN_FILENAME
+
+logger = logging.getLogger(__name__)
+
+
+def merge_domain_with_overwrite(
+    data_from_studio: TrainingDataImporter,
+    data_local: TrainingDataImporter,
+    domain_path: Path,
+) -> None:
+    """
+    Merges the domain from Rasa Studio with the local domain.
+
+    Args:
+        data_from_studio: The training data importer for the Rasa Studio domain.
+        data_local: The training data importer for the local domain.
+        domain_path: The path to the domain file or directory.
+    """
+    if domain_path.is_file():
+        all_local_domain_files = [str(domain_path)]
+        domain_path = domain_path.parent
+    else:
+        all_local_domain_files = data_local.get_domain_files([str(domain_path)])
+
+    # leftover_domain represents the items in the studio
+    # domain that are not in the local domain
+    leftover_domain = data_from_studio.get_user_domain()
+    for file_path in all_local_domain_files:
+        # For each local domain file, we do a partial merge
+        local_domain = Domain.from_file(str(file_path))
+        updated_local_domain = local_domain.partial_merge(leftover_domain)
+
+        # If this partial merge made changes, persist them
+        if local_domain != updated_local_domain:
+            updated_local_domain.persist(file_path)
+
+        # Remove every item now present in updated_local_domain from leftover_domain
+        leftover_domain = leftover_domain.difference(updated_local_domain)
+
+    # If there are still items in leftover_domain, persist them
+    if not leftover_domain.is_empty():
+        leftover_domain.persist(domain_path / STUDIO_DOMAIN_FILENAME)

rasa/studio/download/download.py

@@ -0,0 +1,439 @@
+import argparse
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import questionary
+import structlog
+from ruamel import yaml
+from ruamel.yaml.scalarstring import LiteralScalarString
+
+import rasa.cli.utils
+import rasa.shared.utils.cli
+from rasa.shared.constants import (
+    DEFAULT_CONFIG_PATH,
+    DEFAULT_DATA_PATH,
+    DEFAULT_DOMAIN_PATH,
+    DEFAULT_DOMAIN_PATHS,
+    DEFAULT_ENDPOINTS_PATH,
+)
+from rasa.shared.core.flows.yaml_flows_io import YamlFlowsWriter
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.utils.yaml import read_yaml
+from rasa.studio import data_handler
+from rasa.studio.config import StudioConfig
+from rasa.studio.constants import (
+    STUDIO_DOMAIN_FILENAME,
+    STUDIO_FLOWS_FILENAME,
+    STUDIO_NLU_FILENAME,
+)
+from rasa.studio.data_handler import StudioDataHandler, import_data_from_studio
+from rasa.studio.download.domains import merge_domain_with_overwrite
+from rasa.studio.download.flows import merge_flows_with_overwrite
+from rasa.utils.mapper import RasaPrimitiveStorageMapper
+
+structlogger = structlog.getLogger(__name__)
+
+
+def handle_download(args: argparse.Namespace) -> None:
+    """Main function to handle downloading and merging data.
+
+    Args:
+        args: The command line arguments.
+    """
+    handler = StudioDataHandler(
+        studio_config=StudioConfig.read_config(),
+        assistant_name=args.assistant_name[0],
+    )
+    handler.request_all_data()
+
+    domain_path, data_paths = _prepare_data_and_domain_paths(args)
+
+    # Handle config and endpoints.
+    config_path, write_config = _handle_file_overwrite(
+        args.config, DEFAULT_CONFIG_PATH, "config"
+    )
+    endpoints_path, write_endpoints = _handle_file_overwrite(
+        args.endpoints, DEFAULT_ENDPOINTS_PATH, "endpoints"
+    )
+    message_parts = []
+    config_path = config_path if write_config else None
+    endpoints_path = endpoints_path if write_endpoints else None
+
+    if config_path:
+        config_data = handler.get_config()
+        if not config_data:
+            rasa.shared.utils.cli.print_error_and_exit("No config data found.")
+        with open(config_path, "w") as f:
+            f.write(config_data)
+            message_parts.append(f"config to '{config_path}'")
+    if endpoints_path:
+        endpoints_data = handler.get_endpoints()
+        if not endpoints_data:
+            raise ValueError("No endpoints data found.")
+        with open(endpoints_path, "w") as f:
+            f.write(endpoints_data)
+            message_parts.append(f"endpoints to '{endpoints_path}'")
+    if message_parts:
+        message = "Downloaded " + " and ".join(message_parts)
+        structlogger.info("studio.download.config_endpoints", event_info=message)
+
+    if not args.overwrite:
+        _handle_download_no_overwrite(handler, domain_path, data_paths)
+    else:
+        _handle_download_with_overwrite(handler, domain_path, data_paths)
+
+
+def _prepare_data_and_domain_paths(args: argparse.Namespace) -> Tuple[Path, List[Path]]:
+    """Prepars the domain and data paths based on the provided arguments.
+
+    Args:
+        args: The command line arguments.
+
+    Returns:
+        A tuple containing the domain path and a list of data paths.
+    """
+    # Prepare domain path.
+    domain_path = rasa.cli.utils.get_validated_path(
+        args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=True
+    )
+    domain_or_default_path = args.domain or DEFAULT_DOMAIN_PATH
+
+    if domain_path is None:
+        domain_path = Path(domain_or_default_path)
+        domain_path.touch()
+
+    if isinstance(domain_path, str):
+        domain_path = Path(domain_path)
+
+    if domain_path.is_file():
+        if not args.overwrite:
+            domain_path.unlink()
+            domain_path.touch()
+
+    if domain_path.is_dir():
+        if not args.overwrite:
+            domain_path = domain_path / STUDIO_DOMAIN_FILENAME
+            domain_path.touch()
+
+    # Prepare data paths.
+    data_paths: List[Path] = []
+    for f in args.data:
+        data_path = rasa.cli.utils.get_validated_path(
+            f, "data", DEFAULT_DATA_PATH, none_is_valid=True
+        )
+
+        if data_path is None:
+            data_path = Path(f)
+            data_path.mkdir(parents=True, exist_ok=True)
+        else:
+            data_path = Path(data_path)
+
+        if data_path.is_file() or data_path.is_dir():
+            data_paths.append(data_path)
+        else:
+            data_path.mkdir(parents=True, exist_ok=True)
+            data_paths.append(data_path)
+
+    # Remove duplicates while preserving order.
+    data_paths = list(dict.fromkeys(data_paths))
+    return domain_path, data_paths
+
+
+def _handle_file_overwrite(
+    file_path: Optional[str], default_path: str, file_type: str
+) -> Tuple[Optional[Path], bool]:
+    """Determines whether to overwrite an existing file or create a new one.
+
+    Works for config and endpoints at this moment.
+
+    Args:
+        file_path: The path to the file.
+        default_path: The default path to use if no file path is provided.
+        file_type: The type of the file (e.g., config, endpoints).
+
+    Returns:
+        A tuple of the file path and a boolean indicating overwrite status.
+    """
+    file_already_exists = rasa.cli.utils.get_validated_path(
+        file_path, file_type, default_path, none_is_valid=True
+    )
+    write_file = False
+    path = None
+    file_or_default_path = file_path or default_path
+
+    if file_already_exists is None:
+        path = Path(file_or_default_path)
+        if path.is_dir():
+            path = path / default_path
+        return path, True
+
+    if questionary.confirm(
+        f"{file_type.capitalize()} file '{file_or_default_path}' already exists. "
+        f"Do you want to overwrite it?"
+    ).ask():
+        write_file = True
+        path = Path(file_or_default_path)
+    return path, write_file
+
+
+def _handle_download_no_overwrite(
+    handler: StudioDataHandler, domain_path: Path, data_paths: List[Path]
+) -> None:
+    """Handles downloading without overwriting existing files.
+
+    Args:
+        handler: The StudioDataHandler instance.
+        domain_path: The path to the domain file or directory.
+        data_paths: The paths to the data files or directories.
+    """
+    data_from_studio, data_local = import_data_from_studio(
+        handler, domain_path, data_paths
+    )
+    _merge_domain_no_overwrite(domain_path, data_from_studio, data_local)
+    _merge_data_no_overwrite(data_paths, handler, data_from_studio, data_local)
+
+
+def _merge_domain_no_overwrite(
+    domain_path: Path,
+    data_from_studio: TrainingDataImporter,
+    data_local: TrainingDataImporter,
+) -> None:
+    """
+    Merges the domain data without overwriting.
+
+    If the domain path is a directory, a new domain file is created under that folder.
+    If the domain path is a file, it merges both domains into that file.
+
+    Args:
+        domain_path: The path to the domain file or directory.
+        data_from_studio: The Studio data importer.
+        data_local: The local data importer.
+    """
+    if domain_path.is_dir():
+        _merge_directory_domain(domain_path, data_from_studio, data_local)
+    elif domain_path.is_file():
+        _merge_file_domain(domain_path, data_from_studio, data_local)
+
+
+def _merge_directory_domain(
+    domain_path: Path,
+    data_from_studio: TrainingDataImporter,
+    data_local: TrainingDataImporter,
+) -> None:
+    """Merges domain data without overwriting when the domain is a directory.
+
+    Args:
+        domain_path: The path to the domain directory.
+        data_from_studio: The Studio data importer.
+        data_local: The local data importer.
+    """
+    from rasa.shared.core.domain import Domain
+
+    studio_domain_path = domain_path / STUDIO_NLU_FILENAME
+    new_domain_data = data_handler.combine_domains(
+        data_from_studio.get_user_domain().as_dict(),
+        data_local.get_user_domain().as_dict(),
+    )
+    studio_domain = Domain.from_dict(new_domain_data)
+
+    if not studio_domain.is_empty():
+        studio_domain.persist(studio_domain_path)
+    else:
+        structlogger.warning(
+            "studio.download.merge_directory_domain",
+            event_info="No additional domain data found in Studio assistant.",
+        )
+
+
+def _merge_file_domain(
+    domain_path: Path,
+    data_from_studio: TrainingDataImporter,
+    data_local: TrainingDataImporter,
+) -> None:
+    """Merges domain data without overwriting when the domain is a file.
+
+    Args:
+        domain_path: The path to the domain file.
+        data_from_studio: The Studio data importer.
+        data_local: The local data importer.
+    """
+    domain_merged = data_local.get_user_domain().merge(
+        data_from_studio.get_user_domain()
+    )
+    domain_merged.persist(domain_path)
+
+
+def _merge_data_no_overwrite(
+    data_paths: List[Path],
+    handler: StudioDataHandler,
+    data_from_studio: TrainingDataImporter,
+    data_local: TrainingDataImporter,
+) -> None:
+    """Merges NLU and flow data without overwriting existing data.
+
+    Args:
+        data_paths: The paths to the data files or directories.
+        handler: The StudioDataHandler instance.
+        data_from_studio: The Studio data importer.
+        data_local: The local data importer.
+    """
+    if not data_paths:
+        structlogger.warning(
+            "studio.download.merge_data_no_overwrite.no_path",
+            event_info="No data paths provided. Skipping data merge.",
+        )
+        return
+
+    if len(data_paths) == 1:
+        data_path = data_paths[0]
+        if data_path.is_file():
+            _merge_file_data_no_overwrite(
+                data_path, handler, data_from_studio, data_local
+            )
+        elif data_path.is_dir():
+            _merge_dir_data_no_overwrite(
+                data_path, handler, data_from_studio, data_local
+            )
+        else:
+            structlogger.warning(
+                "studio.download.merge_data_no_overwrite.invalid_path",
+                event_info=(
+                    f"Provided path '{data_path}' is neither a file nor a directory."
+                ),
+            )
+    else:
+        # TODO: Handle multiple data paths.
+        raise NotImplementedError("Multiple data paths are not supported yet.")
+
+
+def _merge_file_data_no_overwrite(
+    data_path: Path,
+    handler: StudioDataHandler,
+    data_from_studio: TrainingDataImporter,
+    data_local: TrainingDataImporter,
+) -> None:
+    """Merges NLU and flows data into a single file without overwriting.
+
+    Args:
+        data_path: Path to the single data file.
+        handler: The StudioDataHandler instance.
+        data_from_studio: The Studio data importer.
+        data_local: The local data importer.
+    """
+    if handler.has_nlu():
+        merged_nlu = data_local.get_nlu_data().merge(data_from_studio.get_nlu_data())
+        merged_nlu.persist_nlu(str(data_path))
+    if handler.has_flows():
+        merged_flows = data_local.get_user_flows().merge(
+            data_from_studio.get_user_flows()
+        )
+        YamlFlowsWriter.dump(merged_flows.underlying_flows, data_path)
+
+
+def _merge_dir_data_no_overwrite(
+    dir_path: Path,
+    handler: StudioDataHandler,
+    data_from_studio: TrainingDataImporter,
+    data_local: TrainingDataImporter,
+) -> None:
+    """Merges NLU and flows data into a single directory without overwriting.
+
+    Args:
+        dir_path: Path to the data directory.
+        handler: The StudioDataHandler instance.
+        data_from_studio: The Studio data importer.
+        data_local: The local data importer.
+    """
+    if handler.has_nlu():
+        nlu_path = dir_path / Path(STUDIO_NLU_FILENAME)
+        _persist_nlu_diff(data_local, data_from_studio, nlu_path)
+    if handler.has_flows():
+        flows_path = dir_path / Path(STUDIO_FLOWS_FILENAME)
+        _persist_flows_diff(data_local, data_from_studio, flows_path)
+
+
+def _handle_download_with_overwrite(
+    handler: StudioDataHandler, domain_path: Path, data_paths: List[Path]
+) -> None:
+    """Handles downloading and merging data when the user opts for overwrite.
+
+    Args:
+        handler: The StudioDataHandler instance.
+        domain_path: The path to the domain file or directory.
+        data_paths: The paths to the data files or directories.
+    """
+    data_from_studio, data_local = import_data_from_studio(
+        handler, domain_path, data_paths
+    )
+    mapper = RasaPrimitiveStorageMapper(
+        domain_path=domain_path, training_data_paths=data_paths
+    )
+    merge_domain_with_overwrite(data_from_studio, data_local, domain_path)
+    merge_flows_with_overwrite(
+        data_paths, handler, data_from_studio, data_local, mapper
+    )
+
+
+def _persist_nlu_diff(
+    data_local: TrainingDataImporter,
+    data_from_studio: TrainingDataImporter,
+    data_path: Path,
+) -> None:
+    """Creates a new NLU file from the diff of local and studio data.
+
+    Args:
+        data_local: The local training data.
+        data_from_studio: The training data from Rasa Studio.
+        data_path: The path to the NLU file.
+    """
+    new_nlu_data = data_handler.create_new_nlu_from_diff(
+        read_yaml(data_from_studio.get_nlu_data().nlu_as_yaml()),
+        read_yaml(data_local.get_nlu_data().nlu_as_yaml()),
+    )
+    if new_nlu_data["nlu"]:
+        pretty_write_nlu_yaml(new_nlu_data, data_path)
+    else:
+        structlogger.warning(
+            "studio.download.persist_nlu_diff",
+            event_info="No additional nlu data found.",
+        )
+
+
+def _persist_flows_diff(
+    data_local: TrainingDataImporter,
+    data_from_studio: TrainingDataImporter,
+    data_path: Path,
+) -> None:
+    """Creates a new flows file from the diff of local and studio data.
+
+    Args:
+        data_local: The local training data.
+        data_from_studio: The training data from Rasa Studio.
+        data_path: The path to the flows file.
+    """
+    new_flows_data = data_handler.create_new_flows_from_diff(
+        data_from_studio.get_user_flows().underlying_flows,
+        data_local.get_user_flows().underlying_flows,
+    )
+    if new_flows_data:
+        YamlFlowsWriter.dump(new_flows_data, data_path)
+    else:
+        structlogger.warning(
+            "studio.download.persist_flows_diff",
+            event_info="No additional flows data found.",
+        )
+
+
+def pretty_write_nlu_yaml(data: Dict, file: Path) -> None:
+    """Writes the NLU YAML in a pretty way.
+
+    Args:
+        data: The data to write.
+        file: The file to write to.
+    """
+    dumper = yaml.YAML()
+    for item in data["nlu"]:
+        if item.get("examples"):
+            item["examples"] = LiteralScalarString(item["examples"])
+    with file.open("w", encoding="utf-8") as outfile:
+        dumper.dump(data, outfile)