rasa-pro 3.9.18__py3-none-any.whl → 3.10.3__py3-none-any.whl

rasa/shared/importers/remote_importer.py

@@ -0,0 +1,196 @@
+import os
+from typing import Dict, List, Optional, Text, Union
+
+import structlog
+from tarsafe import TarSafe
+
+import rasa.shared.core.flows.yaml_flows_io
+import rasa.shared.data
+import rasa.shared.utils.common
+import rasa.shared.utils.io
+from rasa.nlu.persistor import StorageType
+from rasa.shared.core.domain import Domain, InvalidDomain
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.exceptions import RasaException
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.yaml import read_model_configuration
+
+structlogger = structlog.get_logger()
+
+TRAINING_DATA_ARCHIVE = "training_data.tar.gz"
+
+
+class RemoteTrainingDataImporter(TrainingDataImporter):
+    """Remote `TrainingFileImporter` implementation.
+
+    Fetches training data from a remote storage and extracts it to a local directory.
+    Extracted training data is then used to load flows, NLU, stories,
+        domain, and config files.
+    """
+
+    def __init__(
+        self,
+        config_file: Optional[Text] = None,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+        project_directory: Optional[Text] = None,
+        remote_storage: Optional[StorageType] = None,
+        training_data_path: Optional[Text] = None,
+    ):
+        """Initializes `RemoteTrainingDataImporter`.
+
+        Args:
+            config_file: Path to the model configuration file.
+            domain_path: Path to the domain file.
+            training_data_paths: List of paths to the training data files.
+            project_directory: Path to the project directory.
+            remote_storage: Storage to use to load the training data.
+            training_data_path: Path to the training data.
+        """
+        self.remote_storage = remote_storage
+        self.training_data_path = training_data_path
+
+        self.extracted_path = self._fetch_and_extract_training_archive(
+            TRAINING_DATA_ARCHIVE, self.training_data_path
+        )
+
+        self._nlu_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.data.is_nlu_file
+        )
+        self._story_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_stories_file
+        )
+        self._flow_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.core.flows.yaml_flows_io.is_flows_file
+        )
+        self._conversation_test_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_test_stories_file
+        )
+
+        self.config_file = config_file
+
+    def _fetch_training_archive(
+        self, training_file: str, training_data_path: Optional[str] = None
+    ) -> str:
+        """Fetches training files from remote storage."""
+        from rasa.nlu.persistor import get_persistor
+
+        persistor = get_persistor(self.remote_storage)
+        if persistor is None:
+            raise RasaException(
+                f"Could not find a persistor for "
+                f"the storage type '{self.remote_storage}'."
+            )
+
+        return persistor.retrieve(training_file, training_data_path)
+
+    def _fetch_and_extract_training_archive(
+        self, training_file: str, training_data_path: Optional[Text] = None
+    ) -> Optional[str]:
+        """Fetches and extracts training files from remote storage.
+
+        If the `training_data_path` is not provided, the training
+        data is extracted to the current working directory.
+
+        Args:
+            training_file: Name of the training data archive file.
+            training_data_path: Path to the training data.
+
+        Returns:
+            Path to the extracted training data.
+        """
+
+        if training_data_path is None:
+            training_data_path = os.path.join(os.getcwd(), "data")
+
+        if os.path.isfile(training_data_path):
+            raise ValueError(
+                f"Training data path '{training_data_path}' is a file. "
+                f"Please provide a directory path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.fetch_training_archive",
+            training_data_path=training_data_path,
+        )
+        training_archive_file_path = self._fetch_training_archive(
+            training_file, training_data_path
+        )
+
+        if not os.path.isfile(training_archive_file_path):
+            raise FileNotFoundError(
+                f"Training data archive '{training_archive_file_path}' not found. "
+                f"Please make sure to provide the correct path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.extract_training_archive",
+            training_archive_file_path=training_archive_file_path,
+            training_data_path=training_data_path,
+        )
+        with TarSafe.open(training_archive_file_path, "r:gz") as tar:
+            tar.extractall(path=training_data_path)
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.remove_downloaded_archive",
+            training_data_path=training_data_path,
+        )
+        os.remove(training_archive_file_path)
+        return training_data_path
+
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        if not self.config_file or not os.path.exists(self.config_file):
+            structlogger.debug(
+                "rasa.importers.remote_training_data_importer.no_config_file",
+                message="No configuration file was provided to the RasaFileImporter.",
+            )
+            return {}
+
+        config = read_model_configuration(self.config_file)
+        return config
+
+    @rasa.shared.utils.common.cached_method
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return self.config_file
+
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_files, self.get_domain(), exclusion_percentage
+        )
+
+    def get_flows(self) -> FlowsList:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.flows_from_paths(self._flow_files)
+
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._conversation_test_files, self.get_domain()
+        )
+
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_files, language)
+
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domain = Domain.empty()
+        domain_path = f"{self.extracted_path}"
+        try:
+            domain = Domain.load(domain_path)
+        except InvalidDomain as e:
+            rasa.shared.utils.io.raise_warning(
+                f"Loading domain from '{domain_path}' failed. Using "
+                f"empty domain. Error: '{e}'"
+            )
+
+        return domain

rasa/shared/nlu/constants.py

@@ -2,6 +2,8 @@ TEXT = "text"
 TEXT_TOKENS = "text_tokens"
 INTENT = "intent"
 COMMANDS = "commands"
+LLM_COMMANDS = "llm_commands"  # needed for fine-tuning
+LLM_PROMPT = "llm_prompt"  # needed for fine-tuning
 FLOWS_FROM_SEMANTIC_SEARCH = "flows_from_semantic_search"
 FLOWS_IN_PROMPT = "flows_in_prompt"
 NOT_INTENT = "not_intent"

rasa/shared/nlu/training_data/features.py

@@ -1,133 +1,15 @@
 from __future__ import annotations
-
-import itertools
-from dataclasses import dataclass
 from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set
+import itertools
 
 import numpy as np
 import scipy.sparse
-from safetensors.numpy import save_file, load_file
 
-import rasa.shared.nlu.training_data.util
 import rasa.shared.utils.io
+import rasa.shared.nlu.training_data.util
 from rasa.shared.nlu.constants import FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE
 
 
-@dataclass
-class FeatureMetadata:
-    data_type: str
-    attribute: str
-    origin: Union[str, List[str]]
-    is_sparse: bool
-    shape: tuple
-    safetensors_key: str
-
-
-def save_features(
-    features_dict: Dict[Text, List[Features]], file_name: str
-) -> Dict[str, Any]:
-    """Save a dictionary of Features lists to disk using safetensors.
-
-    Args:
-        features_dict: Dictionary mapping strings to lists of Features objects
-        file_name: File to save the features to
-
-    Returns:
-        The metadata to reconstruct the features.
-    """
-    # All tensors are stored in a single safetensors file
-    tensors_to_save = {}
-    # Metadata will be stored separately
-    metadata = {}
-
-    for key, features_list in features_dict.items():
-        feature_metadata_list = []
-
-        for idx, feature in enumerate(features_list):
-            # Create a unique key for this tensor in the safetensors file
-            safetensors_key = f"{key}_{idx}"
-
-            # Convert sparse matrices to dense if needed
-            if feature.is_sparse():
-                # For sparse matrices, use the COO format
-                coo = feature.features.tocoo()  # type:ignore[union-attr]
-                # Save data, row indices and col indices separately
-                tensors_to_save[f"{safetensors_key}_data"] = coo.data
-                tensors_to_save[f"{safetensors_key}_row"] = coo.row
-                tensors_to_save[f"{safetensors_key}_col"] = coo.col
-            else:
-                tensors_to_save[safetensors_key] = feature.features
-
-            # Store metadata
-            metadata_item = FeatureMetadata(
-                data_type=feature.type,
-                attribute=feature.attribute,
-                origin=feature.origin,
-                is_sparse=feature.is_sparse(),
-                shape=feature.features.shape,
-                safetensors_key=safetensors_key,
-            )
-            feature_metadata_list.append(vars(metadata_item))
-
-        metadata[key] = feature_metadata_list
-
-    # Save tensors
-    save_file(tensors_to_save, file_name)
-
-    return metadata
-
-
-def load_features(
-    filename: str, metadata: Dict[str, Any]
-) -> Dict[Text, List[Features]]:
-    """Load Features dictionary from disk.
-
-    Args:
-        filename: File name of the safetensors file.
-        metadata: Metadata to reconstruct the features.
-
-    Returns:
-        Dictionary mapping strings to lists of Features objects
-    """
-    # Load tensors
-    tensors = load_file(filename)
-
-    # Reconstruct the features dictionary
-    features_dict: Dict[Text, List[Features]] = {}
-
-    for key, feature_metadata_list in metadata.items():
-        features_list = []
-
-        for meta in feature_metadata_list:
-            safetensors_key = meta["safetensors_key"]
-
-            if meta["is_sparse"]:
-                # Reconstruct sparse matrix from COO format
-                data = tensors[f"{safetensors_key}_data"]
-                row = tensors[f"{safetensors_key}_row"]
-                col = tensors[f"{safetensors_key}_col"]
-
-                features_matrix = scipy.sparse.coo_matrix(
-                    (data, (row, col)), shape=tuple(meta["shape"])
-                ).tocsr()  # Convert back to CSR format
-            else:
-                features_matrix = tensors[safetensors_key]
-
-            # Reconstruct Features object
-            features = Features(
-                features=features_matrix,
-                feature_type=meta["data_type"],
-                attribute=meta["attribute"],
-                origin=meta["origin"],
-            )
-
-            features_list.append(features)
-
-        features_dict[key] = features_list
-
-    return features_dict
-
-
 class Features:
     """Stores the features produced by any featurizer."""
 

rasa/shared/providers/_configs/azure_openai_client_config.py

@@ -0,0 +1,181 @@
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Optional
+
+import structlog
+
+from rasa.shared.constants import (
+    MODEL_CONFIG_KEY,
+    MODEL_NAME_CONFIG_KEY,
+    OPENAI_API_BASE_CONFIG_KEY,
+    API_BASE_CONFIG_KEY,
+    OPENAI_API_TYPE_CONFIG_KEY,
+    API_TYPE_CONFIG_KEY,
+    OPENAI_API_VERSION_CONFIG_KEY,
+    API_VERSION_CONFIG_KEY,
+    DEPLOYMENT_CONFIG_KEY,
+    DEPLOYMENT_NAME_CONFIG_KEY,
+    ENGINE_CONFIG_KEY,
+    RASA_TYPE_CONFIG_KEY,
+    LANGCHAIN_TYPE_CONFIG_KEY,
+    STREAM_CONFIG_KEY,
+    N_REPHRASES_CONFIG_KEY,
+    REQUEST_TIMEOUT_CONFIG_KEY,
+    TIMEOUT_CONFIG_KEY,
+    PROVIDER_CONFIG_KEY,
+    AZURE_OPENAI_PROVIDER,
+    AZURE_API_TYPE,
+)
+from rasa.shared.providers._configs.utils import (
+    resolve_aliases,
+    raise_deprecation_warnings,
+    validate_required_keys,
+    validate_forbidden_keys,
+)
+
+structlogger = structlog.get_logger()
+
+DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING = {
+    # Deployment name aliases
+    DEPLOYMENT_NAME_CONFIG_KEY: DEPLOYMENT_CONFIG_KEY,
+    ENGINE_CONFIG_KEY: DEPLOYMENT_CONFIG_KEY,
+    # Provider aliases
+    RASA_TYPE_CONFIG_KEY: PROVIDER_CONFIG_KEY,
+    LANGCHAIN_TYPE_CONFIG_KEY: PROVIDER_CONFIG_KEY,
+    # API type aliases
+    OPENAI_API_TYPE_CONFIG_KEY: API_TYPE_CONFIG_KEY,
+    # API base aliases
+    OPENAI_API_BASE_CONFIG_KEY: API_BASE_CONFIG_KEY,
+    # API version aliases
+    OPENAI_API_VERSION_CONFIG_KEY: API_VERSION_CONFIG_KEY,
+    # Model name aliases
+    MODEL_NAME_CONFIG_KEY: MODEL_CONFIG_KEY,
+    # Timeout aliases
+    REQUEST_TIMEOUT_CONFIG_KEY: TIMEOUT_CONFIG_KEY,
+}
+
+REQUIRED_KEYS = [DEPLOYMENT_CONFIG_KEY]
+
+FORBIDDEN_KEYS = [
+    STREAM_CONFIG_KEY,
+    N_REPHRASES_CONFIG_KEY,
+]
+
+
+@dataclass
+class AzureOpenAIClientConfig:
+    """Parses configuration for Azure OpenAI client, resolves aliases and
+    raises deprecation warnings.
+
+    Raises:
+        ValueError: Raised in cases of invalid configuration:
+            - If any of the required configuration keys are missing.
+            - If `api_type` has a value different from `azure`.
+    """
+
+    deployment: str
+
+    model: Optional[str]
+    api_base: Optional[str]
+    api_version: Optional[str]
+    # API Type is not used by LiteLLM backend, but we define
+    # it here for backward compatibility.
+    api_type: Optional[str] = AZURE_API_TYPE
+
+    # Provider is not used by LiteLLM backend, but we define it here since it's
+    # used as switch between different clients.
+    provider: str = AZURE_OPENAI_PROVIDER
+
+    extra_parameters: dict = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.provider != AZURE_OPENAI_PROVIDER:
+            message = f"Provider must be set to '{AZURE_OPENAI_PROVIDER}'."
+            structlogger.error(
+                "azure_openai_client_config.validation_error",
+                message=message,
+                provider=self.provider,
+            )
+            raise ValueError(message)
+        if self.deployment is None:
+            message = "Deployment cannot be set to None."
+            structlogger.error(
+                "azure_openai_client_config.validation_error",
+                message=message,
+                deployment=self.deployment,
+            )
+            raise ValueError(message)
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "AzureOpenAIClientConfig":
+        """
+        Initializes a dataclass from the passed config.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Raises:
+            ValueError: Raised in cases of invalid configuration:
+                - If any of the required configuration keys are missing.
+                - If `api_type` has a value different from `azure`.
+
+        Returns:
+            AzureOpenAIClientConfig
+        """
+        # Check for deprecated keys
+        raise_deprecation_warnings(config, DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING)
+        # Resolve any potential aliases
+        config = cls.resolve_config_aliases(config)
+        # Validate that required keys are set
+        validate_required_keys(config, REQUIRED_KEYS)
+        # Validate that the forbidden keys are not present
+        validate_forbidden_keys(config, FORBIDDEN_KEYS)
+        # Init client config
+        this = AzureOpenAIClientConfig(
+            # Required parameters
+            deployment=config.pop(DEPLOYMENT_CONFIG_KEY),
+            # Pop the 'provider' key. Currently, it's *optional* because of
+            # backward compatibility with older versions.
+            provider=config.pop(PROVIDER_CONFIG_KEY, AZURE_OPENAI_PROVIDER),
+            # Optional
+            api_type=config.pop(API_TYPE_CONFIG_KEY, AZURE_API_TYPE),
+            model=config.pop(MODEL_CONFIG_KEY, None),
+            # Optional, can also be set through environment variables
+            # in clients.
+            api_base=config.pop(API_BASE_CONFIG_KEY, None),
+            api_version=config.pop(API_VERSION_CONFIG_KEY, None),
+            # The rest of parameters (e.g. model parameters) are considered
+            # as extra parameters (this also includes timeout).
+            extra_parameters=config,
+        )
+        return this
+
+    def to_dict(self) -> dict:
+        """Converts the config instance into a dictionary."""
+        d = asdict(self)
+        # Extra parameters should also be on the top level
+        d.pop("extra_parameters", None)
+        d.update(self.extra_parameters)
+        return d
+
+    @staticmethod
+    def resolve_config_aliases(config: Dict[str, Any]) -> Dict[str, Any]:
+        return resolve_aliases(config, DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING)
+
+
+def is_azure_openai_config(config: dict) -> bool:
+    """Check whether the configuration is meant to configure
+    an Azure OpenAI client.
+    """
+    # Resolve any aliases that are specific to Azure OpenAI configuration
+    config = AzureOpenAIClientConfig.resolve_config_aliases(config)
+
+    # Case: Configuration contains `provider: azure`.
+    if config.get(PROVIDER_CONFIG_KEY) == AZURE_OPENAI_PROVIDER:
+        return True
+
+    # Case: Configuration contains `deployment` key
+    # (specific to Azure OpenAI configuration)
+    if config.get(DEPLOYMENT_CONFIG_KEY) is not None:
+        return True
+
+    return False

rasa/shared/providers/_configs/client_config.py

@@ -0,0 +1,57 @@
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class ClientConfig(Protocol):
+    """
+    Protocol for the client config that specifies the interface for interacting
+    with the API.
+    """
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "ClientConfig":
+        """
+        Initializes the client config with the given configuration.
+
+        This class method should be implemented to parse the given
+        configuration and create an instance of an client config.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Raises:
+            ValueError: Config is missing required keys.
+
+        Returns:
+            ClientConfig
+        """
+        ...
+
+    def to_dict(self) -> dict:
+        """
+        Returns the configuration for that the client config is initialized with.
+
+        This method should be implemented to return a dictionary containing
+        the configuration settings for the client config.
+
+        Returns:
+            dictionary containing the configuration settings for the client config.
+        """
+        ...
+
+    @staticmethod
+    def resolve_config_aliases(config: dict) -> dict:
+        """
+        Resolve any potential aliases in the configuration.
+
+        This method should be implemented to resolve any potential aliases in the
+        configuration.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Returns:
+            dictionary containing the resolved configuration settings for the
+            client config.
+        """
+        ...