rasa-pro 3.10.15__py3-none-any.whl → 3.11.0__py3-none-any.whl

rasa/shared/importers/rasa.py

@@ -6,7 +6,6 @@ import rasa.shared.core.flows.yaml_flows_io
 from rasa.shared.core.flows import FlowsList
 
 import rasa.shared.data
-import rasa.shared.utils.common
 import rasa.shared.utils.io
 from rasa.shared.core.training_data.structures import StoryGraph
 from rasa.shared.importers import utils
@@ -16,6 +15,7 @@ from rasa.shared.core.domain import InvalidDomain, Domain
 from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
     YAMLStoryReader,
 )
+from rasa.shared.utils.common import cached_method
 from rasa.shared.utils.yaml import read_model_configuration
 
 logger = logging.getLogger(__name__)
@@ -29,7 +29,9 @@ class RasaFileImporter(TrainingDataImporter):
         config_file: Optional[Text] = None,
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[Union[List[Text], Text]] = None,
+        expand_env_vars: bool = True,
     ):
+        self.expand_env_vars = expand_env_vars
         self._domain_path = domain_path
 
         self._nlu_files = rasa.shared.data.get_data_files(
@@ -47,40 +49,47 @@ class RasaFileImporter(TrainingDataImporter):
 
         self.config_file = config_file
 
+    @cached_method
     def get_config(self) -> Dict:
         """Retrieves model config (see parent class for full docstring)."""
         if not self.config_file or not os.path.exists(self.config_file):
             logger.debug("No configuration file was provided to the RasaFileImporter.")
             return {}
 
-        config = read_model_configuration(self.config_file)
+        config = read_model_configuration(
+            self.config_file, expand_env_vars=self.expand_env_vars
+        )
         return config
 
-    @rasa.shared.utils.common.cached_method
     def get_config_file_for_auto_config(self) -> Optional[Text]:
         """Returns config file path for auto-config only if there is a single one."""
         return self.config_file
 
+    @cached_method
     def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
         """Retrieves training stories / rules (see parent class for full docstring)."""
         return utils.story_graph_from_paths(
             self._story_files, self.get_domain(), exclusion_percentage
         )
 
+    @cached_method
     def get_flows(self) -> FlowsList:
         """Retrieves training stories / rules (see parent class for full docstring)."""
         return utils.flows_from_paths(self._flow_files)
 
+    @cached_method
     def get_conversation_tests(self) -> StoryGraph:
         """Retrieves conversation test stories (see parent class for full docstring)."""
         return utils.story_graph_from_paths(
             self._conversation_test_files, self.get_domain()
         )
 
+    @cached_method
     def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
         """Retrieves NLU training data (see parent class for full docstring)."""
         return utils.training_data_from_paths(self._nlu_files, language)
 
+    @cached_method
     def get_domain(self) -> Domain:
         """Retrieves model domain (see parent class for full docstring)."""
         domain = Domain.empty()

rasa/shared/importers/remote_importer.py

@@ -0,0 +1,196 @@
+import os
+from typing import Dict, List, Optional, Text, Union
+
+import structlog
+from tarsafe import TarSafe
+
+import rasa.shared.core.flows.yaml_flows_io
+import rasa.shared.data
+import rasa.shared.utils.common
+import rasa.shared.utils.io
+from rasa.core.persistor import StorageType
+from rasa.shared.core.domain import Domain, InvalidDomain
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.exceptions import RasaException
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.yaml import read_model_configuration
+
+structlogger = structlog.get_logger()
+
+TRAINING_DATA_ARCHIVE = "training_data.tar.gz"
+
+
+class RemoteTrainingDataImporter(TrainingDataImporter):
+    """Remote `TrainingFileImporter` implementation.
+
+    Fetches training data from a remote storage and extracts it to a local directory.
+    Extracted training data is then used to load flows, NLU, stories,
+        domain, and config files.
+    """
+
+    def __init__(
+        self,
+        config_file: Optional[Text] = None,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+        project_directory: Optional[Text] = None,
+        remote_storage: Optional[StorageType] = None,
+        training_data_path: Optional[Text] = None,
+    ):
+        """Initializes `RemoteTrainingDataImporter`.
+
+        Args:
+            config_file: Path to the model configuration file.
+            domain_path: Path to the domain file.
+            training_data_paths: List of paths to the training data files.
+            project_directory: Path to the project directory.
+            remote_storage: Storage to use to load the training data.
+            training_data_path: Path to the training data.
+        """
+        self.remote_storage = remote_storage
+        self.training_data_path = training_data_path
+
+        self.extracted_path = self._fetch_and_extract_training_archive(
+            TRAINING_DATA_ARCHIVE, self.training_data_path
+        )
+
+        self._nlu_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.data.is_nlu_file
+        )
+        self._story_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_stories_file
+        )
+        self._flow_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.core.flows.yaml_flows_io.is_flows_file
+        )
+        self._conversation_test_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_test_stories_file
+        )
+
+        self.config_file = config_file
+
+    def _fetch_training_archive(
+        self, training_file: str, training_data_path: Optional[str] = None
+    ) -> str:
+        """Fetches training files from remote storage."""
+        from rasa.core.persistor import get_persistor
+
+        persistor = get_persistor(self.remote_storage)
+        if persistor is None:
+            raise RasaException(
+                f"Could not find a persistor for "
+                f"the storage type '{self.remote_storage}'."
+            )
+
+        return persistor.retrieve(training_file, training_data_path)
+
+    def _fetch_and_extract_training_archive(
+        self, training_file: str, training_data_path: Optional[Text] = None
+    ) -> Optional[str]:
+        """Fetches and extracts training files from remote storage.
+
+        If the `training_data_path` is not provided, the training
+        data is extracted to the current working directory.
+
+        Args:
+            training_file: Name of the training data archive file.
+            training_data_path: Path to the training data.
+
+        Returns:
+            Path to the extracted training data.
+        """
+
+        if training_data_path is None:
+            training_data_path = os.path.join(os.getcwd(), "data")
+
+        if os.path.isfile(training_data_path):
+            raise ValueError(
+                f"Training data path '{training_data_path}' is a file. "
+                f"Please provide a directory path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.fetch_training_archive",
+            training_data_path=training_data_path,
+        )
+        training_archive_file_path = self._fetch_training_archive(
+            training_file, training_data_path
+        )
+
+        if not os.path.isfile(training_archive_file_path):
+            raise FileNotFoundError(
+                f"Training data archive '{training_archive_file_path}' not found. "
+                f"Please make sure to provide the correct path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.extract_training_archive",
+            training_archive_file_path=training_archive_file_path,
+            training_data_path=training_data_path,
+        )
+        with TarSafe.open(training_archive_file_path, "r:gz") as tar:
+            tar.extractall(path=training_data_path)
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.remove_downloaded_archive",
+            training_data_path=training_data_path,
+        )
+        os.remove(training_archive_file_path)
+        return training_data_path
+
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        if not self.config_file or not os.path.exists(self.config_file):
+            structlogger.debug(
+                "rasa.importers.remote_training_data_importer.no_config_file",
+                message="No configuration file was provided to the RasaFileImporter.",
+            )
+            return {}
+
+        config = read_model_configuration(self.config_file)
+        return config
+
+    @rasa.shared.utils.common.cached_method
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return self.config_file
+
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_files, self.get_domain(), exclusion_percentage
+        )
+
+    def get_flows(self) -> FlowsList:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.flows_from_paths(self._flow_files)
+
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._conversation_test_files, self.get_domain()
+        )
+
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_files, language)
+
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domain = Domain.empty()
+        domain_path = f"{self.extracted_path}"
+        try:
+            domain = Domain.load(domain_path)
+        except InvalidDomain as e:
+            rasa.shared.utils.io.raise_warning(
+                f"Loading domain from '{domain_path}' failed. Using "
+                f"empty domain. Error: '{e}'"
+            )
+
+        return domain

rasa/shared/importers/utils.py

@@ -29,6 +29,8 @@ def flows_from_paths(files: List[Text]) -> FlowsList:
 
     flows = FlowsList(underlying_flows=[])
     for file in files:
-        flows = flows.merge(YAMLFlowsReader.read_from_file(file))
+        flows = flows.merge(
+            YAMLFlowsReader.read_from_file(file), ignore_duplicates=False
+        )
     flows.validate()
     return flows

rasa/shared/nlu/training_data/formats/rasa_yaml.py

@@ -1,7 +1,18 @@
 import logging
 from collections import OrderedDict
 from pathlib import Path
-from typing import Text, Any, List, Dict, Tuple, Union, Iterator, Optional, Callable
+from typing import (
+    ClassVar,
+    Text,
+    Any,
+    List,
+    Dict,
+    Tuple,
+    Union,
+    Iterator,
+    Optional,
+    Callable,
+)
 
 import rasa.shared.data
 from rasa.shared.core.domain import Domain
@@ -55,6 +66,8 @@ STRIP_SYMBOLS = "\n\r "
 class RasaYAMLReader(TrainingDataReader):
     """Reads YAML training data and creates a TrainingData object."""
 
+    expand_env_vars: ClassVar[bool] = True
+
     def __init__(self) -> None:
         super().__init__()
         self.training_examples: List[Message] = []
@@ -69,7 +82,9 @@ class RasaYAMLReader(TrainingDataReader):
         If the string is not in the right format, an exception will be raised.
         """
         try:
-            validate_raw_yaml_using_schema_file_with_responses(string, NLU_SCHEMA_FILE)
+            validate_raw_yaml_using_schema_file_with_responses(
+                string, NLU_SCHEMA_FILE, expand_env_vars=self.expand_env_vars
+            )
         except YamlException as e:
             e.filename = self.filename
             raise e
@@ -88,7 +103,7 @@ class RasaYAMLReader(TrainingDataReader):
         """
         self.validate(string)
 
-        yaml_content = read_yaml(string)
+        yaml_content = read_yaml(string, expand_env_vars=self.expand_env_vars)
 
         if not validate_training_data_format_version(yaml_content, self.filename):
             return TrainingData()

rasa/shared/nlu/training_data/training_data.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from functools import cached_property
 from pathlib import Path
 import random
 from collections import Counter, OrderedDict
@@ -9,7 +10,6 @@ from typing import Any, Dict, List, Optional, Set, Text, Tuple, Callable
 import operator
 
 import rasa.shared.data
-from rasa.shared.utils.common import lazy_property
 import rasa.shared.utils.io
 from rasa.shared.nlu.constants import (
     RESPONSE,
@@ -202,7 +202,7 @@ class TrainingData:
 
         return list(OrderedDict.fromkeys(examples))
 
-    @lazy_property
+    @cached_property
     def nlu_examples(self) -> List[Message]:
         """Return examples which have come from NLU training data.
 
@@ -215,32 +215,32 @@ class TrainingData:
             ex for ex in self.training_examples if not ex.is_core_or_domain_message()
         ]
 
-    @lazy_property
+    @cached_property
     def intent_examples(self) -> List[Message]:
         """Returns the list of examples that have intent."""
         return [ex for ex in self.nlu_examples if ex.get(INTENT)]
 
-    @lazy_property
+    @cached_property
     def response_examples(self) -> List[Message]:
         """Returns the list of examples that have response."""
         return [ex for ex in self.nlu_examples if ex.get(INTENT_RESPONSE_KEY)]
 
-    @lazy_property
+    @cached_property
     def entity_examples(self) -> List[Message]:
         """Returns the list of examples that have entities."""
         return [ex for ex in self.nlu_examples if ex.get(ENTITIES)]
 
-    @lazy_property
+    @cached_property
     def intents(self) -> Set[Text]:
         """Returns the set of intents in the training data."""
         return {ex.get(INTENT) for ex in self.training_examples} - {None}
 
-    @lazy_property
+    @cached_property
     def action_names(self) -> Set[Text]:
         """Returns the set of action names in the training data."""
         return {ex.get(ACTION_NAME) for ex in self.training_examples} - {None}
 
-    @lazy_property
+    @cached_property
     def retrieval_intents(self) -> Set[Text]:
         """Returns the total number of response types in the training data."""
         return {
@@ -249,13 +249,13 @@ class TrainingData:
             if ex.get(INTENT_RESPONSE_KEY)
         }
 
-    @lazy_property
+    @cached_property
     def number_of_examples_per_intent(self) -> Dict[Text, int]:
         """Calculates the number of examples per intent."""
         intents = [ex.get(INTENT) for ex in self.nlu_examples]
         return dict(Counter(intents))
 
-    @lazy_property
+    @cached_property
     def number_of_examples_per_response(self) -> Dict[Text, int]:
         """Calculates the number of examples per response."""
         responses = [
@@ -265,12 +265,12 @@ class TrainingData:
         ]
         return dict(Counter(responses))
 
-    @lazy_property
+    @cached_property
     def entities(self) -> Set[Text]:
         """Returns the set of entity types in the training data."""
         return {e.get(ENTITY_ATTRIBUTE_TYPE) for e in self.sorted_entities()}
 
-    @lazy_property
+    @cached_property
     def entity_roles(self) -> Set[Text]:
         """Returns the set of entity roles in the training data."""
         entity_types = {
@@ -280,7 +280,7 @@ class TrainingData:
         }
         return entity_types - {NO_ENTITY_TAG}
 
-    @lazy_property
+    @cached_property
     def entity_groups(self) -> Set[Text]:
         """Returns the set of entity groups in the training data."""
         entity_types = {
@@ -299,7 +299,7 @@ class TrainingData:
 
         return entity_groups_used or entity_roles_used
 
-    @lazy_property
+    @cached_property
     def number_of_examples_per_entity(self) -> Dict[Text, int]:
         """Calculates the number of examples per entity."""
         entities = []
@@ -426,8 +426,9 @@ class TrainingData:
     def persist(
         self, dir_name: Text, filename: Text = DEFAULT_TRAINING_DATA_OUTPUT_PATH
     ) -> Dict[Text, Any]:
-        """Persists this training data to disk and returns necessary
-        information to load it again.
+        """Persists this training data to disk.
+
+        Returns: necessary information to load it again.
         """
         if not os.path.exists(dir_name):
             os.makedirs(dir_name)
@@ -498,9 +499,7 @@ class TrainingData:
     def train_test_split(
         self, train_frac: float = 0.8, random_seed: Optional[int] = None
     ) -> Tuple["TrainingData", "TrainingData"]:
-        """Split into a training and test dataset,
-        preserving the fraction of examples per intent.
-        """
+        """Split into a training and test dataset, preserving the fraction of examples per intent."""  # noqa: E501
         # collect all nlu data
         test, train = self.split_nlu_examples(train_frac, random_seed)
 

rasa/shared/providers/_configs/litellm_router_client_config.py

@@ -0,0 +1,220 @@
+import copy
+from dataclasses import dataclass, field
+from typing import Any, Dict, List
+
+import structlog
+from rasa.shared.constants import (
+    ROUTER_CONFIG_KEY,
+    MODELS_CONFIG_KEY,
+    MODEL_GROUP_ID_CONFIG_KEY,
+    MODEL_NAME_CONFIG_KEY,
+    LITELLM_PARAMS_KEY,
+    PROVIDER_CONFIG_KEY,
+    DEPLOYMENT_CONFIG_KEY,
+    API_TYPE_CONFIG_KEY,
+    MODEL_CONFIG_KEY,
+    MODEL_LIST_KEY,
+    USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY,
+)
+from rasa.shared.providers._configs.model_group_config import (
+    ModelGroupConfig,
+    ModelConfig,
+)
+from rasa.shared.providers.mappings import get_prefix_from_provider
+from rasa.shared.utils.llm import DEPLOYMENT_CENTRIC_PROVIDERS
+
+
+structlogger = structlog.get_logger()
+
+_LITELLM_UNSUPPORTED_KEYS = [
+    PROVIDER_CONFIG_KEY,
+    DEPLOYMENT_CONFIG_KEY,
+    API_TYPE_CONFIG_KEY,
+    USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY,
+]
+
+
+@dataclass
+class LiteLLMRouterClientConfig:
+    """Parses configuration for a LiteLLM Router client. The configuration is expected
+    to be in the following format:
+
+    {
+        "id": "model_group_id",
+        "models": [
+            {
+                "provider": "provider_name",
+                "model": "model_name",
+                "api_base": "api_base",
+                "api_key": "api_key",
+                "api_version": "api_version",
+            },
+            {
+                "provider": "provider_name",
+                "model": "model_name",
+            },
+        "router": {}
+    }
+
+    This configuration is converted into the LiteLLM required format:
+
+    {
+        "id": "model_group_id",
+        "model_list": [
+            {
+                "model_name": "model_group_id",
+                "litellm_params": {
+                    "model": "provider_name/model_name",
+                    "api_base": "api_base",
+                    "api_key": "api_key",
+                    "api_version": "api_version",
+                },
+            },
+            {
+                "model_name": "model_group_id",
+                "litellm_params": {
+                    "model": "provider_name/model_name",
+                },
+            },
+        ],
+        "router": {},
+    }
+
+    Raises:
+        ValueError: If the configuration is missing required keys.
+    """
+
+    _model_group_config: ModelGroupConfig
+    router: Dict[str, Any]
+    _use_chat_completions_endpoint: bool = True
+    extra_parameters: dict = field(default_factory=dict)
+
+    @property
+    def model_group_id(self) -> str:
+        return self._model_group_config.model_group_id
+
+    @property
+    def models(self) -> List[ModelConfig]:
+        return self._model_group_config.models
+
+    @property
+    def litellm_model_list(self) -> List[Dict[str, Any]]:
+        return self._convert_models_to_litellm_model_list()
+
+    @property
+    def litellm_router_settings(self) -> Dict[str, Any]:
+        return self._convert_router_to_litellm_router_settings()
+
+    @property
+    def use_chat_completions_endpoint(self) -> bool:
+        return self._use_chat_completions_endpoint
+
+    def __post_init__(self) -> None:
+        if not self.router:
+            message = "Router cannot be empty."
+            structlogger.error(
+                "litellm_router_client_config.validation_error",
+                message=message,
+                model_group_id=self._model_group_config.model_group_id,
+            )
+            raise ValueError(message)
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "LiteLLMRouterClientConfig":
+        """Initializes a dataclass from the passed config.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Raises:
+            ValueError: Config is missing required keys.
+
+        Returns:
+            LiteLLMRouterClientConfig
+        """
+        model_group_config = ModelGroupConfig.from_dict(config)
+
+        # Copy config to avoid mutating the original
+        config_copy = copy.deepcopy(config)
+        # Pop the keys used by ModelGroupConfig
+        config_copy.pop(MODEL_GROUP_ID_CONFIG_KEY, None)
+        config_copy.pop(MODELS_CONFIG_KEY, None)
+        # Get the router settings
+        router_settings = config_copy.pop(ROUTER_CONFIG_KEY, {})
+        # Get the use_chat_completions_endpoint setting
+        use_chat_completions_endpoint = router_settings.get(
+            USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY, True
+        )
+        # The rest is considered as extra parameters
+        extra_parameters = config_copy
+
+        this = LiteLLMRouterClientConfig(
+            _model_group_config=model_group_config,
+            router=router_settings,
+            _use_chat_completions_endpoint=use_chat_completions_endpoint,
+            extra_parameters=extra_parameters,
+        )
+        return this
+
+    def to_dict(self) -> dict:
+        """Converts the config instance into a dictionary."""
+        d = self._model_group_config.to_dict()
+        d[ROUTER_CONFIG_KEY] = self.router
+        if self.extra_parameters:
+            d.update(self.extra_parameters)
+        return d
+
+    def to_litellm_dict(self) -> dict:
+        return {
+            **self.extra_parameters,
+            MODEL_GROUP_ID_CONFIG_KEY: self.model_group_id,
+            MODEL_LIST_KEY: self._convert_models_to_litellm_model_list(),
+            ROUTER_CONFIG_KEY: self._convert_router_to_litellm_router_settings(),
+        }
+
+    def _convert_router_to_litellm_router_settings(self) -> Dict[str, Any]:
+        _router_settings_copy = copy.deepcopy(self.router)
+        _router_settings_copy.pop(USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY, None)
+        return _router_settings_copy
+
+    def _convert_models_to_litellm_model_list(self) -> List[Dict[str, Any]]:
+        litellm_model_list = []
+
+        for model_config_object in self.models:
+            # Convert the model config to a dict representation
+            litellm_model_config = model_config_object.to_dict()
+
+            provider = litellm_model_config[PROVIDER_CONFIG_KEY]
+
+            # Get the litellm prefixing for the provider
+            prefix = get_prefix_from_provider(provider)
+
+            # Determine whether to use model or deployment key based on the provider.
+            litellm_model_name = (
+                litellm_model_config[DEPLOYMENT_CONFIG_KEY]
+                if provider in DEPLOYMENT_CENTRIC_PROVIDERS
+                else litellm_model_config[MODEL_CONFIG_KEY]
+            )
+
+            # Set 'model' to a provider prefixed model name e.g. openai/gpt-4
+            litellm_model_config[MODEL_CONFIG_KEY] = (
+                litellm_model_name
+                if f"{prefix}/" in litellm_model_name
+                else f"{prefix}/{litellm_model_name}"
+            )
+
+            # Remove parameters that are None and not supported by LiteLLM.
+            litellm_model_config = {
+                key: value
+                for key, value in litellm_model_config.items()
+                if key not in _LITELLM_UNSUPPORTED_KEYS and value is not None
+            }
+
+            litellm_model_list_item = {
+                MODEL_NAME_CONFIG_KEY: self.model_group_id,
+                LITELLM_PARAMS_KEY: litellm_model_config,
+            }
+
+            litellm_model_list.append(litellm_model_list_item)
+
+        return litellm_model_list