rasa-pro 3.10.16__py3-none-any.whl → 3.11.0a1__py3-none-any.whl

rasa/shared/importers/importer.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import logging
 from abc import ABC, abstractmethod
 from functools import reduce
@@ -25,6 +23,7 @@ from rasa.shared.core.training_data.structures import StoryGraph
 from rasa.shared.nlu.constants import ACTION_NAME, ENTITIES
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.common import cached_method
 from rasa.shared.utils.yaml import read_config_file
 
 logger = logging.getLogger(__name__)
@@ -115,8 +114,8 @@ class TrainingDataImporter(ABC):
         config_path: Text,
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[List[Text]] = None,
-        args: Optional[Dict[Text, Any]] = {},
-    ) -> TrainingDataImporter:
+        args: Optional[Dict[Text, Any]] = None,
+    ) -> "TrainingDataImporter":
         """Loads a `TrainingDataImporter` instance from a configuration file."""
         config = read_config_file(config_path)
         return TrainingDataImporter.load_from_dict(
@@ -128,8 +127,8 @@ class TrainingDataImporter(ABC):
         config_path: Text,
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[List[Text]] = None,
-        args: Optional[Dict[Text, Any]] = {},
-    ) -> TrainingDataImporter:
+        args: Optional[Dict[Text, Any]] = None,
+    ) -> "TrainingDataImporter":
         """Loads core `TrainingDataImporter` instance.
 
         Instance loaded from configuration file will only read Core training data.
@@ -144,8 +143,8 @@ class TrainingDataImporter(ABC):
         config_path: Text,
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[List[Text]] = None,
-        args: Optional[Dict[Text, Any]] = {},
-    ) -> TrainingDataImporter:
+        args: Optional[Dict[Text, Any]] = None,
+    ) -> "TrainingDataImporter":
         """Loads nlu `TrainingDataImporter` instance.
 
         Instance loaded from configuration file will only read NLU training data.
@@ -168,7 +167,7 @@ class TrainingDataImporter(ABC):
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[List[Text]] = None,
         args: Optional[Dict[Text, Any]] = None,
-    ) -> TrainingDataImporter:
+    ) -> "TrainingDataImporter":
         """Loads a `TrainingDataImporter` instance from a dictionary."""
         from rasa.shared.importers.rasa import RasaFileImporter
 
@@ -197,15 +196,18 @@ class TrainingDataImporter(ABC):
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[List[Text]] = None,
         args: Optional[Dict[Text, Any]] = None,
-    ) -> Optional[TrainingDataImporter]:
+    ) -> Optional["TrainingDataImporter"]:
         from rasa.shared.importers.multi_project import MultiProjectImporter
         from rasa.shared.importers.rasa import RasaFileImporter
+        from rasa.shared.importers.remote_importer import RemoteTrainingDataImporter
 
         module_path = importer_config.pop("name", None)
         if module_path == RasaFileImporter.__name__:
             importer_class: Type[TrainingDataImporter] = RasaFileImporter
         elif module_path == MultiProjectImporter.__name__:
             importer_class = MultiProjectImporter
+        elif module_path == RemoteTrainingDataImporter.__name__:
+            importer_class = RemoteTrainingDataImporter
         else:
             try:
                 importer_class = rasa.shared.utils.common.class_from_module_path(
@@ -225,7 +227,8 @@ class TrainingDataImporter(ABC):
             **constructor_arguments,
         )
 
-    def fingerprint(self) -> Text:
+    @staticmethod
+    def fingerprint() -> Text:
         """Returns a random fingerprint as data shouldn't be cached."""
         return rasa.shared.utils.io.random_string(25)
 
@@ -281,7 +284,6 @@ class NluDataImporter(TrainingDataImporter):
         """Retrieves NLU training data (see parent class for full docstring)."""
         return self._importer.get_nlu_data(language)
 
-    @rasa.shared.utils.common.cached_method
     def get_config_file_for_auto_config(self) -> Optional[Text]:
         """Returns config file path for auto-config only if there is a single one."""
         return self._importer.get_config_file_for_auto_config()
@@ -297,14 +299,14 @@ class CombinedDataImporter(TrainingDataImporter):
     def __init__(self, importers: List[TrainingDataImporter]):
         self._importers = importers
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_config(self) -> Dict:
         """Retrieves model config (see parent class for full docstring)."""
         configs = [importer.get_config() for importer in self._importers]
 
         return reduce(lambda merged, other: {**merged, **(other or {})}, configs, {})
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_domain(self) -> Domain:
         """Retrieves model domain (see parent class for full docstring)."""
         domains = [importer.get_domain() for importer in self._importers]
@@ -315,7 +317,7 @@ class CombinedDataImporter(TrainingDataImporter):
             Domain.empty(),
         )
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
         """Retrieves training stories / rules (see parent class for full docstring)."""
         stories = [
@@ -326,7 +328,7 @@ class CombinedDataImporter(TrainingDataImporter):
             lambda merged, other: merged.merge(other), stories, StoryGraph([])
         )
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_flows(self) -> FlowsList:
         """Retrieves training stories / rules (see parent class for full docstring)."""
         flow_lists = [importer.get_flows() for importer in self._importers]
@@ -337,7 +339,7 @@ class CombinedDataImporter(TrainingDataImporter):
             FlowsList(underlying_flows=[]),
         )
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_conversation_tests(self) -> StoryGraph:
         """Retrieves conversation test stories (see parent class for full docstring)."""
         stories = [importer.get_conversation_tests() for importer in self._importers]
@@ -346,7 +348,7 @@ class CombinedDataImporter(TrainingDataImporter):
             lambda merged, other: merged.merge(other), stories, StoryGraph([])
         )
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
         """Retrieves NLU training data (see parent class for full docstring)."""
         nlu_data = [importer.get_nlu_data(language) for importer in self._importers]
@@ -355,7 +357,7 @@ class CombinedDataImporter(TrainingDataImporter):
             lambda merged, other: merged.merge(other), nlu_data, TrainingData()
         )
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_config_file_for_auto_config(self) -> Optional[Text]:
         """Returns config file path for auto-config only if there is a single one."""
         if len(self._importers) != 1:
@@ -414,26 +416,22 @@ class FlowSyncImporter(PassThroughImporter):
         """Loads the default flows from the file system."""
         from rasa.shared.core.flows.yaml_flows_io import YAMLFlowsReader
 
-        default_flows_file = str(
-            importlib_resources.files("rasa.dialogue_understanding.patterns").joinpath(
-                DEFAULT_PATTERN_FLOWS_FILE_NAME
-            )
-        )
-
-        flows = YAMLFlowsReader.read_from_file(default_flows_file)
+        flows = YAMLFlowsReader.read_from_file(FlowSyncImporter.default_pattern_path())
         flows.validate()
         return flows
 
     @staticmethod
-    def load_default_pattern_flows_domain() -> Domain:
-        """Loads the default flows from the file system."""
-        default_flows_file = str(
+    def default_pattern_path() -> str:
+        return str(
             importlib_resources.files("rasa.dialogue_understanding.patterns").joinpath(
                 DEFAULT_PATTERN_FLOWS_FILE_NAME
             )
         )
 
-        return Domain.from_path(default_flows_file)
+    @staticmethod
+    def load_default_pattern_flows_domain() -> Domain:
+        """Loads the default flows from the file system."""
+        return Domain.from_path(FlowSyncImporter.default_pattern_path())
 
     @classmethod
     def merge_with_default_flows(cls, flows: FlowsList) -> FlowsList:
@@ -459,7 +457,7 @@ class FlowSyncImporter(PassThroughImporter):
 
         return flows.merge(FlowsList(missing_default_flows))
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_flows(self) -> FlowsList:
         flows = self._importer.get_flows()
 
@@ -469,11 +467,11 @@ class FlowSyncImporter(PassThroughImporter):
 
         return self.merge_with_default_flows(flows)
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_user_flows(self) -> FlowsList:
         return self._importer.get_flows()
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_domain(self) -> Domain:
         """Merge existing domain with properties of flows."""
         # load domain data from user defined domain files
@@ -515,7 +513,7 @@ class ResponsesSyncImporter(PassThroughImporter):
     back to the Domain.
     """
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_domain(self) -> Domain:
         """Merge existing domain with properties of retrieval intents in NLU data."""
         existing_domain = self._importer.get_domain()
@@ -597,7 +595,7 @@ class ResponsesSyncImporter(PassThroughImporter):
             }
         )
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
         """Updates NLU data with responses for retrieval intents from domain."""
         existing_nlu_data = self._importer.get_nlu_data(language)
@@ -632,7 +630,7 @@ class E2EImporter(PassThroughImporter):
     - adds potential end-to-end bot messages from stories as actions to the domain
     """
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_user_flows(self) -> FlowsList:
         if not isinstance(self._importer, FlowSyncImporter):
             raise NotImplementedError(
@@ -641,9 +639,12 @@ class E2EImporter(PassThroughImporter):
 
         return self._importer.get_user_flows()
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_domain(self) -> Domain:
-        """Retrieves model domain (see parent class for full docstring)."""
+        """Merge existing domain with properties of end-to-end actions in stories.
+
+        Returns: Domain with end-to-end actions added to action names.
+        """
         original = self._importer.get_domain()
         e2e_domain = self._get_domain_with_e2e_actions()
 
@@ -673,7 +674,7 @@ class E2EImporter(PassThroughImporter):
 
         return Domain.from_dict({KEY_E2E_ACTIONS: list(additional_e2e_action_names)})
 
-    @rasa.shared.utils.common.cached_method
+    @cached_method
     def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
         """Retrieves NLU training data (see parent class for full docstring)."""
         training_datasets = [

rasa/shared/importers/multi_project.py

@@ -1,22 +1,23 @@
-import logging
-from functools import reduce
-from typing import Text, Set, Dict, Optional, List, Union, Any
 import os
+from functools import reduce
+from typing import Any, Dict, List, Optional, Set, Text, Union
+
+import structlog
 
 import rasa.shared.data
 import rasa.shared.utils.io
 from rasa.shared.core.domain import Domain
-from rasa.shared.importers.importer import TrainingDataImporter
-from rasa.shared.importers import utils
-from rasa.shared.nlu.training_data.training_data import TrainingData
-from rasa.shared.core.training_data.structures import StoryGraph
-from rasa.shared.utils.common import mark_as_experimental_feature
 from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
     YAMLStoryReader,
 )
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.common import cached_method, mark_as_experimental_feature
 from rasa.shared.utils.yaml import read_config_file, read_model_configuration
 
-logger = logging.getLogger(__name__)
+structlogger = structlog.get_logger()
 
 
 class MultiProjectImporter(TrainingDataImporter):
@@ -50,8 +51,13 @@ class MultiProjectImporter(TrainingDataImporter):
         self._story_paths += extra_story_files
         self._nlu_paths += extra_nlu_files
 
-        logger.debug(
-            "Selected projects: {}".format("".join([f"\n-{i}" for i in self._imports]))
+        structlogger.debug(
+            "multi_project_importer.initialisation",
+            event_info=(
+                "Selected projects: {}".format(
+                    "".join([f"\n-{i}" for i in self._imports])
+                )
+            ),
         )
 
         mark_as_experimental_feature(feature_name="MultiProjectImporter")
@@ -135,6 +141,7 @@ class MultiProjectImporter(TrainingDataImporter):
 
         return training_paths
 
+    @cached_method
     def is_imported(self, path: Text) -> bool:
         """Checks whether a path is imported by a skill.
 
@@ -175,6 +182,7 @@ class MultiProjectImporter(TrainingDataImporter):
             [rasa.shared.utils.io.is_subdirectory(path, i) for i in self._imports]
         )
 
+    @cached_method
     def get_domain(self) -> Domain:
         """Retrieves model domain (see parent class for full docstring)."""
         domains = [Domain.load(path) for path in self._domain_paths]
@@ -184,20 +192,24 @@ class MultiProjectImporter(TrainingDataImporter):
             Domain.empty(),
         )
 
+    @cached_method
     def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
         """Retrieves training stories / rules (see parent class for full docstring)."""
         return utils.story_graph_from_paths(
             self._story_paths, self.get_domain(), exclusion_percentage
         )
 
+    @cached_method
     def get_conversation_tests(self) -> StoryGraph:
         """Retrieves conversation test stories (see parent class for full docstring)."""
         return utils.story_graph_from_paths(self._e2e_story_paths, self.get_domain())
 
+    @cached_method
     def get_config(self) -> Dict:
         """Retrieves model config (see parent class for full docstring)."""
         return self.config
 
+    @cached_method
     def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
         """Retrieves NLU training data (see parent class for full docstring)."""
         return utils.training_data_from_paths(self._nlu_paths, language)

rasa/shared/importers/rasa.py

@@ -6,7 +6,6 @@ import rasa.shared.core.flows.yaml_flows_io
 from rasa.shared.core.flows import FlowsList
 
 import rasa.shared.data
-import rasa.shared.utils.common
 import rasa.shared.utils.io
 from rasa.shared.core.training_data.structures import StoryGraph
 from rasa.shared.importers import utils
@@ -16,6 +15,7 @@ from rasa.shared.core.domain import InvalidDomain, Domain
 from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
     YAMLStoryReader,
 )
+from rasa.shared.utils.common import cached_method
 from rasa.shared.utils.yaml import read_model_configuration
 
 logger = logging.getLogger(__name__)
@@ -47,6 +47,7 @@ class RasaFileImporter(TrainingDataImporter):
 
         self.config_file = config_file
 
+    @cached_method
     def get_config(self) -> Dict:
         """Retrieves model config (see parent class for full docstring)."""
         if not self.config_file or not os.path.exists(self.config_file):
@@ -56,31 +57,35 @@ class RasaFileImporter(TrainingDataImporter):
         config = read_model_configuration(self.config_file)
         return config
 
-    @rasa.shared.utils.common.cached_method
     def get_config_file_for_auto_config(self) -> Optional[Text]:
         """Returns config file path for auto-config only if there is a single one."""
         return self.config_file
 
+    @cached_method
     def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
         """Retrieves training stories / rules (see parent class for full docstring)."""
         return utils.story_graph_from_paths(
             self._story_files, self.get_domain(), exclusion_percentage
         )
 
+    @cached_method
     def get_flows(self) -> FlowsList:
         """Retrieves training stories / rules (see parent class for full docstring)."""
         return utils.flows_from_paths(self._flow_files)
 
+    @cached_method
     def get_conversation_tests(self) -> StoryGraph:
         """Retrieves conversation test stories (see parent class for full docstring)."""
         return utils.story_graph_from_paths(
             self._conversation_test_files, self.get_domain()
         )
 
+    @cached_method
     def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
         """Retrieves NLU training data (see parent class for full docstring)."""
         return utils.training_data_from_paths(self._nlu_files, language)
 
+    @cached_method
     def get_domain(self) -> Domain:
         """Retrieves model domain (see parent class for full docstring)."""
         domain = Domain.empty()

rasa/shared/importers/remote_importer.py

@@ -0,0 +1,196 @@
+import os
+from typing import Dict, List, Optional, Text, Union
+
+import structlog
+from tarsafe import TarSafe
+
+import rasa.shared.core.flows.yaml_flows_io
+import rasa.shared.data
+import rasa.shared.utils.common
+import rasa.shared.utils.io
+from rasa.core.persistor import StorageType
+from rasa.shared.core.domain import Domain, InvalidDomain
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.exceptions import RasaException
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.yaml import read_model_configuration
+
+structlogger = structlog.get_logger()
+
+TRAINING_DATA_ARCHIVE = "training_data.tar.gz"
+
+
+class RemoteTrainingDataImporter(TrainingDataImporter):
+    """Remote `TrainingFileImporter` implementation.
+
+    Fetches training data from a remote storage and extracts it to a local directory.
+    Extracted training data is then used to load flows, NLU, stories,
+        domain, and config files.
+    """
+
+    def __init__(
+        self,
+        config_file: Optional[Text] = None,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+        project_directory: Optional[Text] = None,
+        remote_storage: Optional[StorageType] = None,
+        training_data_path: Optional[Text] = None,
+    ):
+        """Initializes `RemoteTrainingDataImporter`.
+
+        Args:
+            config_file: Path to the model configuration file.
+            domain_path: Path to the domain file.
+            training_data_paths: List of paths to the training data files.
+            project_directory: Path to the project directory.
+            remote_storage: Storage to use to load the training data.
+            training_data_path: Path to the training data.
+        """
+        self.remote_storage = remote_storage
+        self.training_data_path = training_data_path
+
+        self.extracted_path = self._fetch_and_extract_training_archive(
+            TRAINING_DATA_ARCHIVE, self.training_data_path
+        )
+
+        self._nlu_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.data.is_nlu_file
+        )
+        self._story_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_stories_file
+        )
+        self._flow_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.core.flows.yaml_flows_io.is_flows_file
+        )
+        self._conversation_test_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_test_stories_file
+        )
+
+        self.config_file = config_file
+
+    def _fetch_training_archive(
+        self, training_file: str, training_data_path: Optional[str] = None
+    ) -> str:
+        """Fetches training files from remote storage."""
+        from rasa.core.persistor import get_persistor
+
+        persistor = get_persistor(self.remote_storage)
+        if persistor is None:
+            raise RasaException(
+                f"Could not find a persistor for "
+                f"the storage type '{self.remote_storage}'."
+            )
+
+        return persistor.retrieve(training_file, training_data_path)
+
+    def _fetch_and_extract_training_archive(
+        self, training_file: str, training_data_path: Optional[Text] = None
+    ) -> Optional[str]:
+        """Fetches and extracts training files from remote storage.
+
+        If the `training_data_path` is not provided, the training
+        data is extracted to the current working directory.
+
+        Args:
+            training_file: Name of the training data archive file.
+            training_data_path: Path to the training data.
+
+        Returns:
+            Path to the extracted training data.
+        """
+
+        if training_data_path is None:
+            training_data_path = os.path.join(os.getcwd(), "data")
+
+        if os.path.isfile(training_data_path):
+            raise ValueError(
+                f"Training data path '{training_data_path}' is a file. "
+                f"Please provide a directory path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.fetch_training_archive",
+            training_data_path=training_data_path,
+        )
+        training_archive_file_path = self._fetch_training_archive(
+            training_file, training_data_path
+        )
+
+        if not os.path.isfile(training_archive_file_path):
+            raise FileNotFoundError(
+                f"Training data archive '{training_archive_file_path}' not found. "
+                f"Please make sure to provide the correct path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.extract_training_archive",
+            training_archive_file_path=training_archive_file_path,
+            training_data_path=training_data_path,
+        )
+        with TarSafe.open(training_archive_file_path, "r:gz") as tar:
+            tar.extractall(path=training_data_path)
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.remove_downloaded_archive",
+            training_data_path=training_data_path,
+        )
+        os.remove(training_archive_file_path)
+        return training_data_path
+
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        if not self.config_file or not os.path.exists(self.config_file):
+            structlogger.debug(
+                "rasa.importers.remote_training_data_importer.no_config_file",
+                message="No configuration file was provided to the RasaFileImporter.",
+            )
+            return {}
+
+        config = read_model_configuration(self.config_file)
+        return config
+
+    @rasa.shared.utils.common.cached_method
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return self.config_file
+
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_files, self.get_domain(), exclusion_percentage
+        )
+
+    def get_flows(self) -> FlowsList:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.flows_from_paths(self._flow_files)
+
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._conversation_test_files, self.get_domain()
+        )
+
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_files, language)
+
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domain = Domain.empty()
+        domain_path = f"{self.extracted_path}"
+        try:
+            domain = Domain.load(domain_path)
+        except InvalidDomain as e:
+            rasa.shared.utils.io.raise_warning(
+                f"Loading domain from '{domain_path}' failed. Using "
+                f"empty domain. Error: '{e}'"
+            )
+
+        return domain

rasa/shared/importers/utils.py

@@ -29,6 +29,8 @@ def flows_from_paths(files: List[Text]) -> FlowsList:
 
     flows = FlowsList(underlying_flows=[])
     for file in files:
-        flows = flows.merge(YAMLFlowsReader.read_from_file(file))
+        flows = flows.merge(
+            YAMLFlowsReader.read_from_file(file), ignore_duplicates=False
+        )
     flows.validate()
     return flows