rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/shared/importers/multi_project.py

@@ -0,0 +1,215 @@
+import os
+from functools import reduce
+from typing import Any, Dict, List, Optional, Set, Text, Union
+
+import structlog
+
+import rasa.shared.data
+import rasa.shared.utils.io
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.common import cached_method, mark_as_experimental_feature
+from rasa.shared.utils.yaml import read_config_file, read_model_configuration
+
+structlogger = structlog.get_logger()
+
+
+class MultiProjectImporter(TrainingDataImporter):
+    def __init__(
+        self,
+        config_file: Text,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+        project_directory: Optional[Text] = None,
+    ):
+        self.config = read_model_configuration(config_file)
+        if domain_path:
+            self._domain_paths = [domain_path]
+        else:
+            self._domain_paths = []
+        self._story_paths = []
+        self._e2e_story_paths: List[Text] = []
+        self._nlu_paths = []
+        self._imports: List[Text] = []
+        self._additional_paths = training_data_paths or []
+        self._project_directory = project_directory or os.path.dirname(config_file)
+
+        self._init_from_dict(self.config, self._project_directory)
+
+        extra_nlu_files = rasa.shared.data.get_data_files(
+            training_data_paths, rasa.shared.data.is_nlu_file
+        )
+        extra_story_files = rasa.shared.data.get_data_files(
+            training_data_paths, YAMLStoryReader.is_stories_file
+        )
+        self._story_paths += extra_story_files
+        self._nlu_paths += extra_nlu_files
+
+        structlogger.debug(
+            "multi_project_importer.initialisation",
+            event_info=(
+                "Selected projects: {}".format(
+                    "".join([f"\n-{i}" for i in self._imports])
+                )
+            ),
+        )
+
+        mark_as_experimental_feature(feature_name="MultiProjectImporter")
+
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return None
+
+    def _init_from_path(self, path: Text) -> None:
+        if os.path.isfile(path):
+            self._init_from_file(path)
+        elif os.path.isdir(path):
+            self._init_from_directory(path)
+
+    def _init_from_file(self, path: Text) -> None:
+        path = os.path.abspath(path)
+        if os.path.exists(path) and rasa.shared.data.is_config_file(path):
+            config = read_config_file(path)
+
+            parent_directory = os.path.dirname(path)
+            self._init_from_dict(config, parent_directory)
+        else:
+            rasa.shared.utils.io.raise_warning(
+                f"'{path}' does not exist or is not a valid config file."
+            )
+
+    def _init_from_dict(self, _dict: Dict[Text, Any], parent_directory: Text) -> None:
+        imports = _dict.get("imports") or []
+        imports = [os.path.join(parent_directory, i) for i in imports]
+        # clean out relative paths
+        imports = [os.path.abspath(i) for i in imports]
+
+        # remove duplication
+        import_candidates = []
+        for i in imports:
+            if i not in import_candidates and not self._is_explicitly_imported(i):
+                import_candidates.append(i)
+
+        self._imports.extend(import_candidates)
+
+        # import config files from paths which have not been processed so far
+        for p in import_candidates:
+            self._init_from_path(p)
+
+    def _is_explicitly_imported(self, path: Text) -> bool:
+        return not self.no_skills_selected() and self.is_imported(path)
+
+    def _init_from_directory(self, path: Text) -> None:
+        for parent, _, files in os.walk(path, followlinks=True):
+            for file in files:
+                full_path = os.path.join(parent, file)
+                if not self.is_imported(full_path):
+                    # Check next file
+                    continue
+
+                if YAMLStoryReader.is_test_stories_file(full_path):
+                    self._e2e_story_paths.append(full_path)
+                elif Domain.is_domain_file(full_path):
+                    self._domain_paths.append(full_path)
+                elif rasa.shared.data.is_nlu_file(full_path):
+                    self._nlu_paths.append(full_path)
+                elif YAMLStoryReader.is_stories_file(full_path):
+                    self._story_paths.append(full_path)
+                elif rasa.shared.data.is_config_file(full_path):
+                    self._init_from_file(full_path)
+
+    def no_skills_selected(self) -> bool:
+        return not self._imports
+
+    def training_paths(self) -> Set[Text]:
+        """Returns the paths which should be searched for training data."""
+        # only include extra paths if they are not part of the current project directory
+        training_paths = {
+            i
+            for i in self._imports
+            if not self._project_directory or self._project_directory not in i
+        }
+
+        if self._project_directory:
+            training_paths.add(self._project_directory)
+
+        return training_paths
+
+    @cached_method
+    def is_imported(self, path: Text) -> bool:
+        """Checks whether a path is imported by a skill.
+
+        Args:
+            path: File or directory path which should be checked.
+
+        Returns:
+            `True` if path is imported by a skill, `False` if not.
+        """
+        absolute_path = os.path.abspath(path)
+
+        return (
+            self.no_skills_selected()
+            or self._is_in_project_directory(absolute_path)
+            or self._is_in_additional_paths(absolute_path)
+            or self._is_in_imported_paths(absolute_path)
+        )
+
+    def _is_in_project_directory(self, path: Text) -> bool:
+        if os.path.isfile(path):
+            parent_directory = os.path.abspath(os.path.dirname(path))
+
+            return parent_directory == self._project_directory
+        else:
+            return path == self._project_directory
+
+    def _is_in_additional_paths(self, path: Text) -> bool:
+        included = path in self._additional_paths
+
+        if not included and os.path.isfile(path):
+            parent_directory = os.path.abspath(os.path.dirname(path))
+            included = parent_directory in self._additional_paths
+
+        return included
+
+    def _is_in_imported_paths(self, path: Text) -> bool:
+        return any(
+            [rasa.shared.utils.io.is_subdirectory(path, i) for i in self._imports]
+        )
+
+    @cached_method
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domains = [Domain.load(path) for path in self._domain_paths]
+        return reduce(
+            lambda merged, other: merged.merge(other),
+            domains,
+            Domain.empty(),
+        )
+
+    @cached_method
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_paths, self.get_domain(), exclusion_percentage
+        )
+
+    @cached_method
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(self._e2e_story_paths, self.get_domain())
+
+    @cached_method
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        return self.config
+
+    @cached_method
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_paths, language)

rasa/shared/importers/rasa.py

@@ -0,0 +1,108 @@
+import logging
+import os
+from typing import Dict, List, Optional, Text, Union
+
+import rasa.shared.core.flows.yaml_flows_io
+from rasa.shared.core.flows import FlowsList
+
+import rasa.shared.data
+import rasa.shared.utils.io
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.core.domain import InvalidDomain, Domain
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.utils.common import cached_method
+from rasa.shared.utils.yaml import read_model_configuration
+
+logger = logging.getLogger(__name__)
+
+
+class RasaFileImporter(TrainingDataImporter):
+    """Default `TrainingFileImporter` implementation."""
+
+    def __init__(
+        self,
+        config_file: Optional[Text] = None,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+        expand_env_vars: bool = True,
+    ):
+        self.expand_env_vars = expand_env_vars
+        self._domain_path = domain_path
+
+        self._nlu_files = rasa.shared.data.get_data_files(
+            training_data_paths, rasa.shared.data.is_nlu_file
+        )
+        self._story_files = rasa.shared.data.get_data_files(
+            training_data_paths, YAMLStoryReader.is_stories_file
+        )
+        self._flow_files = rasa.shared.data.get_data_files(
+            training_data_paths, rasa.shared.core.flows.yaml_flows_io.is_flows_file
+        )
+        self._conversation_test_files = rasa.shared.data.get_data_files(
+            training_data_paths, YAMLStoryReader.is_test_stories_file
+        )
+
+        self.config_file = config_file
+
+    @cached_method
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        if not self.config_file or not os.path.exists(self.config_file):
+            logger.debug("No configuration file was provided to the RasaFileImporter.")
+            return {}
+
+        config = read_model_configuration(
+            self.config_file, expand_env_vars=self.expand_env_vars
+        )
+        return config
+
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return self.config_file
+
+    @cached_method
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_files, self.get_domain(), exclusion_percentage
+        )
+
+    @cached_method
+    def get_flows(self) -> FlowsList:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.flows_from_paths(self._flow_files)
+
+    @cached_method
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._conversation_test_files, self.get_domain()
+        )
+
+    @cached_method
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_files, language)
+
+    @cached_method
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domain = Domain.empty()
+
+        # If domain path is None, return an empty domain
+        if not self._domain_path:
+            return domain
+        try:
+            domain = Domain.load(self._domain_path)
+        except InvalidDomain as e:
+            rasa.shared.utils.io.raise_warning(
+                f"Loading domain from '{self._domain_path}' failed. Using "
+                f"empty domain. Error: '{e}'"
+            )
+
+        return domain

rasa/shared/importers/remote_importer.py

@@ -0,0 +1,196 @@
+import os
+from typing import Dict, List, Optional, Text, Union
+
+import structlog
+from tarsafe import TarSafe
+
+import rasa.shared.core.flows.yaml_flows_io
+import rasa.shared.data
+import rasa.shared.utils.common
+import rasa.shared.utils.io
+from rasa.core.persistor import StorageType
+from rasa.shared.core.domain import Domain, InvalidDomain
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.exceptions import RasaException
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.yaml import read_model_configuration
+
+structlogger = structlog.get_logger()
+
+TRAINING_DATA_ARCHIVE = "training_data.tar.gz"
+
+
+class RemoteTrainingDataImporter(TrainingDataImporter):
+    """Remote `TrainingFileImporter` implementation.
+
+    Fetches training data from a remote storage and extracts it to a local directory.
+    Extracted training data is then used to load flows, NLU, stories,
+        domain, and config files.
+    """
+
+    def __init__(
+        self,
+        config_file: Optional[Text] = None,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+        project_directory: Optional[Text] = None,
+        remote_storage: Optional[StorageType] = None,
+        training_data_path: Optional[Text] = None,
+    ):
+        """Initializes `RemoteTrainingDataImporter`.
+
+        Args:
+            config_file: Path to the model configuration file.
+            domain_path: Path to the domain file.
+            training_data_paths: List of paths to the training data files.
+            project_directory: Path to the project directory.
+            remote_storage: Storage to use to load the training data.
+            training_data_path: Path to the training data.
+        """
+        self.remote_storage = remote_storage
+        self.training_data_path = training_data_path
+
+        self.extracted_path = self._fetch_and_extract_training_archive(
+            TRAINING_DATA_ARCHIVE, self.training_data_path
+        )
+
+        self._nlu_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.data.is_nlu_file
+        )
+        self._story_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_stories_file
+        )
+        self._flow_files = rasa.shared.data.get_data_files(
+            self.extracted_path, rasa.shared.core.flows.yaml_flows_io.is_flows_file
+        )
+        self._conversation_test_files = rasa.shared.data.get_data_files(
+            self.extracted_path, YAMLStoryReader.is_test_stories_file
+        )
+
+        self.config_file = config_file
+
+    def _fetch_training_archive(
+        self, training_file: str, training_data_path: Optional[str] = None
+    ) -> str:
+        """Fetches training files from remote storage."""
+        from rasa.core.persistor import get_persistor
+
+        persistor = get_persistor(self.remote_storage)
+        if persistor is None:
+            raise RasaException(
+                f"Could not find a persistor for "
+                f"the storage type '{self.remote_storage}'."
+            )
+
+        return persistor.retrieve(training_file, training_data_path)
+
+    def _fetch_and_extract_training_archive(
+        self, training_file: str, training_data_path: Optional[Text] = None
+    ) -> Optional[str]:
+        """Fetches and extracts training files from remote storage.
+
+        If the `training_data_path` is not provided, the training
+        data is extracted to the current working directory.
+
+        Args:
+            training_file: Name of the training data archive file.
+            training_data_path: Path to the training data.
+
+        Returns:
+            Path to the extracted training data.
+        """
+
+        if training_data_path is None:
+            training_data_path = os.path.join(os.getcwd(), "data")
+
+        if os.path.isfile(training_data_path):
+            raise ValueError(
+                f"Training data path '{training_data_path}' is a file. "
+                f"Please provide a directory path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.fetch_training_archive",
+            training_data_path=training_data_path,
+        )
+        training_archive_file_path = self._fetch_training_archive(
+            training_file, training_data_path
+        )
+
+        if not os.path.isfile(training_archive_file_path):
+            raise FileNotFoundError(
+                f"Training data archive '{training_archive_file_path}' not found. "
+                f"Please make sure to provide the correct path."
+            )
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.extract_training_archive",
+            training_archive_file_path=training_archive_file_path,
+            training_data_path=training_data_path,
+        )
+        with TarSafe.open(training_archive_file_path, "r:gz") as tar:
+            tar.extractall(path=training_data_path)
+
+        structlogger.debug(
+            "rasa.importers.remote_training_data_importer.remove_downloaded_archive",
+            training_data_path=training_data_path,
+        )
+        os.remove(training_archive_file_path)
+        return training_data_path
+
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        if not self.config_file or not os.path.exists(self.config_file):
+            structlogger.debug(
+                "rasa.importers.remote_training_data_importer.no_config_file",
+                message="No configuration file was provided to the RasaFileImporter.",
+            )
+            return {}
+
+        config = read_model_configuration(self.config_file)
+        return config
+
+    @rasa.shared.utils.common.cached_method
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return self.config_file
+
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_files, self.get_domain(), exclusion_percentage
+        )
+
+    def get_flows(self) -> FlowsList:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.flows_from_paths(self._flow_files)
+
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._conversation_test_files, self.get_domain()
+        )
+
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_files, language)
+
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domain = Domain.empty()
+        domain_path = f"{self.extracted_path}"
+        try:
+            domain = Domain.load(domain_path)
+        except InvalidDomain as e:
+            rasa.shared.utils.io.raise_warning(
+                f"Loading domain from '{domain_path}' failed. Using "
+                f"empty domain. Error: '{e}'"
+            )
+
+        return domain

rasa/shared/importers/utils.py

@@ -0,0 +1,36 @@
+from typing import Iterable, Text, Optional, List
+
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.nlu.training_data.training_data import TrainingData
+
+
+def training_data_from_paths(paths: Iterable[Text], language: Text) -> TrainingData:
+    from rasa.shared.nlu.training_data import loading
+
+    training_data_sets = [loading.load_data(nlu_file, language) for nlu_file in paths]
+    return TrainingData().merge(*training_data_sets)
+
+
+def story_graph_from_paths(
+    files: List[Text], domain: Domain, exclusion_percentage: Optional[int] = None
+) -> StoryGraph:
+    """Returns the `StoryGraph` from paths."""
+    from rasa.shared.core.training_data import loading
+
+    story_steps = loading.load_data_from_files(files, domain, exclusion_percentage)
+    return StoryGraph(story_steps)
+
+
+def flows_from_paths(files: List[Text]) -> FlowsList:
+    """Returns the flows from paths."""
+    from rasa.shared.core.flows.yaml_flows_io import YAMLFlowsReader
+
+    flows = FlowsList(underlying_flows=[])
+    for file in files:
+        flows = flows.merge(
+            YAMLFlowsReader.read_from_file(file), ignore_duplicates=False
+        )
+    flows.validate()
+    return flows

rasa/shared/nlu/constants.py

@@ -0,0 +1,53 @@
+TEXT = "text"
+TEXT_TOKENS = "text_tokens"
+INTENT = "intent"
+COMMANDS = "commands"
+PREDICTED_COMMANDS = "predicted_commands"
+PROMPTS = "prompts"
+KEY_USER_PROMPT = "user_prompt"
+KEY_SYSTEM_PROMPT = "system_prompt"
+LLM_COMMANDS = "llm_commands"  # needed for fine-tuning
+LLM_PROMPT = "llm_prompt"  # needed for fine-tuning
+FLOWS_FROM_SEMANTIC_SEARCH = "flows_from_semantic_search"
+FLOWS_IN_PROMPT = "flows_in_prompt"
+NOT_INTENT = "not_intent"
+RESPONSE = "response"
+RESPONSE_SELECTOR = "response_selector"
+INTENT_RESPONSE_KEY = "intent_response_key"
+ACTION_TEXT = "action_text"
+ACTION_NAME = "action_name"
+INTENT_NAME_KEY = "name"
+FULL_RETRIEVAL_INTENT_NAME_KEY = "full_retrieval_intent_name"
+METADATA = "metadata"
+METADATA_INTENT = "intent"
+METADATA_EXAMPLE = "example"
+METADATA_MODEL_ID = "model_id"
+INTENT_RANKING_KEY = "intent_ranking"
+PREDICTED_CONFIDENCE_KEY = "confidence"
+
+RESPONSE_IDENTIFIER_DELIMITER = "/"
+
+FEATURE_TYPE_SENTENCE = "sentence"
+FEATURE_TYPE_SEQUENCE = "sequence"
+VALID_FEATURE_TYPES = [FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE]
+
+EXTRACTOR = "extractor"
+PRETRAINED_EXTRACTORS = {"DucklingEntityExtractor", "SpacyEntityExtractor"}
+TRAINABLE_EXTRACTORS = {"MitieEntityExtractor", "CRFEntityExtractor", "DIETClassifier"}
+
+ENTITIES = "entities"
+ENTITY_TAGS = "entity_tags"
+ENTITY_ATTRIBUTE_TYPE = "entity"
+ENTITY_ATTRIBUTE_GROUP = "group"
+ENTITY_ATTRIBUTE_ROLE = "role"
+ENTITY_ATTRIBUTE_VALUE = "value"
+ENTITY_ATTRIBUTE_START = "start"
+ENTITY_ATTRIBUTE_END = "end"
+ENTITY_ATTRIBUTE_TEXT = "text"
+ENTITY_ATTRIBUTE_CONFIDENCE = "confidence"
+NO_ENTITY_TAG = "O"
+SPLIT_ENTITIES_BY_COMMA = "split_entities_by_comma"
+SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE = True
+SINGLE_ENTITY_ALLOWED_INTERLEAVING_CHARSET = {".", ",", " ", ";"}
+
+SET_SLOT_COMMAND = "set slot"

rasa/shared/nlu/interpreter.py

@@ -0,0 +1,10 @@
+class NaturalLanguageInterpreter:
+    """Remove once all old components are deleted."""
+
+    pass
+
+
+class RegexInterpreter:
+    """Remove once all old components are deleted."""
+
+    pass