rasa-pro 3.8.16__py3-none-any.whl

rasa/shared/importers/multi_project.py

@@ -0,0 +1,203 @@
+import logging
+from functools import reduce
+from typing import Text, Set, Dict, Optional, List, Union, Any
+import os
+
+import rasa.shared.data
+import rasa.shared.utils.io
+from rasa.shared.core.domain import Domain
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.importers import utils
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.utils.common import mark_as_experimental_feature
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.utils.yaml import read_config_file, read_model_configuration
+
+logger = logging.getLogger(__name__)
+
+
+class MultiProjectImporter(TrainingDataImporter):
+    def __init__(
+        self,
+        config_file: Text,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+        project_directory: Optional[Text] = None,
+    ):
+        self.config = read_model_configuration(config_file)
+        if domain_path:
+            self._domain_paths = [domain_path]
+        else:
+            self._domain_paths = []
+        self._story_paths = []
+        self._e2e_story_paths: List[Text] = []
+        self._nlu_paths = []
+        self._imports: List[Text] = []
+        self._additional_paths = training_data_paths or []
+        self._project_directory = project_directory or os.path.dirname(config_file)
+
+        self._init_from_dict(self.config, self._project_directory)
+
+        extra_nlu_files = rasa.shared.data.get_data_files(
+            training_data_paths, rasa.shared.data.is_nlu_file
+        )
+        extra_story_files = rasa.shared.data.get_data_files(
+            training_data_paths, YAMLStoryReader.is_stories_file
+        )
+        self._story_paths += extra_story_files
+        self._nlu_paths += extra_nlu_files
+
+        logger.debug(
+            "Selected projects: {}".format("".join([f"\n-{i}" for i in self._imports]))
+        )
+
+        mark_as_experimental_feature(feature_name="MultiProjectImporter")
+
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return None
+
+    def _init_from_path(self, path: Text) -> None:
+        if os.path.isfile(path):
+            self._init_from_file(path)
+        elif os.path.isdir(path):
+            self._init_from_directory(path)
+
+    def _init_from_file(self, path: Text) -> None:
+        path = os.path.abspath(path)
+        if os.path.exists(path) and rasa.shared.data.is_config_file(path):
+            config = read_config_file(path)
+
+            parent_directory = os.path.dirname(path)
+            self._init_from_dict(config, parent_directory)
+        else:
+            rasa.shared.utils.io.raise_warning(
+                f"'{path}' does not exist or is not a valid config file."
+            )
+
+    def _init_from_dict(self, _dict: Dict[Text, Any], parent_directory: Text) -> None:
+        imports = _dict.get("imports") or []
+        imports = [os.path.join(parent_directory, i) for i in imports]
+        # clean out relative paths
+        imports = [os.path.abspath(i) for i in imports]
+
+        # remove duplication
+        import_candidates = []
+        for i in imports:
+            if i not in import_candidates and not self._is_explicitly_imported(i):
+                import_candidates.append(i)
+
+        self._imports.extend(import_candidates)
+
+        # import config files from paths which have not been processed so far
+        for p in import_candidates:
+            self._init_from_path(p)
+
+    def _is_explicitly_imported(self, path: Text) -> bool:
+        return not self.no_skills_selected() and self.is_imported(path)
+
+    def _init_from_directory(self, path: Text) -> None:
+        for parent, _, files in os.walk(path, followlinks=True):
+            for file in files:
+                full_path = os.path.join(parent, file)
+                if not self.is_imported(full_path):
+                    # Check next file
+                    continue
+
+                if YAMLStoryReader.is_test_stories_file(full_path):
+                    self._e2e_story_paths.append(full_path)
+                elif Domain.is_domain_file(full_path):
+                    self._domain_paths.append(full_path)
+                elif rasa.shared.data.is_nlu_file(full_path):
+                    self._nlu_paths.append(full_path)
+                elif YAMLStoryReader.is_stories_file(full_path):
+                    self._story_paths.append(full_path)
+                elif rasa.shared.data.is_config_file(full_path):
+                    self._init_from_file(full_path)
+
+    def no_skills_selected(self) -> bool:
+        return not self._imports
+
+    def training_paths(self) -> Set[Text]:
+        """Returns the paths which should be searched for training data."""
+        # only include extra paths if they are not part of the current project directory
+        training_paths = {
+            i
+            for i in self._imports
+            if not self._project_directory or self._project_directory not in i
+        }
+
+        if self._project_directory:
+            training_paths.add(self._project_directory)
+
+        return training_paths
+
+    def is_imported(self, path: Text) -> bool:
+        """Checks whether a path is imported by a skill.
+
+        Args:
+            path: File or directory path which should be checked.
+
+        Returns:
+            `True` if path is imported by a skill, `False` if not.
+        """
+        absolute_path = os.path.abspath(path)
+
+        return (
+            self.no_skills_selected()
+            or self._is_in_project_directory(absolute_path)
+            or self._is_in_additional_paths(absolute_path)
+            or self._is_in_imported_paths(absolute_path)
+        )
+
+    def _is_in_project_directory(self, path: Text) -> bool:
+        if os.path.isfile(path):
+            parent_directory = os.path.abspath(os.path.dirname(path))
+
+            return parent_directory == self._project_directory
+        else:
+            return path == self._project_directory
+
+    def _is_in_additional_paths(self, path: Text) -> bool:
+        included = path in self._additional_paths
+
+        if not included and os.path.isfile(path):
+            parent_directory = os.path.abspath(os.path.dirname(path))
+            included = parent_directory in self._additional_paths
+
+        return included
+
+    def _is_in_imported_paths(self, path: Text) -> bool:
+        return any(
+            [rasa.shared.utils.io.is_subdirectory(path, i) for i in self._imports]
+        )
+
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domains = [Domain.load(path) for path in self._domain_paths]
+        return reduce(
+            lambda merged, other: merged.merge(other),
+            domains,
+            Domain.empty(),
+        )
+
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_paths, self.get_domain(), exclusion_percentage
+        )
+
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(self._e2e_story_paths, self.get_domain())
+
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        return self.config
+
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_paths, language)

rasa/shared/importers/rasa.py

@@ -0,0 +1,100 @@
+import logging
+import os
+from typing import Dict, List, Optional, Text, Union
+
+import rasa.shared.core.flows.yaml_flows_io
+from rasa.shared.core.flows import FlowsList
+
+import rasa.shared.data
+import rasa.shared.utils.common
+import rasa.shared.utils.io
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.importers import utils
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.core.domain import InvalidDomain, Domain
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.utils.yaml import read_model_configuration
+
+logger = logging.getLogger(__name__)
+
+
+class RasaFileImporter(TrainingDataImporter):
+    """Default `TrainingFileImporter` implementation."""
+
+    def __init__(
+        self,
+        config_file: Optional[Text] = None,
+        domain_path: Optional[Text] = None,
+        training_data_paths: Optional[Union[List[Text], Text]] = None,
+    ):
+
+        self._domain_path = domain_path
+
+        self._nlu_files = rasa.shared.data.get_data_files(
+            training_data_paths, rasa.shared.data.is_nlu_file
+        )
+        self._story_files = rasa.shared.data.get_data_files(
+            training_data_paths, YAMLStoryReader.is_stories_file
+        )
+        self._flow_files = rasa.shared.data.get_data_files(
+            training_data_paths, rasa.shared.core.flows.yaml_flows_io.is_flows_file
+        )
+        self._conversation_test_files = rasa.shared.data.get_data_files(
+            training_data_paths, YAMLStoryReader.is_test_stories_file
+        )
+
+        self.config_file = config_file
+
+    def get_config(self) -> Dict:
+        """Retrieves model config (see parent class for full docstring)."""
+        if not self.config_file or not os.path.exists(self.config_file):
+            logger.debug("No configuration file was provided to the RasaFileImporter.")
+            return {}
+
+        config = read_model_configuration(self.config_file)
+        return config
+
+    @rasa.shared.utils.common.cached_method
+    def get_config_file_for_auto_config(self) -> Optional[Text]:
+        """Returns config file path for auto-config only if there is a single one."""
+        return self.config_file
+
+    def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._story_files, self.get_domain(), exclusion_percentage
+        )
+
+    def get_flows(self) -> FlowsList:
+        """Retrieves training stories / rules (see parent class for full docstring)."""
+        return utils.flows_from_paths(self._flow_files)
+
+    def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves conversation test stories (see parent class for full docstring)."""
+        return utils.story_graph_from_paths(
+            self._conversation_test_files, self.get_domain()
+        )
+
+    def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
+        """Retrieves NLU training data (see parent class for full docstring)."""
+        return utils.training_data_from_paths(self._nlu_files, language)
+
+    def get_domain(self) -> Domain:
+        """Retrieves model domain (see parent class for full docstring)."""
+        domain = Domain.empty()
+
+        # If domain path is None, return an empty domain
+        if not self._domain_path:
+            return domain
+        try:
+            domain = Domain.load(self._domain_path)
+        except InvalidDomain as e:
+            rasa.shared.utils.io.raise_warning(
+                f"Loading domain from '{self._domain_path}' failed. Using "
+                f"empty domain. Error: '{e}'"
+            )
+
+        return domain

rasa/shared/importers/utils.py

@@ -0,0 +1,34 @@
+from typing import Iterable, Text, Optional, List
+
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.nlu.training_data.training_data import TrainingData
+
+
+def training_data_from_paths(paths: Iterable[Text], language: Text) -> TrainingData:
+    from rasa.shared.nlu.training_data import loading
+
+    training_data_sets = [loading.load_data(nlu_file, language) for nlu_file in paths]
+    return TrainingData().merge(*training_data_sets)
+
+
+def story_graph_from_paths(
+    files: List[Text], domain: Domain, exclusion_percentage: Optional[int] = None
+) -> StoryGraph:
+    """Returns the `StoryGraph` from paths."""
+    from rasa.shared.core.training_data import loading
+
+    story_steps = loading.load_data_from_files(files, domain, exclusion_percentage)
+    return StoryGraph(story_steps)
+
+
+def flows_from_paths(files: List[Text]) -> FlowsList:
+    """Returns the flows from paths."""
+    from rasa.shared.core.flows.yaml_flows_io import YAMLFlowsReader
+
+    flows = FlowsList(underlying_flows=[])
+    for file in files:
+        flows = flows.merge(YAMLFlowsReader.read_from_file(file))
+    flows.validate()
+    return flows

rasa/shared/nlu/constants.py

@@ -0,0 +1,45 @@
+TEXT = "text"
+TEXT_TOKENS = "text_tokens"
+INTENT = "intent"
+COMMANDS = "commands"
+FLOWS_FROM_SEMANTIC_SEARCH = "flows_from_semantic_search"
+FLOWS_IN_PROMPT = "flows_in_prompt"
+NOT_INTENT = "not_intent"
+RESPONSE = "response"
+RESPONSE_SELECTOR = "response_selector"
+INTENT_RESPONSE_KEY = "intent_response_key"
+ACTION_TEXT = "action_text"
+ACTION_NAME = "action_name"
+INTENT_NAME_KEY = "name"
+FULL_RETRIEVAL_INTENT_NAME_KEY = "full_retrieval_intent_name"
+METADATA = "metadata"
+METADATA_INTENT = "intent"
+METADATA_EXAMPLE = "example"
+METADATA_MODEL_ID = "model_id"
+INTENT_RANKING_KEY = "intent_ranking"
+PREDICTED_CONFIDENCE_KEY = "confidence"
+
+RESPONSE_IDENTIFIER_DELIMITER = "/"
+
+FEATURE_TYPE_SENTENCE = "sentence"
+FEATURE_TYPE_SEQUENCE = "sequence"
+VALID_FEATURE_TYPES = [FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE]
+
+EXTRACTOR = "extractor"
+PRETRAINED_EXTRACTORS = {"DucklingEntityExtractor", "SpacyEntityExtractor"}
+TRAINABLE_EXTRACTORS = {"MitieEntityExtractor", "CRFEntityExtractor", "DIETClassifier"}
+
+ENTITIES = "entities"
+ENTITY_TAGS = "entity_tags"
+ENTITY_ATTRIBUTE_TYPE = "entity"
+ENTITY_ATTRIBUTE_GROUP = "group"
+ENTITY_ATTRIBUTE_ROLE = "role"
+ENTITY_ATTRIBUTE_VALUE = "value"
+ENTITY_ATTRIBUTE_START = "start"
+ENTITY_ATTRIBUTE_END = "end"
+ENTITY_ATTRIBUTE_TEXT = "text"
+ENTITY_ATTRIBUTE_CONFIDENCE = "confidence"
+NO_ENTITY_TAG = "O"
+SPLIT_ENTITIES_BY_COMMA = "split_entities_by_comma"
+SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE = True
+SINGLE_ENTITY_ALLOWED_INTERLEAVING_CHARSET = {".", ",", " ", ";"}

rasa/shared/nlu/interpreter.py

@@ -0,0 +1,10 @@
+class NaturalLanguageInterpreter:
+    """Remove once all old components are deleted."""
+
+    pass
+
+
+class RegexInterpreter:
+    """Remove once all old components are deleted."""
+
+    pass

rasa/shared/nlu/training_data/entities_parser.py

@@ -0,0 +1,209 @@
+import re
+from json import JSONDecodeError
+from typing import Text, List, Dict, Match, Optional, NamedTuple, Any
+import logging
+
+import rasa.shared.nlu.training_data.util
+from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU
+from rasa.shared.exceptions import InvalidEntityFormatException
+from rasa.shared.nlu.constants import (
+    ENTITY_ATTRIBUTE_VALUE,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_GROUP,
+    ENTITY_ATTRIBUTE_ROLE,
+)
+from rasa.shared.nlu.training_data.message import Message
+
+
+GROUP_ENTITY_VALUE = "value"
+GROUP_ENTITY_TYPE = "entity"
+GROUP_ENTITY_DICT = "entity_dict"
+GROUP_ENTITY_DICT_LIST = "list_entity_dicts"
+GROUP_ENTITY_TEXT = "entity_text"
+GROUP_COMPLETE_MATCH = 0
+
+# regex for: `[entity_text]((entity_type(:entity_synonym)?)|{entity_dict}|[list_entity_dicts])` # noqa: E501
+ENTITY_REGEX = re.compile(
+    r"\[(?P<entity_text>[^\]]+?)\](\((?P<entity>[^:)]+?)(?:\:(?P<value>[^)]+))?\)|\{(?P<entity_dict>[^}]+?)\}|\[(?P<list_entity_dicts>.*?)\])"  # noqa: E501
+)
+
+SINGLE_ENTITY_DICT = re.compile(r"{(?P<entity_dict>[^}]+?)\}")
+
+logger = logging.getLogger(__name__)
+
+
+class EntityAttributes(NamedTuple):
+    """Attributes of an entity defined in markdown data."""
+
+    type: Text
+    value: Text
+    text: Text
+    group: Optional[Text]
+    role: Optional[Text]
+
+
+def find_entities_in_training_example(example: Text) -> List[Dict[Text, Any]]:
+    """Extracts entities from an annotated utterance.
+
+    Args:
+        example: Annotated utterance.
+
+    Returns:
+        Extracted entities.
+    """
+    entities = []
+    offset = 0
+
+    for match in re.finditer(ENTITY_REGEX, example):
+        logger.debug(f"Entity annotation regex match: {match}")
+        if match.groupdict()[GROUP_ENTITY_DICT] or match.groupdict()[GROUP_ENTITY_TYPE]:
+            # Text is annotated with a single entity
+            entity_attributes = extract_entity_attributes(match)
+
+            start_index = match.start() - offset
+            end_index = start_index + len(entity_attributes.text)
+            offset += len(match.group(0)) - len(entity_attributes.text)
+
+            entity = rasa.shared.nlu.training_data.util.build_entity(
+                start_index,
+                end_index,
+                entity_attributes.value,
+                entity_attributes.type,
+                entity_attributes.role,
+                entity_attributes.group,
+            )
+            entities.append(entity)
+        else:
+            # Text is annotated with multiple entities for the same text
+            entity_text = match.groupdict()[GROUP_ENTITY_TEXT]
+
+            start_index = match.start() - offset
+            end_index = start_index + len(entity_text)
+            offset += len(match.group(0)) - len(entity_text)
+
+            for match_inner in re.finditer(
+                SINGLE_ENTITY_DICT, match.groupdict()[GROUP_ENTITY_DICT_LIST]
+            ):
+
+                entity_attributes = extract_entity_attributes_from_dict(
+                    entity_text=entity_text, match=match_inner
+                )
+
+                entity = rasa.shared.nlu.training_data.util.build_entity(
+                    start_index,
+                    end_index,
+                    entity_attributes.value,
+                    entity_attributes.type,
+                    entity_attributes.role,
+                    entity_attributes.group,
+                )
+                entities.append(entity)
+
+    return entities
+
+
+def extract_entity_attributes(match: Match) -> EntityAttributes:
+    """Extract the entity attributes, i.e. type, value, etc., from the
+    regex match.
+
+    Args:
+        match: Regex match to extract the entity attributes from.
+
+    Returns:
+        EntityAttributes object.
+    """
+    entity_text = match.groupdict()[GROUP_ENTITY_TEXT]
+
+    if match.groupdict()[GROUP_ENTITY_DICT]:
+        return extract_entity_attributes_from_dict(entity_text, match)
+
+    entity_type = match.groupdict()[GROUP_ENTITY_TYPE]
+
+    if match.groupdict()[GROUP_ENTITY_VALUE]:
+        entity_value = match.groupdict()[GROUP_ENTITY_VALUE]
+    else:
+        entity_value = entity_text
+
+    return EntityAttributes(entity_type, entity_value, entity_text, None, None)
+
+
+def extract_entity_attributes_from_dict(
+    entity_text: Text, match: Match
+) -> EntityAttributes:
+    """Extract entity attributes from dict format.
+
+    Args:
+        entity_text: Original entity text.
+        match: Regex match.
+
+    Returns:
+        Extracted entity attributes.
+    """
+    entity_dict_str = match.groupdict()[GROUP_ENTITY_DICT]
+    entity_dict = get_validated_dict(entity_dict_str)
+    return EntityAttributes(
+        entity_dict.get(ENTITY_ATTRIBUTE_TYPE),
+        entity_dict.get(ENTITY_ATTRIBUTE_VALUE, entity_text),
+        entity_text,
+        entity_dict.get(ENTITY_ATTRIBUTE_GROUP),
+        entity_dict.get(ENTITY_ATTRIBUTE_ROLE),
+    )
+
+
+def get_validated_dict(json_str: Text) -> Dict[Text, Text]:
+    """Converts the provided `json_str` to a valid dict containing the entity
+    attributes.
+
+    Users can specify entity roles, synonyms, groups for an entity in a dict, e.g.
+    [LA]{"entity": "city", "role": "to", "value": "Los Angeles"}.
+
+    Args:
+        json_str: The entity dict as string without "{}".
+
+    Raises:
+        SchemaValidationError if validation of parsed entity fails.
+        InvalidEntityFormatException if provided entity is not valid json.
+
+    Returns:
+        Deserialized and validated `json_str`.
+    """
+    import json
+    from rasa.shared.utils.yaml import validate_training_data
+    import rasa.shared.nlu.training_data.schemas.data_schema as schema
+
+    # add {} as they are not part of the regex
+    try:
+        data = json.loads(f"{{{json_str}}}")
+    except JSONDecodeError as e:
+        raise InvalidEntityFormatException.create_from(
+            e,
+            f"Incorrect training data format ('{{{json_str}}}'). "
+            f"More info at {DOCS_URL_TRAINING_DATA_NLU}",
+        ) from e
+
+    validate_training_data(data, schema.entity_dict_schema())
+
+    return data
+
+
+def replace_entities(training_example: Text) -> Text:
+    """Replace special symbols related to the entities in the provided
+       training example.
+
+    Args:
+        training_example: Original training example with special symbols.
+
+    Returns:
+        String with removed special symbols.
+    """
+    return re.sub(
+        ENTITY_REGEX, lambda m: m.groupdict()[GROUP_ENTITY_TEXT], training_example
+    )
+
+
+def parse_training_example(example: Text, intent: Optional[Text] = None) -> "Message":
+    """Extract entities and synonyms, and convert to plain text."""
+    entities = find_entities_in_training_example(example)
+    plain_text = replace_entities(example)
+
+    return Message.build(plain_text, intent, entities)