rasa-pro 3.8.16__py3-none-any.whl

rasa/shared/nlu/training_data/features.py

@@ -0,0 +1,374 @@
+from __future__ import annotations
+from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set
+import itertools
+
+import numpy as np
+import scipy.sparse
+
+import rasa.shared.utils.io
+import rasa.shared.nlu.training_data.util
+from rasa.shared.nlu.constants import FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE
+
+
+class Features:
+    """Stores the features produced by any featurizer."""
+
+    def __init__(
+        self,
+        features: Union[np.ndarray, scipy.sparse.spmatrix],
+        feature_type: Text,
+        attribute: Text,
+        origin: Union[Text, List[Text]],
+    ) -> None:
+        """Initializes the Features object.
+
+        Args:
+            features: The features.
+            feature_type: Type of the feature, e.g. FEATURE_TYPE_SENTENCE.
+            attribute: Message attribute, e.g. INTENT or TEXT.
+            origin: Name of the component that created the features.
+        """
+        self.features = features
+        self.type = feature_type
+        self.origin = origin
+        self.attribute = attribute
+        self._cached_fingerprint: Optional[Text] = None
+        if not self.is_dense() and not self.is_sparse():
+            raise ValueError(
+                "Features must either be a numpy array for dense "
+                "features or a scipy sparse matrix for sparse features."
+            )
+
+    def __repr__(self) -> Text:
+        return (
+            f"{self.__class__.__name__}("
+            f"features={self.features}, "
+            f"type={self.type}, "
+            f"origin={self.origin}, "
+            f"attribute={self.attribute})"
+        )
+
+    def __str__(self) -> Text:
+        return (
+            f"{self.__class__.__name__}("
+            f"features.shape={self.features.shape}, "
+            f"is_sparse={self.is_sparse()}, "
+            f"type={self.type}, "
+            f"origin={self.origin}, "
+            f"attribute={self.attribute})"
+        )
+
+    def is_sparse(self) -> bool:
+        """Checks if features are sparse or not.
+
+        Returns:
+            True, if features are sparse, false otherwise.
+        """
+        return isinstance(self.features, scipy.sparse.spmatrix)
+
+    def is_dense(self) -> bool:
+        """Checks if features are dense or not.
+
+        Returns:
+            True, if features are dense, false otherwise.
+        """
+        return not self.is_sparse()
+
+    def combine_with_features(self, additional_features: Optional[Features]) -> None:
+        """Combine the incoming features with this instance's features.
+
+        Args:
+            additional_features: additional features to add
+
+        Returns:
+            Combined features.
+        """
+        if additional_features is None:
+            return
+
+        if self.is_dense() and additional_features.is_dense():
+            self._combine_dense_features(additional_features)
+        elif self.is_sparse() and additional_features.is_sparse():
+            self._combine_sparse_features(additional_features)
+        else:
+            raise ValueError("Cannot combine sparse and dense features.")
+
+    def _combine_dense_features(self, additional_features: Features) -> None:
+        if self.features.ndim != additional_features.features.ndim:
+            raise ValueError(
+                f"Cannot combine dense features as sequence dimensions do not "
+                f"match: {self.features.ndim} != {additional_features.features.ndim}."
+            )
+        self.features = np.concatenate(
+            (self.features, additional_features.features), axis=-1
+        )
+        self._cached_fingerprint = None
+
+    def _combine_sparse_features(self, additional_features: Features) -> None:
+        from scipy.sparse import hstack
+
+        if self.features.shape[0] != additional_features.features.shape[0]:
+            raise ValueError(
+                f"Cannot combine sparse features as sequence dimensions do not "
+                f"match: {self.features.shape[0]} != "
+                f"{additional_features.features.shape[0]}."
+            )
+
+        self.features = hstack([self.features, additional_features.features])
+        self._cached_fingerprint = None
+
+    def __key__(
+        self,
+    ) -> Tuple[
+        Text, Text, Union[np.ndarray, scipy.sparse.spmatrix], Union[Text, List[Text]]
+    ]:
+        """Returns a 4-tuple of defining properties.
+
+        Returns:
+            Tuple of type, attribute, features, and origin properties.
+        """
+        return self.type, self.attribute, self.features, self.origin
+
+    def __eq__(self, other: Any) -> bool:
+        """Tests if the `self` `Feature` equals to the `other`.
+
+        Args:
+            other: The other object.
+
+        Returns:
+            `True` when the other object is a `Feature` and has the same
+            type, attribute, and feature tensors.
+        """
+        if not isinstance(other, Features):
+            return False
+
+        return (
+            other.type == self.type
+            and other.attribute == self.attribute
+            and other.features == self.features
+        )
+
+    def fingerprint(self) -> Text:
+        """Calculate a stable string fingerprint for the features."""
+        if self._cached_fingerprint is None:
+            if self.is_dense():
+                f_as_text = self.features.tobytes()
+            else:
+                f_as_text = rasa.shared.nlu.training_data.util.sparse_matrix_to_string(
+                    self.features
+                )
+            self._cached_fingerprint = rasa.shared.utils.io.deep_container_fingerprint(
+                [self.type, self.origin, self.attribute, f_as_text]
+            )
+        return self._cached_fingerprint
+
+    @staticmethod
+    def filter(
+        features_list: List[Features],
+        attributes: Optional[Iterable[Text]] = None,
+        type: Optional[Text] = None,
+        origin: Optional[List[Text]] = None,
+        is_sparse: Optional[bool] = None,
+    ) -> List[Features]:
+        """Filters the given list of features.
+
+        Args:
+          features_list: list of features to be filtered
+          attributes: List of attributes that we're interested in. Set this to `None`
+            to disable this filter.
+          type: The type of feature we're interested in. Set this to `None`
+            to disable this filter.
+          origin: If specified, this method will check that the exact order of origins
+            matches the given list of origins. The reason for this is that if
+            multiple origins are listed for a Feature, this means that this feature
+            has been created by concatenating Features from the listed origins in
+            that particular order.
+          is_sparse: Defines whether all features that we're interested in should be
+            sparse. Set this to `None` to disable this filter.
+
+        Returns:
+            sub-list of features with the desired properties
+        """
+        filtered = features_list
+        if attributes is not None:
+            attributes = set(attributes)
+            filtered = [f for f in filtered if f.attribute in attributes]
+        if origin is not None:
+            filtered = [
+                f
+                for f in filtered
+                if (f.origin if not isinstance(f.origin, Text) else list([f.origin]))
+                == origin
+            ]
+        if type is not None:
+            filtered = [f for f in filtered if f.type == type]
+        if is_sparse is not None:
+            filtered = [f for f in filtered if f.is_sparse() == is_sparse]
+        return filtered
+
+    @staticmethod
+    def groupby_attribute(
+        features_list: List[Features], attributes: Optional[Iterable[Text]] = None
+    ) -> Dict[Text, List[Features]]:
+        """Groups the given features according to their attribute.
+
+        Args:
+          features_list: list of features to be grouped
+          attributes: If specified, the result will be a grouping with respect to
+            the given attributes. If some specified attribute has no features attached
+            to it, then the resulting dictionary will map it to an empty list.
+            If this is None, the result will be a grouping according to all attributes
+            for which features can be found.
+
+        Returns:
+           a mapping from the requested attributes to the list of correspoding
+           features
+        """
+        # ensure all requested attributes are present in the output - regardless
+        # of whether we find features later
+        extracted: Dict[Text, List[Features]] = (
+            dict()
+            if attributes is None
+            else {attribute: [] for attribute in attributes}
+        )
+        # extract features for all (requested) attributes
+        for feat in features_list:
+            if attributes is None or feat.attribute in attributes:
+                extracted.setdefault(feat.attribute, []).append(feat)
+        return extracted
+
+    @staticmethod
+    def combine(
+        features_list: List[Features], expected_origins: Optional[List[Text]] = None
+    ) -> Features:
+        """Combine features of the same type and level that describe the same attribute.
+
+        If sequence features are to be combined, then they must have the same
+        sequence dimension.
+
+        Args:
+          features: Non-empty list of Features  of the same type and level that
+            describe the same attribute.
+          expected_origins: The expected origins of the given features. This method
+            will check that the origin information of each feature is as expected, i.e.
+            the origin of the i-th feature in the given list is the i-th origin
+            in this list of origins.
+
+        Raises:
+          `ValueError` will be raised
+           - if the given list is empty
+           - if there are inconsistencies in the given list of `Features`
+           - if the origins aren't as expected
+        """
+        if len(features_list) == 0:
+            raise ValueError("Expected a non-empty list of Features.")
+        if len(features_list) == 1:
+            # nothing to combine here
+            return features_list[0]
+
+        # Un-Pack the Origin information
+        origin_of_combination = [f.origin for f in features_list]
+        origin_of_combination = [
+            featurizer_name
+            for origin in origin_of_combination
+            for featurizer_name in (origin if isinstance(origin, List) else [origin])
+        ]
+
+        # Sanity Checks
+        # (1) origins must be as expected
+        if expected_origins is not None:
+            if origin_of_combination is not None:
+                for idx, (expected, actual) in enumerate(
+                    itertools.zip_longest(expected_origins, origin_of_combination)
+                ):
+                    if expected != actual:
+                        raise ValueError(
+                            f"Expected '{expected}' to be the origin of the {idx}-th "
+                            f"feature (because of `origin_of_combination`) but found a "
+                            f"feature from '{actual}'."
+                        )
+        # (2) attributes (is_sparse, type, attribute) must coincide
+        # Note: we could also use `filter` for this check, but then the error msgs
+        # aren't as nice.
+        sparseness: Set[bool] = set(f.is_sparse() for f in features_list)
+        if len(sparseness) > 1:
+            raise ValueError(
+                "Expected all Features to have the same sparseness property but "
+                "found both (sparse and dense)."
+            )
+        types: Set[Text] = set(f.type for f in features_list)
+        if len(types) > 1:
+            raise ValueError(
+                f"Expected all Features to have the same type but found the "
+                f"following types {types}."
+            )
+        attributes: Set[Text] = set(f.attribute for f in features_list)
+        if len(attributes) > 1:
+            raise ValueError(
+                f"Expected all Features to describe the same attribute but found "
+                f"attributes: {attributes}."
+            )
+        # (3) dimensions must match
+        # Note: We shouldn't have to check sentence-level features here but it doesn't
+        # hurt either.
+        dimensions = set(f.features.shape[0] for f in features_list)
+        if len(dimensions) > 1:
+            raise ValueError(
+                f"Expected all sequence dimensions to match but found {dimensions}."
+            )
+
+        # Combine the features
+        arbitrary_feature = features_list[0]
+        if not arbitrary_feature.is_sparse():
+            features = np.concatenate([f.features for f in features_list], axis=-1)
+        else:
+            features = scipy.sparse.hstack([f.features for f in features_list])
+        return Features(
+            features=features,
+            feature_type=arbitrary_feature.type,
+            attribute=arbitrary_feature.attribute,
+            origin=origin_of_combination,
+        )
+
+    @staticmethod
+    def reduce(
+        features_list: List[Features], expected_origins: Optional[List[Text]] = None
+    ) -> List[Features]:
+        """Combines features of same type and level into one Feature.
+
+        Args:
+           features_list: list of Features which must all describe the same attribute
+           expected_origins: if specified, this list will be used to validate that
+             the features from the right featurizers are combined in the right order
+             (cf. `Features.combine`)
+
+        Returns:
+            a list of the combined Features, i.e. at most 4 Features, where
+            - all the sparse features are listed before the dense features
+            - sequence feature is always listed before the sentence feature with the
+              same sparseness property
+        """
+        if len(features_list) == 1:
+            return features_list
+        # sanity check
+        different_settings = set(f.attribute for f in features_list)
+        if len(different_settings) > 1:
+            raise ValueError(
+                f"Expected all Features to describe the same attribute but found "
+                f" {different_settings}."
+            )
+        output = []
+        for is_sparse in [True, False]:
+            # all sparse features before all dense features
+            for type in [FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE]:
+                # sequence feature that is (not) sparse before sentence feature that is
+                # (not) sparse
+                sublist = Features.filter(
+                    features_list=features_list, type=type, is_sparse=is_sparse
+                )
+                if sublist:
+                    combined_feature = Features.combine(
+                        sublist, expected_origins=expected_origins
+                    )
+                    output.append(combined_feature)
+        return output

rasa/shared/nlu/training_data/formats/__init__.py

@@ -0,0 +1,10 @@
+from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLReader  # noqa: F401
+from rasa.shared.nlu.training_data.formats.dialogflow import (  # noqa: F401
+    DialogflowReader,
+)
+from rasa.shared.nlu.training_data.formats.luis import LuisReader  # noqa: F401
+from rasa.shared.nlu.training_data.formats.rasa import (  # noqa: F401
+    RasaReader,
+    RasaWriter,
+)
+from rasa.shared.nlu.training_data.formats.wit import WitReader  # noqa: F401

rasa/shared/nlu/training_data/formats/dialogflow.py

@@ -0,0 +1,162 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Text, List, Tuple, Union
+
+import rasa.shared.nlu.training_data.util
+from rasa.shared.constants import DOCS_BASE_URL
+from rasa.shared.nlu.training_data.formats.readerwriter import TrainingDataReader
+from rasa.shared.nlu.training_data.util import transform_entity_synonyms
+import rasa.shared.utils.io
+
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+
+logger = logging.getLogger(__name__)
+
+DOCS_URL_MIGRATE_GOOGLE = DOCS_BASE_URL + "/migrate-from/google-dialogflow-to-rasa/"
+
+DIALOGFLOW_PACKAGE = "dialogflow_package"
+DIALOGFLOW_AGENT = "dialogflow_agent"
+DIALOGFLOW_INTENT = "dialogflow_intent"
+DIALOGFLOW_INTENT_EXAMPLES = "dialogflow_intent_examples"
+DIALOGFLOW_ENTITIES = "dialogflow_entities"
+DIALOGFLOW_ENTITY_ENTRIES = "dialogflow_entity_entries"
+
+
+class DialogflowReader(TrainingDataReader):
+    """Reader for NLU training data."""
+
+    def read(self, filename: Union[Text, Path], **kwargs: Any) -> "TrainingData":
+        """Loads training data stored in the Dialogflow data format."""
+        language = kwargs["language"]
+        fformat = kwargs["fformat"]
+
+        if fformat not in {DIALOGFLOW_INTENT, DIALOGFLOW_ENTITIES}:
+            raise ValueError(
+                "fformat must be either {}, or {}"
+                "".format(DIALOGFLOW_INTENT, DIALOGFLOW_ENTITIES)
+            )
+
+        root_js = rasa.shared.utils.io.read_json_file(filename)
+
+        if isinstance(filename, Path):
+            filename = str(filename)
+
+        examples = self._read_examples(filename, language, fformat)
+
+        if not examples:
+            rasa.shared.utils.io.raise_warning(
+                f"No training examples found for dialogflow file {filename}!",
+                docs=DOCS_URL_MIGRATE_GOOGLE,
+            )
+            return TrainingData()
+        elif fformat == DIALOGFLOW_INTENT:
+            return self._read_intent(root_js, examples)
+        else:  # path for DIALOGFLOW_ENTITIES
+            return self._read_entities(root_js, examples)
+
+    def _read_intent(
+        self, intent: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> "TrainingData":
+        """Reads the intent and examples from respective jsons."""
+        intent_name = intent.get("name")
+
+        training_examples = []
+        for ex in examples:
+            text, entities = self._join_text_chunks(ex["data"])
+            training_examples.append(Message.build(text, intent_name, entities))
+
+        return TrainingData(training_examples)
+
+    def _join_text_chunks(
+        self, chunks: List[Dict[Text, Any]]
+    ) -> Tuple[Text, List[Dict[Text, Any]]]:
+        """Combines text chunks and extracts entities."""
+        utterance = ""
+        entities = []
+        for chunk in chunks:
+            entity = self._extract_entity(chunk, len(utterance))
+            if entity:
+                entities.append(entity)
+            utterance += chunk["text"]
+
+        return utterance, entities
+
+    @staticmethod
+    def _extract_entity(
+        chunk: Dict[Text, Any], current_offset: int
+    ) -> Optional[Dict[Text, Any]]:
+        """Extract an entity from a chunk if present."""
+        entity = None
+        if "meta" in chunk or "alias" in chunk:
+            start = current_offset
+            text = chunk["text"]
+            end = start + len(text)
+            entity_type = chunk.get("alias", chunk["meta"])
+            if entity_type != "@sys.ignore":
+                entity = rasa.shared.nlu.training_data.util.build_entity(
+                    start, end, text, entity_type
+                )
+
+        return entity
+
+    @staticmethod
+    def _flatten(list_of_lists: List[List[Any]]) -> List[Any]:
+        return [item for items in list_of_lists for item in items]
+
+    @staticmethod
+    def _extract_lookup_tables(
+        entity: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> Optional[List[Dict[Text, Any]]]:
+        """Extracts the lookup table from the entity synonyms."""
+        synonyms = [e["synonyms"] for e in examples if "synonyms" in e]
+        synonyms = DialogflowReader._flatten(synonyms)
+        elements = [synonym for synonym in synonyms if "@" not in synonym]
+
+        if len(elements) == 0:
+            return None
+        return [{"name": entity.get("name"), "elements": elements}]
+
+    @staticmethod
+    def _extract_regex_features(
+        entity: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> List[Dict[Text, Any]]:
+        """Extract the regex features from the entity synonyms."""
+        synonyms = [e["synonyms"] for e in examples if "synonyms" in e]
+        synonyms = DialogflowReader._flatten(synonyms)
+        return [
+            {"name": entity.get("name"), "pattern": synonym} for synonym in synonyms
+        ]
+
+    @staticmethod
+    def _read_entities(
+        entity: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> "TrainingData":
+        entity_synonyms = transform_entity_synonyms(examples)
+
+        if entity["isRegexp"]:
+            regex_features = DialogflowReader._extract_regex_features(entity, examples)
+            return TrainingData([], entity_synonyms, regex_features, [])
+        else:
+            lookup_tables = DialogflowReader._extract_lookup_tables(entity, examples)
+            return TrainingData([], entity_synonyms, [], lookup_tables)
+
+    @staticmethod
+    def _read_examples(
+        fn: Text, language: Text, fformat: Text
+    ) -> Optional[List[Dict[Text, Any]]]:
+        """Infer and load example file based on root filename and root format."""
+        if fformat == DIALOGFLOW_INTENT:
+            examples_type = "usersays"
+        else:
+            examples_type = "entries"
+        examples_fn_ending = f"_{examples_type}_{language}.json"
+        examples_fn = fn.replace(".json", examples_fn_ending)
+        if os.path.isfile(examples_fn):
+            return rasa.shared.utils.io.read_json_file(examples_fn)
+        else:
+            return None
+
+    def reads(self, s: Text, **kwargs: Any) -> "TrainingData":
+        raise NotImplementedError

rasa/shared/nlu/training_data/formats/luis.py

@@ -0,0 +1,87 @@
+import logging
+from typing import Any, Dict, List, Text
+
+from rasa.shared.nlu.constants import (
+    TEXT,
+    INTENT,
+    ENTITIES,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_ROLE,
+    ENTITY_ATTRIBUTE_VALUE,
+    ENTITY_ATTRIBUTE_START,
+    ENTITY_ATTRIBUTE_END,
+    NO_ENTITY_TAG,
+)
+from rasa.shared.nlu.training_data.formats.readerwriter import JsonTrainingDataReader
+import rasa.shared.utils.io
+
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+
+logger = logging.getLogger(__name__)
+
+
+class LuisReader(JsonTrainingDataReader):
+    """Reads LUIS training data."""
+
+    @staticmethod
+    def _extract_regex_features(js: Dict[Text, Any]) -> List[Dict[Text, Any]]:
+        regex_features: List[Dict[Text, Any]] = []
+
+        for r in js.get("regex_features", []):
+            if r.get("activated", False):
+                regex_features.append(
+                    {"name": r.get("name"), "pattern": r.get("pattern")}
+                )
+
+        # LUIS removed `regex_features` and exports regular expressions in `
+        # regex_entities` now:
+        # https://stackoverflow.com/questions/48170631/what-happened-to-the-regex-features # noqa: E501
+        for r in js.get("regex_entities", []):
+            regex_features.append(
+                {"name": r.get("name"), "pattern": r.get("regexPattern")}
+            )
+
+        return regex_features
+
+    def read_from_json(self, js: Dict[Text, Any], **kwargs: Any) -> "TrainingData":
+        """Loads training data stored in the LUIS.ai data format."""
+        training_examples = []
+
+        max_tested_luis_schema_version = 7
+        major_version = int(js["luis_schema_version"].split(".")[0])
+        if major_version > max_tested_luis_schema_version:
+            rasa.shared.utils.io.raise_warning(
+                f"Your luis data schema version {js['luis_schema_version']} "
+                f"is higher than 7.x.x. "
+                f"Training may not be performed correctly. "
+            )
+
+        for s in js["utterances"]:
+            text = s.get("text")
+            intent = s.get("intent")
+            entities = []
+            for e in s.get("entities") or []:
+                start, end = e["startPos"], e["endPos"] + 1
+                val = text[start:end]
+
+                entities.append(
+                    {
+                        ENTITY_ATTRIBUTE_TYPE: e.get("entity"),
+                        ENTITY_ATTRIBUTE_VALUE: val,
+                        ENTITY_ATTRIBUTE_START: start,
+                        ENTITY_ATTRIBUTE_END: end,
+                        ENTITY_ATTRIBUTE_ROLE: e.get("role", NO_ENTITY_TAG),
+                    }
+                )
+
+            data = {ENTITIES: entities}
+            if intent:
+                data[INTENT] = intent
+            data[TEXT] = text
+
+            training_examples.append(Message(data=data))
+
+        return TrainingData(
+            training_examples, regex_features=self._extract_regex_features(js)
+        )