rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/shared/nlu/training_data/formats/__init__.py

@@ -0,0 +1,10 @@
+from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLReader  # noqa: F401
+from rasa.shared.nlu.training_data.formats.dialogflow import (  # noqa: F401
+    DialogflowReader,
+)
+from rasa.shared.nlu.training_data.formats.luis import LuisReader  # noqa: F401
+from rasa.shared.nlu.training_data.formats.rasa import (  # noqa: F401
+    RasaReader,
+    RasaWriter,
+)
+from rasa.shared.nlu.training_data.formats.wit import WitReader  # noqa: F401

rasa/shared/nlu/training_data/formats/dialogflow.py

@@ -0,0 +1,163 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Text, List, Tuple, Union
+
+import rasa.shared.nlu.training_data.util
+from rasa.shared.constants import DOCS_BASE_URL
+from rasa.shared.nlu.training_data.formats.readerwriter import TrainingDataReader
+from rasa.shared.nlu.training_data.util import transform_entity_synonyms
+import rasa.shared.utils.io
+
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+
+logger = logging.getLogger(__name__)
+
+DOCS_URL_MIGRATE_GOOGLE = DOCS_BASE_URL + "/migrate-from/google-dialogflow-to-rasa/"
+
+DIALOGFLOW_PACKAGE = "dialogflow_package"
+DIALOGFLOW_AGENT = "dialogflow_agent"
+DIALOGFLOW_INTENT = "dialogflow_intent"
+DIALOGFLOW_INTENT_EXAMPLES = "dialogflow_intent_examples"
+DIALOGFLOW_ENTITIES = "dialogflow_entities"
+DIALOGFLOW_ENTITY_ENTRIES = "dialogflow_entity_entries"
+
+
+class DialogflowReader(TrainingDataReader):
+    """Reader for NLU training data."""
+
+    def read(self, filename: Union[Text, Path], **kwargs: Any) -> "TrainingData":
+        """Loads training data stored in the Dialogflow data format."""
+        language = kwargs["language"]
+        fformat = kwargs["fformat"]
+
+        if fformat not in {DIALOGFLOW_INTENT, DIALOGFLOW_ENTITIES}:
+            raise ValueError(
+                "fformat must be either {}, or {}".format(
+                    DIALOGFLOW_INTENT, DIALOGFLOW_ENTITIES
+                )
+            )
+
+        root_js = rasa.shared.utils.io.read_json_file(filename)
+
+        if isinstance(filename, Path):
+            filename = str(filename)
+
+        examples = self._read_examples(filename, language, fformat)
+
+        if not examples:
+            rasa.shared.utils.io.raise_warning(
+                f"No training examples found for dialogflow file {filename}!",
+                docs=DOCS_URL_MIGRATE_GOOGLE,
+            )
+            return TrainingData()
+        elif fformat == DIALOGFLOW_INTENT:
+            return self._read_intent(root_js, examples)
+        else:  # path for DIALOGFLOW_ENTITIES
+            return self._read_entities(root_js, examples)
+
+    def _read_intent(
+        self, intent: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> "TrainingData":
+        """Reads the intent and examples from respective jsons."""
+        intent_name = intent.get("name")
+
+        training_examples = []
+        for ex in examples:
+            text, entities = self._join_text_chunks(ex["data"])
+            training_examples.append(Message.build(text, intent_name, entities))
+
+        return TrainingData(training_examples)
+
+    def _join_text_chunks(
+        self, chunks: List[Dict[Text, Any]]
+    ) -> Tuple[Text, List[Dict[Text, Any]]]:
+        """Combines text chunks and extracts entities."""
+        utterance = ""
+        entities = []
+        for chunk in chunks:
+            entity = self._extract_entity(chunk, len(utterance))
+            if entity:
+                entities.append(entity)
+            utterance += chunk["text"]
+
+        return utterance, entities
+
+    @staticmethod
+    def _extract_entity(
+        chunk: Dict[Text, Any], current_offset: int
+    ) -> Optional[Dict[Text, Any]]:
+        """Extract an entity from a chunk if present."""
+        entity = None
+        if "meta" in chunk or "alias" in chunk:
+            start = current_offset
+            text = chunk["text"]
+            end = start + len(text)
+            entity_type = chunk.get("alias", chunk["meta"])
+            if entity_type != "@sys.ignore":
+                entity = rasa.shared.nlu.training_data.util.build_entity(
+                    start, end, text, entity_type
+                )
+
+        return entity
+
+    @staticmethod
+    def _flatten(list_of_lists: List[List[Any]]) -> List[Any]:
+        return [item for items in list_of_lists for item in items]
+
+    @staticmethod
+    def _extract_lookup_tables(
+        entity: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> Optional[List[Dict[Text, Any]]]:
+        """Extracts the lookup table from the entity synonyms."""
+        synonyms = [e["synonyms"] for e in examples if "synonyms" in e]
+        synonyms = DialogflowReader._flatten(synonyms)
+        elements = [synonym for synonym in synonyms if "@" not in synonym]
+
+        if len(elements) == 0:
+            return None
+        return [{"name": entity.get("name"), "elements": elements}]
+
+    @staticmethod
+    def _extract_regex_features(
+        entity: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> List[Dict[Text, Any]]:
+        """Extract the regex features from the entity synonyms."""
+        synonyms = [e["synonyms"] for e in examples if "synonyms" in e]
+        synonyms = DialogflowReader._flatten(synonyms)
+        return [
+            {"name": entity.get("name"), "pattern": synonym} for synonym in synonyms
+        ]
+
+    @staticmethod
+    def _read_entities(
+        entity: Dict[Text, Any], examples: List[Dict[Text, Any]]
+    ) -> "TrainingData":
+        entity_synonyms = transform_entity_synonyms(examples)
+
+        if entity["isRegexp"]:
+            regex_features = DialogflowReader._extract_regex_features(entity, examples)
+            return TrainingData([], entity_synonyms, regex_features, [])
+        else:
+            lookup_tables = DialogflowReader._extract_lookup_tables(entity, examples)
+            return TrainingData([], entity_synonyms, [], lookup_tables)
+
+    @staticmethod
+    def _read_examples(
+        fn: Text, language: Text, fformat: Text
+    ) -> Optional[List[Dict[Text, Any]]]:
+        """Infer and load example file based on root filename and root format."""
+        if fformat == DIALOGFLOW_INTENT:
+            examples_type = "usersays"
+        else:
+            examples_type = "entries"
+        examples_fn_ending = f"_{examples_type}_{language}.json"
+        examples_fn = fn.replace(".json", examples_fn_ending)
+        if os.path.isfile(examples_fn):
+            return rasa.shared.utils.io.read_json_file(examples_fn)
+        else:
+            return None
+
+    def reads(self, s: Text, **kwargs: Any) -> "TrainingData":
+        raise NotImplementedError

rasa/shared/nlu/training_data/formats/luis.py

@@ -0,0 +1,87 @@
+import logging
+from typing import Any, Dict, List, Text
+
+from rasa.shared.nlu.constants import (
+    TEXT,
+    INTENT,
+    ENTITIES,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_ROLE,
+    ENTITY_ATTRIBUTE_VALUE,
+    ENTITY_ATTRIBUTE_START,
+    ENTITY_ATTRIBUTE_END,
+    NO_ENTITY_TAG,
+)
+from rasa.shared.nlu.training_data.formats.readerwriter import JsonTrainingDataReader
+import rasa.shared.utils.io
+
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+
+logger = logging.getLogger(__name__)
+
+
+class LuisReader(JsonTrainingDataReader):
+    """Reads LUIS training data."""
+
+    @staticmethod
+    def _extract_regex_features(js: Dict[Text, Any]) -> List[Dict[Text, Any]]:
+        regex_features: List[Dict[Text, Any]] = []
+
+        for r in js.get("regex_features", []):
+            if r.get("activated", False):
+                regex_features.append(
+                    {"name": r.get("name"), "pattern": r.get("pattern")}
+                )
+
+        # LUIS removed `regex_features` and exports regular expressions in `
+        # regex_entities` now:
+        # https://stackoverflow.com/questions/48170631/what-happened-to-the-regex-features # noqa: E501
+        for r in js.get("regex_entities", []):
+            regex_features.append(
+                {"name": r.get("name"), "pattern": r.get("regexPattern")}
+            )
+
+        return regex_features
+
+    def read_from_json(self, js: Dict[Text, Any], **kwargs: Any) -> "TrainingData":
+        """Loads training data stored in the LUIS.ai data format."""
+        training_examples = []
+
+        max_tested_luis_schema_version = 7
+        major_version = int(js["luis_schema_version"].split(".")[0])
+        if major_version > max_tested_luis_schema_version:
+            rasa.shared.utils.io.raise_warning(
+                f"Your luis data schema version {js['luis_schema_version']} "
+                f"is higher than 7.x.x. "
+                f"Training may not be performed correctly. "
+            )
+
+        for s in js["utterances"]:
+            text = s.get("text")
+            intent = s.get("intent")
+            entities = []
+            for e in s.get("entities") or []:
+                start, end = e["startPos"], e["endPos"] + 1
+                val = text[start:end]
+
+                entities.append(
+                    {
+                        ENTITY_ATTRIBUTE_TYPE: e.get("entity"),
+                        ENTITY_ATTRIBUTE_VALUE: val,
+                        ENTITY_ATTRIBUTE_START: start,
+                        ENTITY_ATTRIBUTE_END: end,
+                        ENTITY_ATTRIBUTE_ROLE: e.get("role", NO_ENTITY_TAG),
+                    }
+                )
+
+            data = {ENTITIES: entities}
+            if intent:
+                data[INTENT] = intent
+            data[TEXT] = text
+
+            training_examples.append(Message(data=data))
+
+        return TrainingData(
+            training_examples, regex_features=self._extract_regex_features(js)
+        )

rasa/shared/nlu/training_data/formats/rasa.py

@@ -0,0 +1,135 @@
+import logging
+from collections import defaultdict
+from typing import Any, Dict, Text
+
+from rasa.shared.constants import DOCS_URL_MIGRATION_GUIDE
+from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES
+from rasa.shared.nlu.training_data.formats.readerwriter import (
+    JsonTrainingDataReader,
+    TrainingDataWriter,
+)
+from rasa.shared.nlu.training_data.util import transform_entity_synonyms
+from rasa.shared.utils.io import json_to_string
+
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+import rasa.shared.utils.io
+from rasa.shared.utils.yaml import validate_training_data
+
+logger = logging.getLogger(__name__)
+
+
+class RasaReader(JsonTrainingDataReader):
+    """Reader for Rasa NLU training data in JSON format.
+
+    Example:
+        {
+          "rasa_nlu_data": {
+            "regex_features": [
+              {
+                "name": "zipcode",
+                "pattern": "[0-9]{5}"
+              }
+            ],
+            "entity_synonyms": [
+              {
+                "value": "chinese",
+                "synonyms": ["Chinese", "Chines", "chines"]
+              }
+            ],
+            "common_examples": [
+              {
+                "text": "hey",
+                "intent": "greet",
+                "entities": []
+              },
+              {
+                "text": "howdy",
+                "intent": "greet",
+                "entities": []
+              }
+            ]
+          }
+        }
+    """
+
+    def __init__(self) -> None:
+        """Creates reader."""
+        super().__init__()
+        rasa.shared.utils.io.raise_deprecation_warning(
+            "NLU data in Rasa JSON format is deprecated and will be removed in Rasa "
+            "Open Source 4.0.0. Please convert your JSON NLU data to the "
+            "Rasa YAML format.",
+            docs=DOCS_URL_MIGRATION_GUIDE,
+        )
+
+    def read_from_json(self, js: Dict[Text, Any], **_: Any) -> "TrainingData":
+        """Loads training data stored in the rasa NLU data format."""
+        import rasa.shared.nlu.training_data.schemas.data_schema as schema
+
+        validate_training_data(js, schema.rasa_nlu_data_schema())
+
+        data = js["rasa_nlu_data"]
+        common_examples = data.get("common_examples", [])
+        entity_synonyms = data.get("entity_synonyms", [])
+        regex_features = data.get("regex_features", [])
+        lookup_tables = data.get("lookup_tables", [])
+
+        entity_synonyms = transform_entity_synonyms(entity_synonyms)
+
+        training_examples = []
+        for ex in common_examples:
+            # taking care of custom entries
+            msg = Message.build(
+                text=ex.pop(TEXT, ""),
+                intent=ex.pop(INTENT, None),
+                entities=ex.pop(ENTITIES, None),
+                **ex,
+            )
+            training_examples.append(msg)
+
+        return TrainingData(
+            training_examples, entity_synonyms, regex_features, lookup_tables
+        )
+
+
+class RasaWriter(TrainingDataWriter):
+    """Dumps NLU data as Rasa JSON string."""
+
+    def __init__(self) -> None:
+        """Creates writer."""
+        super().__init__()
+        rasa.shared.utils.io.raise_deprecation_warning(
+            "NLU data in Rasa JSON format is deprecated and will be removed in Rasa "
+            "Open Source 4.0.0. Please convert your JSON NLU data to the "
+            "Rasa YAML format.",
+            docs=DOCS_URL_MIGRATION_GUIDE,
+        )
+
+    def dumps(self, training_data: "TrainingData", **kwargs: Any) -> Text:
+        """Writes Training Data to a string in json format."""
+        js_entity_synonyms = defaultdict(list)
+        for k, v in training_data.entity_synonyms.items():
+            if k != v:
+                js_entity_synonyms[v].append(k)
+
+        formatted_synonyms = [
+            {"value": value, "synonyms": syns}
+            for value, syns in js_entity_synonyms.items()
+        ]
+
+        formatted_examples = [
+            example.as_dict_nlu() for example in training_data.training_examples
+        ]
+
+        return json_to_string(
+            {
+                "rasa_nlu_data": {
+                    "common_examples": formatted_examples,
+                    "regex_features": training_data.regex_features,
+                    "lookup_tables": training_data.lookup_tables,
+                    "entity_synonyms": formatted_synonyms,
+                }
+            },
+            **kwargs,
+        )