rasa-pro 3.8.16__py3-none-any.whl

rasa/nlu/classifiers/sklearn_intent_classifier.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+import logging
+from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
+import typing
+import warnings
+from typing import Any, Dict, List, Optional, Text, Tuple, Type
+
+import numpy as np
+
+import rasa.shared.utils.io
+import rasa.utils.io as io_utils
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU
+from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
+from rasa.shared.exceptions import RasaException
+from rasa.shared.nlu.constants import TEXT
+from rasa.nlu.classifiers.classifier import IntentClassifier
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+from rasa.utils.tensorflow.constants import FEATURIZERS
+
+logger = logging.getLogger(__name__)
+
+if typing.TYPE_CHECKING:
+    import sklearn
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER, is_trainable=True
+)
+class SklearnIntentClassifier(GraphComponent, IntentClassifier):
+    """Intent classifier using the sklearn framework."""
+
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [DenseFeaturizer]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """The component's default config (see parent class for full docstring)."""
+        return {
+            # C parameter of the svm - cross validation will select the best value
+            "C": [1, 2, 5, 10, 20, 100],
+            # gamma parameter of the svm
+            "gamma": [0.1],
+            # the kernels to use for the svm training - cross validation will
+            # decide which one of them performs best
+            "kernels": ["linear"],
+            # We try to find a good number of cross folds to use during
+            # intent training, this specifies the max number of folds
+            "max_cross_validation_folds": 5,
+            # Scoring function used for evaluating the hyper parameters
+            # This can be a name or a function (cfr GridSearchCV doc for more info)
+            "scoring_function": "f1_weighted",
+            "num_threads": 1,
+        }
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        clf: Optional["sklearn.model_selection.GridSearchCV"] = None,
+        le: Optional["sklearn.preprocessing.LabelEncoder"] = None,
+    ) -> None:
+        """Construct a new intent classifier using the sklearn framework."""
+        from sklearn.preprocessing import LabelEncoder
+
+        self.component_config = config
+        self._model_storage = model_storage
+        self._resource = resource
+
+        if le is not None:
+            self.le = le
+        else:
+            self.le = LabelEncoder()
+        self.clf = clf
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> SklearnIntentClassifier:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource)
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["sklearn"]
+
+    def transform_labels_str2num(self, labels: List[Text]) -> np.ndarray:
+        """Transforms a list of strings into numeric label representation.
+
+        :param labels: List of labels to convert to numeric representation
+        """
+        return self.le.fit_transform(labels)
+
+    def transform_labels_num2str(self, y: np.ndarray) -> np.ndarray:
+        """Transforms a list of strings into numeric label representation.
+
+        :param y: List of labels to convert to numeric representation
+        """
+        return self.le.inverse_transform(y)
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Train the intent classifier on a data set."""
+        num_threads = self.component_config["num_threads"]
+
+        labels = [e.get("intent") for e in training_data.intent_examples]
+
+        if len(set(labels)) < 2:
+            rasa.shared.utils.io.raise_warning(
+                "Can not train an intent classifier as there are not "
+                "enough intents. Need at least 2 different intents. "
+                "Skipping training of intent classifier.",
+                docs=DOCS_URL_TRAINING_DATA_NLU,
+            )
+            return self._resource
+
+        y = self.transform_labels_str2num(labels)
+        training_examples = [
+            message
+            for message in training_data.intent_examples
+            if message.features_present(
+                attribute=TEXT, featurizers=self.component_config.get(FEATURIZERS)
+            )
+        ]
+        X = np.stack(
+            [self._get_sentence_features(example) for example in training_examples]
+        )
+        # reduce dimensionality
+        X = np.reshape(X, (len(X), -1))
+
+        self.clf = self._create_classifier(num_threads, y)
+
+        with warnings.catch_warnings():
+            # sklearn raises lots of
+            # "UndefinedMetricWarning: F - score is ill - defined"
+            # if there are few intent examples, this is needed to prevent it
+            warnings.simplefilter("ignore")
+            self.clf.fit(X, y)
+
+        self.persist()
+        return self._resource
+
+    @staticmethod
+    def _get_sentence_features(message: Message) -> np.ndarray:
+        _, sentence_features = message.get_dense_features(TEXT)
+        if sentence_features is not None:
+            return sentence_features.features[0]
+
+        raise ValueError(
+            "No sentence features present. Not able to train sklearn policy."
+        )
+
+    def _num_cv_splits(self, y: np.ndarray) -> int:
+        folds = self.component_config["max_cross_validation_folds"]
+        return max(2, min(folds, np.min(np.bincount(y)) // 5))
+
+    def _create_classifier(
+        self, num_threads: int, y: np.ndarray
+    ) -> "sklearn.model_selection.GridSearchCV":
+        from sklearn.model_selection import GridSearchCV
+        from sklearn.svm import SVC
+
+        C = self.component_config["C"]
+        kernels = self.component_config["kernels"]
+        gamma = self.component_config["gamma"]
+        # dirty str fix because sklearn is expecting
+        # str not instance of basestr...
+        tuned_parameters = [
+            {"C": C, "gamma": gamma, "kernel": [str(k) for k in kernels]}
+        ]
+
+        # aim for 5 examples in each fold
+
+        cv_splits = self._num_cv_splits(y)
+
+        return GridSearchCV(
+            SVC(C=1, probability=True, class_weight="balanced"),
+            param_grid=tuned_parameters,
+            n_jobs=num_threads,
+            cv=cv_splits,
+            scoring=self.component_config["scoring_function"],
+            verbose=1,
+        )
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Return the most likely intent and its probability for a message."""
+        for message in messages:
+            if self.clf is None or not message.features_present(
+                attribute=TEXT, featurizers=self.component_config.get(FEATURIZERS)
+            ):
+                # component is either not trained or didn't
+                # receive enough training data or the input doesn't
+                # have required features.
+                intent = None
+                intent_ranking = []
+            else:
+                X = self._get_sentence_features(message).reshape(1, -1)
+
+                intent_ids, probabilities = self.predict(X)
+                intents = self.transform_labels_num2str(np.ravel(intent_ids))
+                # `predict` returns a matrix as it is supposed
+                # to work for multiple examples as well, hence we need to flatten
+                probabilities = probabilities.flatten()
+
+                if intents.size > 0 and probabilities.size > 0:
+                    ranking = list(zip(list(intents), list(probabilities)))[
+                        :LABEL_RANKING_LENGTH
+                    ]
+
+                    intent = {"name": intents[0], "confidence": probabilities[0]}
+
+                    intent_ranking = [
+                        {"name": intent_name, "confidence": score}
+                        for intent_name, score in ranking
+                    ]
+                else:
+                    intent = {"name": None, "confidence": 0.0}
+                    intent_ranking = []
+
+            message.set("intent", intent, add_to_output=True)
+            message.set("intent_ranking", intent_ranking, add_to_output=True)
+
+        return messages
+
+    def predict_prob(self, X: np.ndarray) -> np.ndarray:
+        """Given a bow vector of an input text, predict the intent label.
+
+        Return probabilities for all labels.
+
+        :param X: bow of input text
+        :return: vector of probabilities containing one entry for each label.
+        """
+        if self.clf is None:
+            raise RasaException(
+                "Sklearn intent classifier has not been initialised and trained."
+            )
+
+        return self.clf.predict_proba(X)
+
+    def predict(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        """Given a bow vector of an input text, predict most probable label.
+
+        Return only the most likely label.
+
+        :param X: bow of input text
+        :return: tuple of first, the most probable label and second,
+                 its probability.
+        """
+        pred_result = self.predict_prob(X)
+        # sort the probabilities retrieving the indices of
+        # the elements in sorted order
+
+        sorted_indices = np.fliplr(np.argsort(pred_result, axis=1))
+        return sorted_indices, pred_result[:, sorted_indices]
+
+    def persist(self) -> None:
+        """Persist this model into the passed directory."""
+        with self._model_storage.write_to(self._resource) as model_dir:
+            file_name = self.__class__.__name__
+            classifier_file_name = model_dir / f"{file_name}_classifier.pkl"
+            encoder_file_name = model_dir / f"{file_name}_encoder.pkl"
+
+            if self.clf and self.le:
+                io_utils.json_pickle(encoder_file_name, self.le.classes_)
+                io_utils.json_pickle(classifier_file_name, self.clf.best_estimator_)
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> SklearnIntentClassifier:
+        """Loads trained component (see parent class for full docstring)."""
+        from sklearn.preprocessing import LabelEncoder
+
+        try:
+            with model_storage.read_from(resource) as model_dir:
+                file_name = cls.__name__
+                classifier_file = model_dir / f"{file_name}_classifier.pkl"
+
+                if classifier_file.exists():
+                    classifier = io_utils.json_unpickle(classifier_file)
+
+                    encoder_file = model_dir / f"{file_name}_encoder.pkl"
+                    classes = io_utils.json_unpickle(encoder_file)
+                    encoder = LabelEncoder()
+                    encoder.classes_ = classes
+
+                    return cls(config, model_storage, resource, classifier, encoder)
+        except ValueError:
+            logger.debug(
+                f"Failed to load '{cls.__name__}' from model storage. Resource "
+                f"'{resource.name}' doesn't exist."
+            )
+        return cls(config, model_storage, resource)

rasa/nlu/constants.py

@@ -0,0 +1,77 @@
+import rasa.shared.nlu.constants
+from rasa.shared.nlu.constants import ENTITY_ATTRIBUTE_CONFIDENCE
+
+BILOU_ENTITIES = "bilou_entities"
+BILOU_ENTITIES_ROLE = "bilou_entities_role"
+BILOU_ENTITIES_GROUP = "bilou_entities_group"
+
+ENTITY_ATTRIBUTE_CONFIDENCE_TYPE = (
+    f"{ENTITY_ATTRIBUTE_CONFIDENCE}_{rasa.shared.nlu.constants.ENTITY_ATTRIBUTE_TYPE}"
+)
+ENTITY_ATTRIBUTE_CONFIDENCE_GROUP = (
+    f"{ENTITY_ATTRIBUTE_CONFIDENCE}_{rasa.shared.nlu.constants.ENTITY_ATTRIBUTE_GROUP}"
+)
+ENTITY_ATTRIBUTE_CONFIDENCE_ROLE = (
+    f"{ENTITY_ATTRIBUTE_CONFIDENCE}_{rasa.shared.nlu.constants.ENTITY_ATTRIBUTE_ROLE}"
+)
+
+EXTRACTOR = "extractor"
+
+PRETRAINED_EXTRACTORS = {"DucklingEntityExtractor", "SpacyEntityExtractor"}
+
+NUMBER_OF_SUB_TOKENS = "number_of_sub_tokens"
+
+MESSAGE_ATTRIBUTES = [
+    rasa.shared.nlu.constants.TEXT,
+    rasa.shared.nlu.constants.INTENT,
+    rasa.shared.nlu.constants.RESPONSE,
+    rasa.shared.nlu.constants.ACTION_NAME,
+    rasa.shared.nlu.constants.ACTION_TEXT,
+    rasa.shared.nlu.constants.INTENT_RESPONSE_KEY,
+]
+# the dense featurizable attributes are essentially text attributes
+DENSE_FEATURIZABLE_ATTRIBUTES = [
+    rasa.shared.nlu.constants.TEXT,
+    rasa.shared.nlu.constants.RESPONSE,
+    rasa.shared.nlu.constants.ACTION_TEXT,
+]
+
+LANGUAGE_MODEL_DOCS = {
+    rasa.shared.nlu.constants.TEXT: "text_language_model_doc",
+    rasa.shared.nlu.constants.RESPONSE: "response_language_model_doc",
+    rasa.shared.nlu.constants.ACTION_TEXT: "action_text_model_doc",
+}
+SPACY_DOCS = {
+    rasa.shared.nlu.constants.TEXT: "text_spacy_doc",
+    rasa.shared.nlu.constants.RESPONSE: "response_spacy_doc",
+    rasa.shared.nlu.constants.ACTION_TEXT: "action_text_spacy_doc",
+}
+
+TOKENS_NAMES = {
+    rasa.shared.nlu.constants.TEXT: "text_tokens",
+    rasa.shared.nlu.constants.INTENT: "intent_tokens",
+    rasa.shared.nlu.constants.RESPONSE: "response_tokens",
+    rasa.shared.nlu.constants.ACTION_NAME: "action_name_tokens",
+    rasa.shared.nlu.constants.ACTION_TEXT: "action_text_tokens",
+    rasa.shared.nlu.constants.INTENT_RESPONSE_KEY: "intent_response_key_tokens",
+}
+
+SEQUENCE_FEATURES = "sequence_features"
+SENTENCE_FEATURES = "sentence_features"
+
+RESPONSE_SELECTOR_PROPERTY_NAME = "response_selector"
+RESPONSE_SELECTOR_RETRIEVAL_INTENTS = "all_retrieval_intents"
+RESPONSE_SELECTOR_DEFAULT_INTENT = "default"
+RESPONSE_SELECTOR_PREDICTION_KEY = "response"
+RESPONSE_SELECTOR_RANKING_KEY = "ranking"
+RESPONSE_SELECTOR_RESPONSES_KEY = "responses"
+RESPONSE_SELECTOR_RESPONSE_TEMPLATES_KEY = "response_templates"
+RESPONSE_SELECTOR_UTTER_ACTION_KEY = "utter_action"
+RESPONSE_SELECTOR_TEMPLATE_NAME_KEY = "template_name"
+RESPONSE_IDENTIFIER_DELIMITER = "/"
+
+DEFAULT_TRANSFORMER_SIZE = 256
+
+FEATURIZER_CLASS_ALIAS = "alias"
+
+NO_LENGTH_RESTRICTION = -1

rasa/nlu/convert.py

@@ -0,0 +1,40 @@
+import os
+from typing import Text, Union
+
+from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter
+from rasa.shared.utils.cli import print_error
+import rasa.shared.nlu.training_data.loading
+from rasa.nlu.utils import write_to_file
+
+
+def convert_training_data(
+    data_file: Union[list, Text], out_file: Text, output_format: Text, language: Text
+) -> None:
+    """Convert training data.
+
+    Args:
+        data_file (Union[list, Text]): Path to the file or directory
+            containing Rasa data.
+        out_file (Text): File or existing path where to save
+            training data in Rasa format.
+        output_format (Text): Output format the training data
+            should be converted into.
+        language (Text): Language of the data.
+    """
+    if isinstance(data_file, list):
+        data_file = data_file[0]
+
+    if not os.path.exists(str(data_file)):
+        print_error(
+            "Data file '{}' does not exist. Provide a valid NLU data file using "
+            "the '--data' argument.".format(data_file)
+        )
+        return
+
+    td = rasa.shared.nlu.training_data.loading.load_data(data_file, language)
+    if output_format == "json":
+        output = td.nlu_as_json(indent=2)
+    else:
+        output = RasaYAMLWriter().dumps(td)
+
+    write_to_file(out_file, output)

rasa/nlu/emulators/dialogflow.py

@@ -0,0 +1,55 @@
+import uuid
+from collections import defaultdict
+from typing import Any, Dict, Text
+
+from rasa.shared.nlu.constants import (
+    INTENT_NAME_KEY,
+    ENTITIES,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_VALUE,
+    INTENT,
+    TEXT,
+    PREDICTED_CONFIDENCE_KEY,
+)
+from rasa.nlu.emulators.emulator import Emulator
+
+
+class DialogflowEmulator(Emulator):
+    """Emulates the response format of the DialogFlow.
+
+    # noqa: W505
+    https://cloud.google.com/dialogflow/es/docs/reference/rest/v2/projects.agent.environments.users.sessions/detectIntent
+    https://cloud.google.com/dialogflow/es/docs/reference/rest/v2/DetectIntentResponse
+    """
+
+    def normalise_response_json(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Transform response JSON to DialogFlow format.
+
+        Args:
+            data: input JSON data as a dictionary.
+
+        Returns:
+            The transformed input data.
+        """
+        entities = defaultdict(list)
+        for entity in data[ENTITIES]:
+            entities[entity[ENTITY_ATTRIBUTE_TYPE]].append(
+                entity[ENTITY_ATTRIBUTE_VALUE]
+            )
+
+        return {
+            "responseId": str(uuid.uuid1()),
+            "queryResult": {
+                "queryText": data[TEXT],
+                "action": data[INTENT][INTENT_NAME_KEY],
+                "parameters": entities,
+                "fulfillmentText": "",
+                "fulfillmentMessages": [],
+                "outputContexts": [],
+                "intent": {
+                    "name": data[INTENT][INTENT_NAME_KEY],
+                    "displayName": data[INTENT][INTENT_NAME_KEY],
+                },
+                "intentDetectionConfidence": data[INTENT][PREDICTED_CONFIDENCE_KEY],
+            },
+        }

rasa/nlu/emulators/emulator.py

@@ -0,0 +1,49 @@
+from typing import Any, Dict, Text
+
+
+class Emulator:
+    """Emulator specifies how requests and responses are getting transformed."""
+
+    @classmethod
+    def name(cls) -> Text:
+        """Name that identifies the emulator."""
+        return cls.__name__
+
+    def normalise_request_json(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Transform request JSON to target format.
+
+        Args:
+            data: input JSON data as a dictionary.
+
+        Returns:
+            The transformed input data.
+        """
+        _data = {
+            "text": data["text"][0]
+            if type(data["text"]) == list  # noqa: E721
+            else data["text"]
+        }
+
+        if data.get("model"):
+            if type(data["model"]) == list:  # noqa: E721
+                _data["model"] = data["model"][0]
+            else:
+                _data["model"] = data["model"]
+
+        _data["time"] = data["time"] if "time" in data else None
+        return _data
+
+    def normalise_response_json(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Transform response JSON to target format.
+
+        Args:
+            data: input JSON data as a dictionary.
+
+        Returns:
+            The transformed input data.
+        """
+        raise NotImplementedError
+
+    def __str__(self) -> Text:
+        """Return the string representation of the emulator."""
+        return "Emulator('{}')".format(self.name())

rasa/nlu/emulators/luis.py

@@ -0,0 +1,86 @@
+from typing import Any, Dict, Text
+
+from rasa.nlu.emulators.emulator import Emulator
+from rasa.shared.nlu.constants import (
+    ENTITIES,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_ROLE,
+    ENTITY_ATTRIBUTE_VALUE,
+    ENTITY_ATTRIBUTE_START,
+    ENTITY_ATTRIBUTE_END,
+    EXTRACTOR,
+    INTENT_RANKING_KEY,
+    TEXT,
+    INTENT,
+    INTENT_NAME_KEY,
+    PREDICTED_CONFIDENCE_KEY,
+)
+
+
+class LUISEmulator(Emulator):
+    """Emulates the response format of the LUIS Endpoint API v3.0 /predict endpoint.
+
+    https://westcentralus.dev.cognitive.microsoft.com/docs/services/luis-endpoint-api-v3-0/
+    https://docs.microsoft.com/en-us/azure/cognitive-services/LUIS/luis-concept-data-extraction?tabs=V3
+    """
+
+    def _intents(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        if data.get(INTENT_RANKING_KEY):
+            return {
+                intent[INTENT_NAME_KEY]: {"score": intent[PREDICTED_CONFIDENCE_KEY]}
+                for intent in data[INTENT_RANKING_KEY]
+            }
+
+        top = data.get(INTENT)
+        if not top:
+            return {}
+
+        return {top[INTENT_NAME_KEY]: {"score": top[PREDICTED_CONFIDENCE_KEY]}}
+
+    def _entities(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        if ENTITIES not in data:
+            return {}
+
+        entities: Dict[Text, Any] = {"$instance": {}}
+        for e in data[ENTITIES]:
+            # LUIS API v3 uses entity roles instead of entity names
+            # (it's possible because its roles are unique):
+            # https://docs.microsoft.com/en-us/azure/cognitive-services/LUIS/luis-migration-api-v3#entity-role-name-instead-of-entity-name
+            key = e.get(ENTITY_ATTRIBUTE_ROLE, e[ENTITY_ATTRIBUTE_TYPE])
+            entities[key] = [e[ENTITY_ATTRIBUTE_VALUE]]
+
+            entities["$instance"][key] = [
+                {
+                    "role": e.get(ENTITY_ATTRIBUTE_ROLE),
+                    "type": e[ENTITY_ATTRIBUTE_TYPE],
+                    "text": e[ENTITY_ATTRIBUTE_VALUE],
+                    "startIndex": e.get(ENTITY_ATTRIBUTE_START),
+                    "length": (e[ENTITY_ATTRIBUTE_END] - e[ENTITY_ATTRIBUTE_START])
+                    if ENTITY_ATTRIBUTE_START in e and ENTITY_ATTRIBUTE_END in e
+                    else None,
+                    "score": e.get(PREDICTED_CONFIDENCE_KEY),
+                    "modelType": e.get(EXTRACTOR),
+                }
+            ]
+        return entities
+
+    def normalise_response_json(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Transform response JSON to LUIS format.
+
+        Args:
+            data: input JSON data as a dictionary.
+
+        Returns:
+            The transformed input data.
+        """
+        top = data.get(INTENT)
+
+        return {
+            "query": data[TEXT],
+            "prediction": {
+                "normalizedQuery": data[TEXT],
+                "topIntent": top[INTENT_NAME_KEY] if top else None,
+                "intents": self._intents(data),
+                "entities": self._entities(data),
+            },
+        }

rasa/nlu/emulators/no_emulator.py

@@ -0,0 +1,10 @@
+from typing import Any, Dict, Text
+from rasa.nlu.emulators.emulator import Emulator
+
+
+class NoEmulator(Emulator):
+    """Default emulator that is used when no emulator is specified."""
+
+    def normalise_response_json(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Transform data to target format."""
+        return data

rasa/nlu/emulators/wit.py

@@ -0,0 +1,56 @@
+from collections import defaultdict
+from typing import Any, Dict, Text
+
+from rasa.nlu.emulators.emulator import Emulator
+from rasa.shared.nlu.constants import (
+    ENTITIES,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_ROLE,
+    ENTITY_ATTRIBUTE_VALUE,
+    ENTITY_ATTRIBUTE_END,
+    ENTITY_ATTRIBUTE_START,
+    TEXT,
+    INTENT,
+)
+
+
+class WitEmulator(Emulator):
+    """Emulates the response format of this wit.ai endpoint.
+
+    More information about the endpoint:
+    https://wit.ai/docs/http/20200513/#get__message_link
+    """
+
+    def normalise_response_json(self, data: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Transform response JSON to wit.ai format.
+
+        Args:
+            data: input JSON data as a dictionary.
+
+        Returns:
+            The transformed input data.
+        """
+        entities = defaultdict(list)
+        for entity in data[ENTITIES]:
+            entity_name = entity[ENTITY_ATTRIBUTE_TYPE]
+            role = entity.get(ENTITY_ATTRIBUTE_ROLE, entity_name)
+            entity_name_including_role = f"{entity[ENTITY_ATTRIBUTE_TYPE]}:{role}"
+            normalized_entity: Dict[Text, Any] = {
+                "confidence": entity.get("confidence_entity") or 1,
+                "name": entity_name,
+                "value": entity[ENTITY_ATTRIBUTE_VALUE],
+                # Entity value before value was transformed (e.g. by synonym mapper)
+                "body": data["text"][
+                    entity.get(ENTITY_ATTRIBUTE_START, 0) : entity.get(
+                        ENTITY_ATTRIBUTE_END, 0
+                    )
+                ],
+                "start": entity[ENTITY_ATTRIBUTE_START],
+                "end": entity[ENTITY_ATTRIBUTE_END],
+                "role": role,
+                "entities": [],
+            }
+
+            entities[entity_name_including_role].append(normalized_entity)
+
+        return {"text": data[TEXT], "intents": [data[INTENT]], "entities": entities}