rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/nlu/classifiers/mitie_intent_classifier.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+import logging
+from rasa.nlu.featurizers.featurizer import Featurizer
+import typing
+from typing import Any, Dict, List, Optional, Text, Type
+
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.classifiers.classifier import IntentClassifier
+from rasa.nlu.utils.mitie_utils import MitieModel, MitieNLP
+from rasa.nlu.constants import TOKENS_NAMES
+from rasa.shared.nlu.constants import TEXT, INTENT
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+
+if typing.TYPE_CHECKING:
+    import mitie
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER,
+    is_trainable=True,
+    model_from="MitieNLP",
+)
+class MitieIntentClassifier(GraphComponent, IntentClassifier):
+    """Intent classifier which uses the `mitie` library."""
+
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [MitieNLP, Featurizer]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns default config (see parent class for full docstring)."""
+        return {"num_threads": 1}
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        clf: Optional["mitie.text_categorizer"] = None,
+    ) -> None:
+        """Constructs a new intent classifier using the MITIE framework."""
+        self._config = config
+        self._model_storage = model_storage
+        self._resource = resource
+        self._clf = clf
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Lists required dependencies (see parent class for full docstring)."""
+        return ["mitie"]
+
+    def train(self, training_data: TrainingData, model: MitieModel) -> Resource:
+        """Trains classifier.
+
+        Args:
+            training_data: The NLU training data.
+            model: The loaded mitie model provided by `MitieNLP`.
+
+        Returns:
+            The resource locator for the trained classifier.
+        """
+        import mitie
+
+        trainer = mitie.text_categorizer_trainer(str(model.model_path))
+        trainer.num_threads = self._config["num_threads"]
+
+        for example in training_data.intent_examples:
+            tokens = self._tokens_of_message(example)
+            trainer.add_labeled_text(tokens, example.get(INTENT))
+
+        if training_data.intent_examples:
+            # we can not call train if there are no examples!
+            clf = trainer.train()
+            self._persist(clf)
+
+        return self._resource
+
+    def process(self, messages: List[Message], model: MitieModel) -> List[Message]:
+        """Make intent predictions using `mitie`.
+
+        Args:
+            messages: The message which the intents should be predicted for.
+            model: The loaded mitie model provided by `MitieNLP`.
+        """
+        for message in messages:
+            if self._clf:
+                token_strs = self._tokens_of_message(message)
+                intent, confidence = self._clf(token_strs, model.word_feature_extractor)
+            else:
+                # either the model didn't get trained or it wasn't
+                # provided with any data
+                intent = None
+                confidence = 0.0
+
+            message.set(
+                "intent", {"name": intent, "confidence": confidence}, add_to_output=True
+            )
+
+        return messages
+
+    @staticmethod
+    def _tokens_of_message(message: Message) -> List[Text]:
+        return [token.text for token in message.get(TOKENS_NAMES[TEXT], [])]
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> MitieIntentClassifier:
+        """Creates component for training see parent class for full docstring)."""
+        return cls(config, model_storage, resource)
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> MitieIntentClassifier:
+        """Loads component for inference see parent class for full docstring)."""
+        import mitie
+
+        text_categorizer = None
+
+        try:
+            with model_storage.read_from(resource) as directory:
+                text_categorizer = mitie.text_categorizer(str(directory / "model.dat"))
+        except (
+            ValueError,
+            Exception,
+        ):  # the latter is thrown by the `mitie.text_categorizer`
+            logger.warning(
+                f"Failed to load {cls.__class__.__name__} from model storage. Resource "
+                f"'{resource.name}' doesn't exist."
+            )
+
+        return cls(config, model_storage, resource, text_categorizer)
+
+    def _persist(self, text_categorizer: "mitie.text_categorizer") -> None:
+        """Persists trained model (see parent class for full docstring)."""
+        with self._model_storage.write_to(self._resource) as directory:
+            classifier_file = directory / "model.dat"
+            text_categorizer.save_to_disk(str(classifier_file), pure_model=True)

rasa/nlu/classifiers/regex_message_handler.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+import logging
+from typing import Any, Dict, Optional, Text, List
+
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.extractors.extractor import EntityExtractorMixin
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+from rasa.shared.nlu.training_data.message import Message
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER, is_trainable=False
+)
+class RegexMessageHandler(GraphComponent, EntityExtractorMixin):
+    """Handles hardcoded NLU predictions from messages starting with a `/`."""
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> RegexMessageHandler:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls()
+
+    # TODO: Handle empty domain (NLU only training)
+    def process(
+        self, messages: List[Message], domain: Optional[Domain] = None
+    ) -> List[Message]:
+        """Adds hardcoded intents and entities for messages starting with '/'.
+
+        Args:
+            messages: The messages which should be handled.
+            domain: If given the domain is used to check whether the intent, entities
+                valid.
+
+        Returns:
+            The messages with potentially intent and entity prediction replaced
+            in case the message started with a `/`.
+        """
+        return [
+            YAMLStoryReader.unpack_regex_message(
+                message, domain, entity_extractor_name=self.name
+            )
+            for message in messages
+        ]

rasa/nlu/classifiers/sklearn_intent_classifier.py

@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+import logging
+import typing
+import warnings
+from typing import Any, Dict, List, Optional, Text, Tuple, Type
+
+import numpy as np
+
+import rasa.shared.utils.io
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
+from rasa.nlu.classifiers.classifier import IntentClassifier
+from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
+from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU
+from rasa.shared.exceptions import RasaException
+from rasa.shared.nlu.constants import TEXT
+from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.utils.tensorflow.constants import FEATURIZERS
+
+logger = logging.getLogger(__name__)
+
+if typing.TYPE_CHECKING:
+    import sklearn
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER, is_trainable=True
+)
+class SklearnIntentClassifier(GraphComponent, IntentClassifier):
+    """Intent classifier using the sklearn framework."""
+
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [DenseFeaturizer]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """The component's default config (see parent class for full docstring)."""
+        return {
+            # C parameter of the svm - cross validation will select the best value
+            "C": [1, 2, 5, 10, 20, 100],
+            # gamma parameter of the svm
+            "gamma": [0.1],
+            # the kernels to use for the svm training - cross validation will
+            # decide which one of them performs best
+            "kernels": ["linear"],
+            # We try to find a good number of cross folds to use during
+            # intent training, this specifies the max number of folds
+            "max_cross_validation_folds": 5,
+            # Scoring function used for evaluating the hyper parameters
+            # This can be a name or a function (cfr GridSearchCV doc for more info)
+            "scoring_function": "f1_weighted",
+            "num_threads": 1,
+        }
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        clf: Optional["sklearn.model_selection.GridSearchCV"] = None,
+        le: Optional["sklearn.preprocessing.LabelEncoder"] = None,
+    ) -> None:
+        """Construct a new intent classifier using the sklearn framework."""
+        from sklearn.preprocessing import LabelEncoder
+
+        self.component_config = config
+        self._model_storage = model_storage
+        self._resource = resource
+
+        if le is not None:
+            self.le = le
+        else:
+            self.le = LabelEncoder()
+        self.clf = clf
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> SklearnIntentClassifier:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource)
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["sklearn"]
+
+    def transform_labels_str2num(self, labels: List[Text]) -> np.ndarray:
+        """Transforms a list of strings into numeric label representation.
+
+        :param labels: List of labels to convert to numeric representation
+        """
+        return self.le.fit_transform(labels)
+
+    def transform_labels_num2str(self, y: np.ndarray) -> np.ndarray:
+        """Transforms a list of strings into numeric label representation.
+
+        :param y: List of labels to convert to numeric representation
+        """
+        return self.le.inverse_transform(y)
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Train the intent classifier on a data set."""
+        num_threads = self.component_config["num_threads"]
+
+        labels = [e.get("intent") for e in training_data.intent_examples]
+
+        if len(set(labels)) < 2:
+            rasa.shared.utils.io.raise_warning(
+                "Can not train an intent classifier as there are not "
+                "enough intents. Need at least 2 different intents. "
+                "Skipping training of intent classifier.",
+                docs=DOCS_URL_TRAINING_DATA_NLU,
+            )
+            return self._resource
+
+        y = self.transform_labels_str2num(labels)
+        training_examples = [
+            message
+            for message in training_data.intent_examples
+            if message.features_present(
+                attribute=TEXT, featurizers=self.component_config.get(FEATURIZERS)
+            )
+        ]
+        X = np.stack(
+            [self._get_sentence_features(example) for example in training_examples]
+        )
+        # reduce dimensionality
+        X = np.reshape(X, (len(X), -1))
+
+        self.clf = self._create_classifier(num_threads, y)
+
+        with warnings.catch_warnings():
+            # sklearn raises lots of
+            # "UndefinedMetricWarning: F - score is ill - defined"
+            # if there are few intent examples, this is needed to prevent it
+            warnings.simplefilter("ignore")
+            self.clf.fit(X, y)
+
+        self.persist()
+        return self._resource
+
+    @staticmethod
+    def _get_sentence_features(message: Message) -> np.ndarray:
+        _, sentence_features = message.get_dense_features(TEXT)
+        if sentence_features is not None:
+            return sentence_features.features[0]
+
+        raise ValueError(
+            "No sentence features present. Not able to train sklearn policy."
+        )
+
+    def _num_cv_splits(self, y: np.ndarray) -> int:
+        folds = self.component_config["max_cross_validation_folds"]
+        return max(2, min(folds, np.min(np.bincount(y)) // 5))
+
+    def _create_classifier(
+        self, num_threads: int, y: np.ndarray
+    ) -> "sklearn.model_selection.GridSearchCV":
+        from sklearn.model_selection import GridSearchCV
+        from sklearn.svm import SVC
+
+        C = self.component_config["C"]
+        kernels = self.component_config["kernels"]
+        gamma = self.component_config["gamma"]
+        # dirty str fix because sklearn is expecting
+        # str not instance of basestr...
+        tuned_parameters = [
+            {"C": C, "gamma": gamma, "kernel": [str(k) for k in kernels]}
+        ]
+
+        # aim for 5 examples in each fold
+
+        cv_splits = self._num_cv_splits(y)
+
+        return GridSearchCV(
+            SVC(C=1, probability=True, class_weight="balanced"),
+            param_grid=tuned_parameters,
+            n_jobs=num_threads,
+            cv=cv_splits,
+            scoring=self.component_config["scoring_function"],
+            verbose=1,
+        )
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Return the most likely intent and its probability for a message."""
+        for message in messages:
+            if self.clf is None or not message.features_present(
+                attribute=TEXT, featurizers=self.component_config.get(FEATURIZERS)
+            ):
+                # component is either not trained or didn't
+                # receive enough training data or the input doesn't
+                # have required features.
+                intent = None
+                intent_ranking = []
+            else:
+                X = self._get_sentence_features(message).reshape(1, -1)
+
+                intent_ids, probabilities = self.predict(X)
+                intents = self.transform_labels_num2str(np.ravel(intent_ids))
+                # `predict` returns a matrix as it is supposed
+                # to work for multiple examples as well, hence we need to flatten
+                probabilities = probabilities.flatten()
+
+                if intents.size > 0 and probabilities.size > 0:
+                    ranking = list(zip(list(intents), list(probabilities)))[
+                        :LABEL_RANKING_LENGTH
+                    ]
+
+                    intent = {"name": intents[0], "confidence": probabilities[0]}
+
+                    intent_ranking = [
+                        {"name": intent_name, "confidence": score}
+                        for intent_name, score in ranking
+                    ]
+                else:
+                    intent = {"name": None, "confidence": 0.0}
+                    intent_ranking = []
+
+            message.set("intent", intent, add_to_output=True)
+            message.set("intent_ranking", intent_ranking, add_to_output=True)
+
+        return messages
+
+    def predict_prob(self, X: np.ndarray) -> np.ndarray:
+        """Given a bow vector of an input text, predict the intent label.
+
+        Return probabilities for all labels.
+
+        :param X: bow of input text
+        :return: vector of probabilities containing one entry for each label.
+        """
+        if self.clf is None:
+            raise RasaException(
+                "Sklearn intent classifier has not been initialised and trained."
+            )
+
+        return self.clf.predict_proba(X)
+
+    def predict(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        """Given a bow vector of an input text, predict most probable label.
+
+        Return only the most likely label.
+
+        :param X: bow of input text
+        :return: tuple of first, the most probable label and second,
+                 its probability.
+        """
+        pred_result = self.predict_prob(X)
+        # sort the probabilities retrieving the indices of
+        # the elements in sorted order
+
+        sorted_indices = np.fliplr(np.argsort(pred_result, axis=1))
+        return sorted_indices, pred_result[:, sorted_indices]
+
+    def persist(self) -> None:
+        """Persist this model into the passed directory."""
+        import skops.io as sio
+
+        with self._model_storage.write_to(self._resource) as model_dir:
+            file_name = self.__class__.__name__
+            classifier_file_name = model_dir / f"{file_name}_classifier.skops"
+            encoder_file_name = model_dir / f"{file_name}_encoder.json"
+
+            if self.clf and self.le:
+                # convert self.le.classes_ (numpy array of strings) to a list in order
+                # to use json dump
+                rasa.shared.utils.io.dump_obj_as_json_to_file(
+                    encoder_file_name, list(self.le.classes_)
+                )
+                sio.dump(self.clf.best_estimator_, classifier_file_name)
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> SklearnIntentClassifier:
+        """Loads trained component (see parent class for full docstring)."""
+        from sklearn.preprocessing import LabelEncoder
+        import skops.io as sio
+
+        try:
+            with model_storage.read_from(resource) as model_dir:
+                file_name = cls.__name__
+                classifier_file = model_dir / f"{file_name}_classifier.skops"
+
+                if classifier_file.exists():
+                    unknown_types = sio.get_untrusted_types(file=classifier_file)
+
+                    if unknown_types:
+                        logger.error(
+                            f"Untrusted types ({unknown_types}) found when "
+                            f"loading {classifier_file}!"
+                        )
+                        raise ValueError()
+                    else:
+                        classifier = sio.load(classifier_file, trusted=unknown_types)
+
+                    encoder_file = model_dir / f"{file_name}_encoder.json"
+                    classes = rasa.shared.utils.io.read_json_file(encoder_file)
+
+                    encoder = LabelEncoder()
+                    intent_classifier = cls(
+                        config, model_storage, resource, classifier, encoder
+                    )
+                    # convert list of strings (class labels) back to numpy array of
+                    # strings
+                    intent_classifier.transform_labels_str2num(classes)
+                    return intent_classifier
+        except ValueError:
+            logger.debug(
+                f"Failed to load '{cls.__name__}' from model storage. Resource "
+                f"'{resource.name}' doesn't exist."
+            )
+        return cls(config, model_storage, resource)

rasa/nlu/constants.py

@@ -0,0 +1,77 @@
+import rasa.shared.nlu.constants
+from rasa.shared.nlu.constants import ENTITY_ATTRIBUTE_CONFIDENCE
+
+BILOU_ENTITIES = "bilou_entities"
+BILOU_ENTITIES_ROLE = "bilou_entities_role"
+BILOU_ENTITIES_GROUP = "bilou_entities_group"
+
+ENTITY_ATTRIBUTE_CONFIDENCE_TYPE = (
+    f"{ENTITY_ATTRIBUTE_CONFIDENCE}_{rasa.shared.nlu.constants.ENTITY_ATTRIBUTE_TYPE}"
+)
+ENTITY_ATTRIBUTE_CONFIDENCE_GROUP = (
+    f"{ENTITY_ATTRIBUTE_CONFIDENCE}_{rasa.shared.nlu.constants.ENTITY_ATTRIBUTE_GROUP}"
+)
+ENTITY_ATTRIBUTE_CONFIDENCE_ROLE = (
+    f"{ENTITY_ATTRIBUTE_CONFIDENCE}_{rasa.shared.nlu.constants.ENTITY_ATTRIBUTE_ROLE}"
+)
+
+EXTRACTOR = "extractor"
+
+PRETRAINED_EXTRACTORS = {"DucklingEntityExtractor", "SpacyEntityExtractor"}
+
+NUMBER_OF_SUB_TOKENS = "number_of_sub_tokens"
+
+MESSAGE_ATTRIBUTES = [
+    rasa.shared.nlu.constants.TEXT,
+    rasa.shared.nlu.constants.INTENT,
+    rasa.shared.nlu.constants.RESPONSE,
+    rasa.shared.nlu.constants.ACTION_NAME,
+    rasa.shared.nlu.constants.ACTION_TEXT,
+    rasa.shared.nlu.constants.INTENT_RESPONSE_KEY,
+]
+# the dense featurizable attributes are essentially text attributes
+DENSE_FEATURIZABLE_ATTRIBUTES = [
+    rasa.shared.nlu.constants.TEXT,
+    rasa.shared.nlu.constants.RESPONSE,
+    rasa.shared.nlu.constants.ACTION_TEXT,
+]
+
+LANGUAGE_MODEL_DOCS = {
+    rasa.shared.nlu.constants.TEXT: "text_language_model_doc",
+    rasa.shared.nlu.constants.RESPONSE: "response_language_model_doc",
+    rasa.shared.nlu.constants.ACTION_TEXT: "action_text_model_doc",
+}
+SPACY_DOCS = {
+    rasa.shared.nlu.constants.TEXT: "text_spacy_doc",
+    rasa.shared.nlu.constants.RESPONSE: "response_spacy_doc",
+    rasa.shared.nlu.constants.ACTION_TEXT: "action_text_spacy_doc",
+}
+
+TOKENS_NAMES = {
+    rasa.shared.nlu.constants.TEXT: "text_tokens",
+    rasa.shared.nlu.constants.INTENT: "intent_tokens",
+    rasa.shared.nlu.constants.RESPONSE: "response_tokens",
+    rasa.shared.nlu.constants.ACTION_NAME: "action_name_tokens",
+    rasa.shared.nlu.constants.ACTION_TEXT: "action_text_tokens",
+    rasa.shared.nlu.constants.INTENT_RESPONSE_KEY: "intent_response_key_tokens",
+}
+
+SEQUENCE_FEATURES = "sequence_features"
+SENTENCE_FEATURES = "sentence_features"
+
+RESPONSE_SELECTOR_PROPERTY_NAME = "response_selector"
+RESPONSE_SELECTOR_RETRIEVAL_INTENTS = "all_retrieval_intents"
+RESPONSE_SELECTOR_DEFAULT_INTENT = "default"
+RESPONSE_SELECTOR_PREDICTION_KEY = "response"
+RESPONSE_SELECTOR_RANKING_KEY = "ranking"
+RESPONSE_SELECTOR_RESPONSES_KEY = "responses"
+RESPONSE_SELECTOR_RESPONSE_TEMPLATES_KEY = "response_templates"
+RESPONSE_SELECTOR_UTTER_ACTION_KEY = "utter_action"
+RESPONSE_SELECTOR_TEMPLATE_NAME_KEY = "template_name"
+RESPONSE_IDENTIFIER_DELIMITER = "/"
+
+DEFAULT_TRANSFORMER_SIZE = 256
+
+FEATURIZER_CLASS_ALIAS = "alias"
+
+NO_LENGTH_RESTRICTION = -1

rasa/nlu/convert.py

@@ -0,0 +1,40 @@
+import os
+from typing import Text, Union
+
+from rasa.shared.nlu.training_data.formats.rasa_yaml import RasaYAMLWriter
+from rasa.shared.utils.cli import print_error
+import rasa.shared.nlu.training_data.loading
+from rasa.nlu.utils import write_to_file
+
+
+def convert_training_data(
+    data_file: Union[list, Text], out_file: Text, output_format: Text, language: Text
+) -> None:
+    """Convert training data.
+
+    Args:
+        data_file (Union[list, Text]): Path to the file or directory
+            containing Rasa data.
+        out_file (Text): File or existing path where to save
+            training data in Rasa format.
+        output_format (Text): Output format the training data
+            should be converted into.
+        language (Text): Language of the data.
+    """
+    if isinstance(data_file, list):
+        data_file = data_file[0]
+
+    if not os.path.exists(str(data_file)):
+        print_error(
+            "Data file '{}' does not exist. Provide a valid NLU data file using "
+            "the '--data' argument.".format(data_file)
+        )
+        return
+
+    td = rasa.shared.nlu.training_data.loading.load_data(data_file, language)
+    if output_format == "json":
+        output = td.nlu_as_json(indent=2)
+    else:
+        output = RasaYAMLWriter().dumps(td)
+
+    write_to_file(out_file, output)