rasa-pro 3.9.17__py3-none-any.whl → 3.10.3__py3-none-any.whl

rasa/model_training.py

@@ -1,12 +1,20 @@
 import sys
 import time
 from pathlib import Path
-from typing import Text, NamedTuple, Optional, List, Union, Dict, Any
+from typing import Any, Dict, List, NamedTuple, Optional, Text, Union
 
 import randomname
 import structlog
 
 import rasa.engine.validation
+import rasa.model
+import rasa.shared.constants
+import rasa.shared.exceptions
+import rasa.shared.utils.cli
+import rasa.shared.utils.common
+import rasa.shared.utils.io
+import rasa.utils.common
+from rasa import telemetry
 from rasa.engine.caching import LocalTrainingCache
 from rasa.engine.recipes.recipe import Recipe
 from rasa.engine.runner.dask import DaskGraphRunner
@@ -14,19 +22,13 @@ from rasa.engine.storage.local_model_storage import LocalModelStorage
 from rasa.engine.storage.storage import ModelStorage
 from rasa.engine.training.components import FingerprintStatus
 from rasa.engine.training.graph_trainer import GraphTrainer
+from rasa.nlu.persistor import StorageType
+from rasa.shared.core.domain import Domain
 from rasa.shared.core.events import SlotSet
 from rasa.shared.core.training_data.structures import StoryGraph
 from rasa.shared.data import TrainingType
+from rasa.shared.exceptions import RasaException
 from rasa.shared.importers.importer import TrainingDataImporter
-from rasa import telemetry
-from rasa.shared.core.domain import Domain
-import rasa.utils.common
-import rasa.shared.utils.common
-import rasa.shared.utils.cli
-import rasa.shared.exceptions
-import rasa.shared.utils.io
-import rasa.shared.constants
-import rasa.model
 
 CODE_NEEDS_TO_BE_RETRAINED = 0b0001
 CODE_FORCED_TRAINING = 0b1000
@@ -153,6 +155,7 @@ async def train(
     nlu_additional_arguments: Optional[Dict] = None,
     model_to_finetune: Optional[Text] = None,
     finetuning_epoch_fraction: float = 1.0,
+    remote_storage: Optional[StorageType] = None,
 ) -> TrainingResult:
     """Trains a Rasa model (Core and NLU).
 
@@ -174,6 +177,7 @@ async def train(
             a directory in case the latest trained model should be used.
         finetuning_epoch_fraction: The fraction currently specified training epochs
             in the model configuration which should be used for finetuning.
+        remote_storage: The remote storage which should be used to store the model.
 
     Returns:
         An instance of `TrainingResult`.
@@ -253,6 +257,7 @@ async def train(
             persist_nlu_training_data=persist_nlu_training_data,
             finetuning_epoch_fraction=finetuning_epoch_fraction,
             dry_run=dry_run,
+            remote_storage=remote_storage,
             **(core_additional_arguments or {}),
             **(nlu_additional_arguments or {}),
         )
@@ -266,6 +271,7 @@ async def _train_graph(
     model_to_finetune: Optional[Union[Text, Path]] = None,
     force_full_training: bool = False,
     dry_run: bool = False,
+    remote_storage: Optional[StorageType] = None,
     **kwargs: Any,
 ) -> TrainingResult:
     if model_to_finetune:
@@ -341,12 +347,23 @@ async def _train_graph(
                 force_retraining=force_full_training,
                 is_finetuning=is_finetuning,
             )
-            structlogger.info(
-                "model_training.train.finished_training",
-                event_info=(
-                    f"Your Rasa model is trained " f"and saved at '{full_model_path}'."
-                ),
-            )
+            if remote_storage:
+                push_model_to_remote_storage(full_model_path, remote_storage)
+                full_model_path.unlink()
+                structlogger.info(
+                    "model_training.train.finished_training",
+                    event_info=(
+                        f"Your Rasa model {model_name} is trained "
+                        f"and saved at remote storage provider '{remote_storage}'."
+                    ),
+                )
+            else:
+                structlogger.info(
+                    "model_training.train.finished_training",
+                    event_info=(
+                        f"Your Rasa model is trained and saved at '{full_model_path}'."
+                    ),
+                )
 
         return TrainingResult(str(full_model_path), 0)
 
@@ -534,3 +551,18 @@ async def train_nlu(
             **(additional_arguments or {}),
         )
     ).model
+
+
+def push_model_to_remote_storage(model_path: Path, remote_storage: StorageType) -> None:
+    """push model to remote storage"""
+    from rasa.nlu.persistor import get_persistor
+
+    persistor = get_persistor(remote_storage)
+
+    if persistor is not None:
+        persistor.persist(str(model_path))
+
+    else:
+        raise RasaException(
+            f"Persistor not found for remote storage: '{remote_storage}'."
+        )

rasa/nlu/classifiers/diet_classifier.py

@@ -1,17 +1,18 @@
 from __future__ import annotations
-
 import copy
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type
+
+from rasa.exceptions import ModelNotFound
+from rasa.nlu.featurizers.featurizer import Featurizer
 
 import numpy as np
 import scipy.sparse
 import tensorflow as tf
 
-from rasa.exceptions import ModelNotFound
-from rasa.nlu.featurizers.featurizer import Featurizer
+from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type
+
 from rasa.engine.graph import ExecutionContext, GraphComponent
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
@@ -19,21 +20,18 @@ from rasa.engine.storage.storage import ModelStorage
 from rasa.nlu.extractors.extractor import EntityExtractorMixin
 from rasa.nlu.classifiers.classifier import IntentClassifier
 import rasa.shared.utils.io
+import rasa.utils.io as io_utils
 import rasa.nlu.utils.bilou_utils as bilou_utils
 from rasa.shared.constants import DIAGNOSTIC_DATA
 from rasa.nlu.extractors.extractor import EntityTagSpec
 from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
 from rasa.utils import train_utils
 from rasa.utils.tensorflow import rasa_layers
-from rasa.utils.tensorflow.feature_array import (
-    FeatureArray,
-    serialize_nested_feature_arrays,
-    deserialize_nested_feature_arrays,
-)
 from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
 from rasa.utils.tensorflow.model_data import (
     RasaModelData,
     FeatureSignature,
+    FeatureArray,
 )
 from rasa.nlu.constants import TOKENS_NAMES, DEFAULT_TRANSFORMER_SIZE
 from rasa.shared.nlu.constants import (
@@ -120,6 +118,7 @@ LABEL_SUB_KEY = IDS
 
 POSSIBLE_TAGS = [ENTITY_ATTRIBUTE_TYPE, ENTITY_ATTRIBUTE_ROLE, ENTITY_ATTRIBUTE_GROUP]
 
+
 DIETClassifierT = TypeVar("DIETClassifierT", bound="DIETClassifier")
 
 
@@ -1084,24 +1083,18 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
 
             self.model.save(str(tf_model_file))
 
-            # save data example
-            serialize_nested_feature_arrays(
-                self._data_example,
-                model_path / f"{file_name}.data_example.st",
-                model_path / f"{file_name}.data_example_metadata.json",
-            )
-            # save label data
-            serialize_nested_feature_arrays(
-                dict(self._label_data.data) if self._label_data is not None else {},
-                model_path / f"{file_name}.label_data.st",
-                model_path / f"{file_name}.label_data_metadata.json",
+            io_utils.pickle_dump(
+                model_path / f"{file_name}.data_example.pkl", self._data_example
             )
-
-            rasa.shared.utils.io.dump_obj_as_json_to_file(
-                model_path / f"{file_name}.sparse_feature_sizes.json",
+            io_utils.pickle_dump(
+                model_path / f"{file_name}.sparse_feature_sizes.pkl",
                 self._sparse_feature_sizes,
             )
-            rasa.shared.utils.io.dump_obj_as_json_to_file(
+            io_utils.pickle_dump(
+                model_path / f"{file_name}.label_data.pkl",
+                dict(self._label_data.data) if self._label_data is not None else {},
+            )
+            io_utils.json_pickle(
                 model_path / f"{file_name}.index_label_id_mapping.json",
                 self.index_label_id_mapping,
             )
@@ -1190,22 +1183,15 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
     ]:
         file_name = cls.__name__
 
-        # load data example
-        data_example = deserialize_nested_feature_arrays(
-            str(model_path / f"{file_name}.data_example.st"),
-            str(model_path / f"{file_name}.data_example_metadata.json"),
+        data_example = io_utils.pickle_load(
+            model_path / f"{file_name}.data_example.pkl"
         )
-        # load label data
-        loaded_label_data = deserialize_nested_feature_arrays(
-            str(model_path / f"{file_name}.label_data.st"),
-            str(model_path / f"{file_name}.label_data_metadata.json"),
-        )
-        label_data = RasaModelData(data=loaded_label_data)
-
-        sparse_feature_sizes = rasa.shared.utils.io.read_json_file(
-            model_path / f"{file_name}.sparse_feature_sizes.json"
+        label_data = io_utils.pickle_load(model_path / f"{file_name}.label_data.pkl")
+        label_data = RasaModelData(data=label_data)
+        sparse_feature_sizes = io_utils.pickle_load(
+            model_path / f"{file_name}.sparse_feature_sizes.pkl"
         )
-        index_label_id_mapping = rasa.shared.utils.io.read_json_file(
+        index_label_id_mapping = io_utils.json_unpickle(
             model_path / f"{file_name}.index_label_id_mapping.json"
         )
         entity_tag_specs = rasa.shared.utils.io.read_json_file(
@@ -1225,6 +1211,7 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
             for tag_spec in entity_tag_specs
         ]
 
+        # jsonpickle converts dictionary keys to strings
         index_label_id_mapping = {
             int(key): value for key, value in index_label_id_mapping.items()
         }

rasa/nlu/classifiers/logistic_regression_classifier.py

@@ -1,21 +1,20 @@
 from typing import Any, Text, Dict, List, Type, Tuple
 
+import joblib
 import structlog
 from scipy.sparse import hstack, vstack, csr_matrix
-from sklearn.exceptions import NotFittedError
 from sklearn.linear_model import LogisticRegression
-from sklearn.utils.validation import check_is_fitted
 
-from rasa.engine.graph import ExecutionContext, GraphComponent
-from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.graph import ExecutionContext, GraphComponent
 from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
-from rasa.nlu.classifiers.classifier import IntentClassifier
 from rasa.nlu.featurizers.featurizer import Featurizer
-from rasa.shared.nlu.constants import TEXT, INTENT
-from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.classifiers.classifier import IntentClassifier
 from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.constants import TEXT, INTENT
 from rasa.utils.tensorflow.constants import RANKING_LENGTH
 
 structlogger = structlog.get_logger()
@@ -155,17 +154,6 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
 
     def process(self, messages: List[Message]) -> List[Message]:
         """Return the most likely intent and its probability for a message."""
-        # Check if the classifier is trained
-        if not self.is_trained():
-            structlogger.warning(
-                "logistic_regression_classifier.not_trained.skip_intent_prediction",
-                event_info=(
-                    f"The '{self.__class__.__name__}' is not trained. "
-                    f"Skipping intent prediction."
-                ),
-            )
-            return messages
-
         X = self._create_X(messages)
         probas = self.clf.predict_proba(X)
         for idx, message in enumerate(messages):
@@ -183,11 +171,9 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
 
     def persist(self) -> None:
         """Persist this model into the passed directory."""
-        import skops.io as sio
-
         with self._model_storage.write_to(self._resource) as model_dir:
-            path = model_dir / f"{self._resource.name}.skops"
-            sio.dump(self.clf, path)
+            path = model_dir / f"{self._resource.name}.joblib"
+            joblib.dump(self.clf, path)
             structlogger.debug(
                 "logistic_regression_classifier.persist",
                 event_info=f"Saved intent classifier to '{path}'.",
@@ -203,21 +189,9 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
         **kwargs: Any,
     ) -> "LogisticRegressionClassifier":
         """Loads trained component (see parent class for full docstring)."""
-        import skops.io as sio
-
         try:
             with model_storage.read_from(resource) as model_dir:
-                classifier_file = model_dir / f"{resource.name}.skops"
-                unknown_types = sio.get_untrusted_types(file=classifier_file)
-
-                if unknown_types:
-                    structlogger.error(
-                        f"Untrusted types found when loading {classifier_file}!",
-                        unknown_types=unknown_types,
-                    )
-                    raise ValueError()
-
-                classifier = sio.load(classifier_file, trusted=unknown_types)
+                classifier = joblib.load(model_dir / f"{resource.name}.joblib")
                 component = cls(
                     config, execution_context.node_name, model_storage, resource
                 )
@@ -242,12 +216,3 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
     def validate_config(cls, config: Dict[Text, Any]) -> None:
         """Validates that the component is configured properly."""
         pass
-
-    def is_trained(self) -> bool:
-        """Checks if the model has been trained."""
-        try:
-            # This will raise a NotFittedError if the classifier isn't fitted
-            check_is_fitted(self.clf)
-            return True
-        except NotFittedError:
-            return False

rasa/nlu/classifiers/sklearn_intent_classifier.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
-
 import logging
+from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
 import typing
 import warnings
 from typing import Any, Dict, List, Optional, Text, Tuple, Type
@@ -8,18 +8,18 @@ from typing import Any, Dict, List, Optional, Text, Tuple, Type
 import numpy as np
 
 import rasa.shared.utils.io
+import rasa.utils.io as io_utils
 from rasa.engine.graph import GraphComponent, ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
-from rasa.nlu.classifiers.classifier import IntentClassifier
-from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
 from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU
+from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
 from rasa.shared.exceptions import RasaException
 from rasa.shared.nlu.constants import TEXT
-from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.classifiers.classifier import IntentClassifier
 from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
 from rasa.utils.tensorflow.constants import FEATURIZERS
 
 logger = logging.getLogger(__name__)
@@ -266,20 +266,14 @@ class SklearnIntentClassifier(GraphComponent, IntentClassifier):
 
     def persist(self) -> None:
         """Persist this model into the passed directory."""
-        import skops.io as sio
-
         with self._model_storage.write_to(self._resource) as model_dir:
             file_name = self.__class__.__name__
-            classifier_file_name = model_dir / f"{file_name}_classifier.skops"
-            encoder_file_name = model_dir / f"{file_name}_encoder.json"
+            classifier_file_name = model_dir / f"{file_name}_classifier.pkl"
+            encoder_file_name = model_dir / f"{file_name}_encoder.pkl"
 
             if self.clf and self.le:
-                # convert self.le.classes_ (numpy array of strings) to a list in order
-                # to use json dump
-                rasa.shared.utils.io.dump_obj_as_json_to_file(
-                    encoder_file_name, list(self.le.classes_)
-                )
-                sio.dump(self.clf.best_estimator_, classifier_file_name)
+                io_utils.json_pickle(encoder_file_name, self.le.classes_)
+                io_utils.json_pickle(classifier_file_name, self.clf.best_estimator_)
 
     @classmethod
     def load(
@@ -292,36 +286,21 @@ class SklearnIntentClassifier(GraphComponent, IntentClassifier):
     ) -> SklearnIntentClassifier:
         """Loads trained component (see parent class for full docstring)."""
         from sklearn.preprocessing import LabelEncoder
-        import skops.io as sio
 
         try:
             with model_storage.read_from(resource) as model_dir:
                 file_name = cls.__name__
-                classifier_file = model_dir / f"{file_name}_classifier.skops"
+                classifier_file = model_dir / f"{file_name}_classifier.pkl"
 
                 if classifier_file.exists():
-                    unknown_types = sio.get_untrusted_types(file=classifier_file)
-
-                    if unknown_types:
-                        logger.error(
-                            f"Untrusted types ({unknown_types}) found when "
-                            f"loading {classifier_file}!"
-                        )
-                        raise ValueError()
-                    else:
-                        classifier = sio.load(classifier_file, trusted=unknown_types)
-
-                    encoder_file = model_dir / f"{file_name}_encoder.json"
-                    classes = rasa.shared.utils.io.read_json_file(encoder_file)
+                    classifier = io_utils.json_unpickle(classifier_file)
 
+                    encoder_file = model_dir / f"{file_name}_encoder.pkl"
+                    classes = io_utils.json_unpickle(encoder_file)
                     encoder = LabelEncoder()
-                    intent_classifier = cls(
-                        config, model_storage, resource, classifier, encoder
-                    )
-                    # convert list of strings (class labels) back to numpy array of
-                    # strings
-                    intent_classifier.transform_labels_str2num(classes)
-                    return intent_classifier
+                    encoder.classes_ = classes
+
+                    return cls(config, model_storage, resource, classifier, encoder)
         except ValueError:
             logger.debug(
                 f"Failed to load '{cls.__name__}' from model storage. Resource "