rasa-pro 3.10.10__py3-none-any.whl → 3.10.12__py3-none-any.whl

rasa/engine/storage/local_model_storage.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 import shutil
 import sys
 import tempfile
@@ -8,19 +7,21 @@ import uuid
 from contextlib import contextmanager
 from datetime import datetime
 from pathlib import Path
-from tarsafe import TarSafe
 from typing import Generator, Optional, Text, Tuple, Union
 
-import rasa.utils.common
+import structlog
+from tarsafe import TarSafe
+
+import rasa.model
 import rasa.shared.utils.io
-from rasa.engine.storage.storage import ModelMetadata, ModelStorage
+import rasa.utils.common
 from rasa.engine.graph import GraphModelConfiguration
 from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelMetadata, ModelStorage
 from rasa.exceptions import UnsupportedModelVersionError
 from rasa.shared.core.domain import Domain
-import rasa.model
 
-logger = logging.getLogger(__name__)
+structlogger = structlog.get_logger()
 
 # Paths within model archive
 MODEL_ARCHIVE_COMPONENTS_DIR = "components"
@@ -86,7 +87,14 @@ class LocalModelStorage(ModelStorage):
             cls._extract_archive_to_directory(
                 model_archive_path, temporary_directory_path
             )
-            logger.debug(f"Extracted model to '{temporary_directory_path}'.")
+            structlogger.debug(
+                "local_model_storage.from_model_archive",
+                event_info=(
+                    f"Extracted model '{model_archive_path}' to "
+                    f"'{temporary_directory_path}'."
+                ),
+            )
+
             cls._initialize_model_storage_from_model_archive(
                 temporary_directory_path, storage_path
             )
@@ -142,6 +150,10 @@ class LocalModelStorage(ModelStorage):
         temporary_directory: Path, storage_path: Path
     ) -> None:
         for path in (temporary_directory / MODEL_ARCHIVE_COMPONENTS_DIR).glob("*"):
+            structlogger.debug(
+                "local_model_storage._initialize_model_storage_from_model_archive",
+                event_info=f"Moving '{path}' to '{storage_path}'.",
+            )
             shutil.move(str(path), str(storage_path))
 
     @staticmethod
@@ -155,7 +167,10 @@ class LocalModelStorage(ModelStorage):
     @contextmanager
     def write_to(self, resource: Resource) -> Generator[Path, None, None]:
         """Persists data for a resource (see parent class for full docstring)."""
-        logger.debug(f"Resource '{resource.name}' was requested for writing.")
+        structlogger.debug(
+            "local_model_storage.write_to.resource_write_requested",
+            event_info=f"Resource '{resource.name}' was requested for writing.",
+        )
         directory = self._directory_for_resource(resource)
 
         if not directory.exists():
@@ -163,7 +178,10 @@ class LocalModelStorage(ModelStorage):
 
         yield directory
 
-        logger.debug(f"Resource '{resource.name}' was persisted.")
+        structlogger.debug(
+            "local_model_storage.write_to.resource_persisted",
+            event_info=f"Resource '{resource.name}' was persisted.",
+        )
 
     def _directory_for_resource(self, resource: Resource) -> Path:
         return self._storage_path / resource.name
@@ -171,7 +189,10 @@ class LocalModelStorage(ModelStorage):
     @contextmanager
     def read_from(self, resource: Resource) -> Generator[Path, None, None]:
         """Provides the data of a `Resource` (see parent class for full docstring)."""
-        logger.debug(f"Resource '{resource.name}' was requested for reading.")
+        structlogger.debug(
+            "local_model_storage.read_from",
+            event_info=f"Resource '{resource.name}' was requested for reading.",
+        )
         directory = self._directory_for_resource(resource)
 
         if not directory.exists():
@@ -193,7 +214,12 @@ class LocalModelStorage(ModelStorage):
         domain: Domain,
     ) -> ModelMetadata:
         """Creates model package (see parent class for full docstring)."""
-        logger.debug(f"Start to created model package for path '{model_archive_path}'.")
+        structlogger.debug(
+            "local_model_storage.create_model_package.started",
+            event_info=(
+                f"Start to created model " f"package for path '{model_archive_path}'.",
+            ),
+        )
 
         with windows_safe_temporary_directory() as temp_dir:
             temporary_directory = Path(temp_dir)
@@ -214,7 +240,10 @@ class LocalModelStorage(ModelStorage):
             with TarSafe.open(model_archive_path, "w:gz") as tar:
                 tar.add(temporary_directory, arcname="")
 
-        logger.debug(f"Model package created in path '{model_archive_path}'.")
+        structlogger.debug(
+            "local_model_storage.create_model_package.finished",
+            event_info=f"Model package created in path '{model_archive_path}'.",
+        )
 
         return model_metadata
 

rasa/model_training.py

@@ -22,7 +22,7 @@ from rasa.engine.storage.local_model_storage import LocalModelStorage
 from rasa.engine.storage.storage import ModelStorage
 from rasa.engine.training.components import FingerprintStatus
 from rasa.engine.training.graph_trainer import GraphTrainer
-from rasa.nlu.persistor import StorageType
+from rasa.nlu.persistor import RemoteStorageType, StorageType
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.events import SlotSet
 from rasa.shared.core.training_data.structures import StoryGraph
@@ -350,18 +350,25 @@ async def _train_graph(
             if remote_storage:
                 push_model_to_remote_storage(full_model_path, remote_storage)
                 full_model_path.unlink()
+                remote_storage_string = (
+                    remote_storage.value
+                    if isinstance(remote_storage, RemoteStorageType)
+                    else remote_storage
+                )
                 structlogger.info(
                     "model_training.train.finished_training",
                     event_info=(
                         f"Your Rasa model {model_name} is trained "
-                        f"and saved at remote storage provider '{remote_storage}'."
+                        f"and saved at remote storage provider "
+                        f"'{remote_storage_string}'."
                     ),
                 )
             else:
                 structlogger.info(
                     "model_training.train.finished_training",
                     event_info=(
-                        f"Your Rasa model is trained and saved at '{full_model_path}'."
+                        f"Your Rasa model is trained and saved at "
+                        f"'{full_model_path}'."
                     ),
                 )
 

rasa/nlu/classifiers/diet_classifier.py

@@ -1,18 +1,17 @@
 from __future__ import annotations
+
 import copy
 import logging
 from collections import defaultdict
 from pathlib import Path
-
-from rasa.exceptions import ModelNotFound
-from rasa.nlu.featurizers.featurizer import Featurizer
+from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type
 
 import numpy as np
 import scipy.sparse
 import tensorflow as tf
 
-from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type
-
+from rasa.exceptions import ModelNotFound
+from rasa.nlu.featurizers.featurizer import Featurizer
 from rasa.engine.graph import ExecutionContext, GraphComponent
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
@@ -20,18 +19,21 @@ from rasa.engine.storage.storage import ModelStorage
 from rasa.nlu.extractors.extractor import EntityExtractorMixin
 from rasa.nlu.classifiers.classifier import IntentClassifier
 import rasa.shared.utils.io
-import rasa.utils.io as io_utils
 import rasa.nlu.utils.bilou_utils as bilou_utils
 from rasa.shared.constants import DIAGNOSTIC_DATA
 from rasa.nlu.extractors.extractor import EntityTagSpec
 from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
 from rasa.utils import train_utils
 from rasa.utils.tensorflow import rasa_layers
+from rasa.utils.tensorflow.feature_array import (
+    FeatureArray,
+    serialize_nested_feature_arrays,
+    deserialize_nested_feature_arrays,
+)
 from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
 from rasa.utils.tensorflow.model_data import (
     RasaModelData,
     FeatureSignature,
-    FeatureArray,
 )
 from rasa.nlu.constants import TOKENS_NAMES, DEFAULT_TRANSFORMER_SIZE
 from rasa.shared.nlu.constants import (
@@ -118,7 +120,6 @@ LABEL_SUB_KEY = IDS
 
 POSSIBLE_TAGS = [ENTITY_ATTRIBUTE_TYPE, ENTITY_ATTRIBUTE_ROLE, ENTITY_ATTRIBUTE_GROUP]
 
-
 DIETClassifierT = TypeVar("DIETClassifierT", bound="DIETClassifier")
 
 
@@ -1083,18 +1084,24 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
 
             self.model.save(str(tf_model_file))
 
-            io_utils.pickle_dump(
-                model_path / f"{file_name}.data_example.pkl", self._data_example
-            )
-            io_utils.pickle_dump(
-                model_path / f"{file_name}.sparse_feature_sizes.pkl",
-                self._sparse_feature_sizes,
+            # save data example
+            serialize_nested_feature_arrays(
+                self._data_example,
+                model_path / f"{file_name}.data_example.st",
+                model_path / f"{file_name}.data_example_metadata.json",
             )
-            io_utils.pickle_dump(
-                model_path / f"{file_name}.label_data.pkl",
+            # save label data
+            serialize_nested_feature_arrays(
                 dict(self._label_data.data) if self._label_data is not None else {},
+                model_path / f"{file_name}.label_data.st",
+                model_path / f"{file_name}.label_data_metadata.json",
             )
-            io_utils.json_pickle(
+
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                model_path / f"{file_name}.sparse_feature_sizes.json",
+                self._sparse_feature_sizes,
+            )
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
                 model_path / f"{file_name}.index_label_id_mapping.json",
                 self.index_label_id_mapping,
             )
@@ -1183,15 +1190,22 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
     ]:
         file_name = cls.__name__
 
-        data_example = io_utils.pickle_load(
-            model_path / f"{file_name}.data_example.pkl"
+        # load data example
+        data_example = deserialize_nested_feature_arrays(
+            str(model_path / f"{file_name}.data_example.st"),
+            str(model_path / f"{file_name}.data_example_metadata.json"),
         )
-        label_data = io_utils.pickle_load(model_path / f"{file_name}.label_data.pkl")
-        label_data = RasaModelData(data=label_data)
-        sparse_feature_sizes = io_utils.pickle_load(
-            model_path / f"{file_name}.sparse_feature_sizes.pkl"
+        # load label data
+        loaded_label_data = deserialize_nested_feature_arrays(
+            str(model_path / f"{file_name}.label_data.st"),
+            str(model_path / f"{file_name}.label_data_metadata.json"),
+        )
+        label_data = RasaModelData(data=loaded_label_data)
+
+        sparse_feature_sizes = rasa.shared.utils.io.read_json_file(
+            model_path / f"{file_name}.sparse_feature_sizes.json"
         )
-        index_label_id_mapping = io_utils.json_unpickle(
+        index_label_id_mapping = rasa.shared.utils.io.read_json_file(
             model_path / f"{file_name}.index_label_id_mapping.json"
         )
         entity_tag_specs = rasa.shared.utils.io.read_json_file(
@@ -1211,7 +1225,6 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
             for tag_spec in entity_tag_specs
         ]
 
-        # jsonpickle converts dictionary keys to strings
         index_label_id_mapping = {
             int(key): value for key, value in index_label_id_mapping.items()
         }

rasa/nlu/classifiers/logistic_regression_classifier.py

@@ -1,22 +1,21 @@
 from typing import Any, Text, Dict, List, Type, Tuple
 
-import joblib
 import structlog
 from scipy.sparse import hstack, vstack, csr_matrix
 from sklearn.exceptions import NotFittedError
 from sklearn.linear_model import LogisticRegression
 from sklearn.utils.validation import check_is_fitted
 
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.engine.recipes.default_recipe import DefaultV1Recipe
-from rasa.engine.graph import ExecutionContext, GraphComponent
 from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
-from rasa.nlu.featurizers.featurizer import Featurizer
 from rasa.nlu.classifiers.classifier import IntentClassifier
-from rasa.shared.nlu.training_data.training_data import TrainingData
-from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.featurizers.featurizer import Featurizer
 from rasa.shared.nlu.constants import TEXT, INTENT
+from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.utils.tensorflow.constants import RANKING_LENGTH
 
 structlogger = structlog.get_logger()
@@ -184,9 +183,11 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
 
     def persist(self) -> None:
         """Persist this model into the passed directory."""
+        import skops.io as sio
+
         with self._model_storage.write_to(self._resource) as model_dir:
-            path = model_dir / f"{self._resource.name}.joblib"
-            joblib.dump(self.clf, path)
+            path = model_dir / f"{self._resource.name}.skops"
+            sio.dump(self.clf, path)
             structlogger.debug(
                 "logistic_regression_classifier.persist",
                 event_info=f"Saved intent classifier to '{path}'.",
@@ -202,9 +203,21 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
         **kwargs: Any,
     ) -> "LogisticRegressionClassifier":
         """Loads trained component (see parent class for full docstring)."""
+        import skops.io as sio
+
         try:
             with model_storage.read_from(resource) as model_dir:
-                classifier = joblib.load(model_dir / f"{resource.name}.joblib")
+                classifier_file = model_dir / f"{resource.name}.skops"
+                unknown_types = sio.get_untrusted_types(file=classifier_file)
+
+                if unknown_types:
+                    structlogger.error(
+                        f"Untrusted types found when loading {classifier_file}!",
+                        unknown_types=unknown_types,
+                    )
+                    raise ValueError()
+
+                classifier = sio.load(classifier_file, trusted=unknown_types)
                 component = cls(
                     config, execution_context.node_name, model_storage, resource
                 )

rasa/nlu/classifiers/sklearn_intent_classifier.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
+
 import logging
-from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
 import typing
 import warnings
 from typing import Any, Dict, List, Optional, Text, Tuple, Type
@@ -8,18 +8,18 @@ from typing import Any, Dict, List, Optional, Text, Tuple, Type
 import numpy as np
 
 import rasa.shared.utils.io
-import rasa.utils.io as io_utils
 from rasa.engine.graph import GraphComponent, ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU
 from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
+from rasa.nlu.classifiers.classifier import IntentClassifier
+from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
+from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU
 from rasa.shared.exceptions import RasaException
 from rasa.shared.nlu.constants import TEXT
-from rasa.nlu.classifiers.classifier import IntentClassifier
-from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.utils.tensorflow.constants import FEATURIZERS
 
 logger = logging.getLogger(__name__)
@@ -266,14 +266,20 @@ class SklearnIntentClassifier(GraphComponent, IntentClassifier):
 
     def persist(self) -> None:
         """Persist this model into the passed directory."""
+        import skops.io as sio
+
         with self._model_storage.write_to(self._resource) as model_dir:
             file_name = self.__class__.__name__
-            classifier_file_name = model_dir / f"{file_name}_classifier.pkl"
-            encoder_file_name = model_dir / f"{file_name}_encoder.pkl"
+            classifier_file_name = model_dir / f"{file_name}_classifier.skops"
+            encoder_file_name = model_dir / f"{file_name}_encoder.json"
 
             if self.clf and self.le:
-                io_utils.json_pickle(encoder_file_name, self.le.classes_)
-                io_utils.json_pickle(classifier_file_name, self.clf.best_estimator_)
+                # convert self.le.classes_ (numpy array of strings) to a list in order
+                # to use json dump
+                rasa.shared.utils.io.dump_obj_as_json_to_file(
+                    encoder_file_name, list(self.le.classes_)
+                )
+                sio.dump(self.clf.best_estimator_, classifier_file_name)
 
     @classmethod
     def load(
@@ -286,21 +292,36 @@ class SklearnIntentClassifier(GraphComponent, IntentClassifier):
     ) -> SklearnIntentClassifier:
         """Loads trained component (see parent class for full docstring)."""
         from sklearn.preprocessing import LabelEncoder
+        import skops.io as sio
 
         try:
             with model_storage.read_from(resource) as model_dir:
                 file_name = cls.__name__
-                classifier_file = model_dir / f"{file_name}_classifier.pkl"
+                classifier_file = model_dir / f"{file_name}_classifier.skops"
 
                 if classifier_file.exists():
-                    classifier = io_utils.json_unpickle(classifier_file)
+                    unknown_types = sio.get_untrusted_types(file=classifier_file)
 
-                    encoder_file = model_dir / f"{file_name}_encoder.pkl"
-                    classes = io_utils.json_unpickle(encoder_file)
-                    encoder = LabelEncoder()
-                    encoder.classes_ = classes
+                    if unknown_types:
+                        logger.error(
+                            f"Untrusted types ({unknown_types}) found when "
+                            f"loading {classifier_file}!"
+                        )
+                        raise ValueError()
+                    else:
+                        classifier = sio.load(classifier_file, trusted=unknown_types)
+
+                    encoder_file = model_dir / f"{file_name}_encoder.json"
+                    classes = rasa.shared.utils.io.read_json_file(encoder_file)
 
-                    return cls(config, model_storage, resource, classifier, encoder)
+                    encoder = LabelEncoder()
+                    intent_classifier = cls(
+                        config, model_storage, resource, classifier, encoder
+                    )
+                    # convert list of strings (class labels) back to numpy array of
+                    # strings
+                    intent_classifier.transform_labels_str2num(classes)
+                    return intent_classifier
         except ValueError:
             logger.debug(
                 f"Failed to load '{cls.__name__}' from model storage. Resource "