rasa-pro 3.10.9.dev1__py3-none-any.whl → 3.10.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rasa-pro might be problematic. Click here for more details.
- rasa/constants.py +1 -1
- rasa/core/featurizers/single_state_featurizer.py +1 -22
- rasa/core/featurizers/tracker_featurizers.py +18 -115
- rasa/core/policies/ted_policy.py +33 -58
- rasa/core/policies/unexpected_intent_policy.py +7 -15
- rasa/e2e_test/e2e_test_runner.py +2 -0
- rasa/nlu/classifiers/diet_classifier.py +25 -38
- rasa/nlu/classifiers/logistic_regression_classifier.py +9 -22
- rasa/nlu/classifiers/sklearn_intent_classifier.py +16 -37
- rasa/nlu/extractors/crf_entity_extractor.py +50 -93
- rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py +16 -45
- rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py +17 -52
- rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py +3 -5
- rasa/shared/nlu/training_data/features.py +2 -120
- rasa/shared/utils/io.py +0 -1
- rasa/utils/io.py +66 -0
- rasa/utils/tensorflow/model_data.py +193 -2
- rasa/version.py +1 -1
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.10.dist-info}/METADATA +5 -5
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.10.dist-info}/RECORD +23 -24
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.10.dist-info}/WHEEL +1 -1
- rasa/utils/tensorflow/feature_array.py +0 -366
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.10.dist-info}/NOTICE +0 -0
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.10.dist-info}/entry_points.txt +0 -0
rasa/constants.py
CHANGED
|
@@ -18,7 +18,7 @@ CONFIG_TELEMETRY_ID = "rasa_user_id"
|
|
|
18
18
|
CONFIG_TELEMETRY_ENABLED = "enabled"
|
|
19
19
|
CONFIG_TELEMETRY_DATE = "date"
|
|
20
20
|
|
|
21
|
-
MINIMUM_COMPATIBLE_VERSION = "3.10.
|
|
21
|
+
MINIMUM_COMPATIBLE_VERSION = "3.10.0rc1"
|
|
22
22
|
|
|
23
23
|
GLOBAL_USER_CONFIG_PATH = os.path.expanduser("~/.config/rasa/global.yml")
|
|
24
24
|
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import List, Optional, Dict, Text, Set, Any
|
|
3
|
-
|
|
4
2
|
import numpy as np
|
|
5
3
|
import scipy.sparse
|
|
4
|
+
from typing import List, Optional, Dict, Text, Set, Any
|
|
6
5
|
|
|
7
6
|
from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization
|
|
8
7
|
from rasa.nlu.extractors.extractor import EntityTagSpec
|
|
@@ -361,26 +360,6 @@ class SingleStateFeaturizer:
|
|
|
361
360
|
for action in domain.action_names_or_texts
|
|
362
361
|
]
|
|
363
362
|
|
|
364
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
365
|
-
return {
|
|
366
|
-
"action_texts": self.action_texts,
|
|
367
|
-
"entity_tag_specs": self.entity_tag_specs,
|
|
368
|
-
"feature_states": self._default_feature_states,
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
@classmethod
|
|
372
|
-
def create_from_dict(
|
|
373
|
-
cls, data: Dict[str, Any]
|
|
374
|
-
) -> Optional["SingleStateFeaturizer"]:
|
|
375
|
-
if not data:
|
|
376
|
-
return None
|
|
377
|
-
|
|
378
|
-
featurizer = SingleStateFeaturizer()
|
|
379
|
-
featurizer.action_texts = data["action_texts"]
|
|
380
|
-
featurizer._default_feature_states = data["feature_states"]
|
|
381
|
-
featurizer.entity_tag_specs = data["entity_tag_specs"]
|
|
382
|
-
return featurizer
|
|
383
|
-
|
|
384
363
|
|
|
385
364
|
class IntentTokenizerSingleStateFeaturizer(SingleStateFeaturizer):
|
|
386
365
|
"""A SingleStateFeaturizer for use with policies that predict intent labels."""
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import logging
|
|
4
|
-
from abc import abstractmethod
|
|
5
|
-
from collections import defaultdict
|
|
6
2
|
from pathlib import Path
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from abc import abstractmethod
|
|
5
|
+
import jsonpickle
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from tqdm import tqdm
|
|
7
9
|
from typing import (
|
|
8
10
|
Tuple,
|
|
9
11
|
List,
|
|
@@ -16,30 +18,25 @@ from typing import (
|
|
|
16
18
|
Set,
|
|
17
19
|
DefaultDict,
|
|
18
20
|
cast,
|
|
19
|
-
Type,
|
|
20
|
-
Callable,
|
|
21
|
-
ClassVar,
|
|
22
21
|
)
|
|
23
|
-
|
|
24
22
|
import numpy as np
|
|
25
|
-
from tqdm import tqdm
|
|
26
23
|
|
|
24
|
+
from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer
|
|
25
|
+
from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization
|
|
26
|
+
from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError
|
|
27
27
|
import rasa.shared.core.trackers
|
|
28
28
|
import rasa.shared.utils.io
|
|
29
|
-
from rasa.
|
|
30
|
-
from rasa.
|
|
31
|
-
from rasa.core.
|
|
29
|
+
from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME
|
|
30
|
+
from rasa.shared.nlu.training_data.features import Features
|
|
31
|
+
from rasa.shared.core.trackers import DialogueStateTracker
|
|
32
|
+
from rasa.shared.core.domain import State, Domain
|
|
33
|
+
from rasa.shared.core.events import Event, ActionExecuted, UserUttered
|
|
32
34
|
from rasa.shared.core.constants import (
|
|
33
35
|
USER,
|
|
34
36
|
ACTION_UNLIKELY_INTENT_NAME,
|
|
35
37
|
PREVIOUS_ACTION,
|
|
36
38
|
)
|
|
37
|
-
from rasa.shared.core.domain import State, Domain
|
|
38
|
-
from rasa.shared.core.events import Event, ActionExecuted, UserUttered
|
|
39
|
-
from rasa.shared.core.trackers import DialogueStateTracker
|
|
40
39
|
from rasa.shared.exceptions import RasaException
|
|
41
|
-
from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME
|
|
42
|
-
from rasa.shared.nlu.training_data.features import Features
|
|
43
40
|
from rasa.utils.tensorflow.constants import LABEL_PAD_ID
|
|
44
41
|
from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
|
|
45
42
|
|
|
@@ -67,10 +64,6 @@ class InvalidStory(RasaException):
|
|
|
67
64
|
class TrackerFeaturizer:
|
|
68
65
|
"""Base class for actual tracker featurizers."""
|
|
69
66
|
|
|
70
|
-
# Class registry to store all subclasses
|
|
71
|
-
_registry: ClassVar[Dict[str, Type["TrackerFeaturizer"]]] = {}
|
|
72
|
-
_featurizer_type: str = "TrackerFeaturizer"
|
|
73
|
-
|
|
74
67
|
def __init__(
|
|
75
68
|
self, state_featurizer: Optional[SingleStateFeaturizer] = None
|
|
76
69
|
) -> None:
|
|
@@ -81,36 +74,6 @@ class TrackerFeaturizer:
|
|
|
81
74
|
"""
|
|
82
75
|
self.state_featurizer = state_featurizer
|
|
83
76
|
|
|
84
|
-
@classmethod
|
|
85
|
-
def register(cls, featurizer_type: str) -> Callable:
|
|
86
|
-
"""Decorator to register featurizer subclasses."""
|
|
87
|
-
|
|
88
|
-
def wrapper(subclass: Type["TrackerFeaturizer"]) -> Type["TrackerFeaturizer"]:
|
|
89
|
-
cls._registry[featurizer_type] = subclass
|
|
90
|
-
# Store the type identifier in the class for serialization
|
|
91
|
-
subclass._featurizer_type = featurizer_type
|
|
92
|
-
return subclass
|
|
93
|
-
|
|
94
|
-
return wrapper
|
|
95
|
-
|
|
96
|
-
@classmethod
|
|
97
|
-
def from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer":
|
|
98
|
-
"""Create featurizer instance from dictionary."""
|
|
99
|
-
featurizer_type = data.pop("type")
|
|
100
|
-
|
|
101
|
-
if featurizer_type not in cls._registry:
|
|
102
|
-
raise ValueError(f"Unknown featurizer type: {featurizer_type}")
|
|
103
|
-
|
|
104
|
-
# Get the correct subclass and instantiate it
|
|
105
|
-
subclass = cls._registry[featurizer_type]
|
|
106
|
-
return subclass.create_from_dict(data)
|
|
107
|
-
|
|
108
|
-
@classmethod
|
|
109
|
-
@abstractmethod
|
|
110
|
-
def create_from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer":
|
|
111
|
-
"""Each subclass must implement its own creation from dict method."""
|
|
112
|
-
pass
|
|
113
|
-
|
|
114
77
|
@staticmethod
|
|
115
78
|
def _create_states(
|
|
116
79
|
tracker: DialogueStateTracker,
|
|
@@ -502,7 +465,9 @@ class TrackerFeaturizer:
|
|
|
502
465
|
self.state_featurizer.entity_tag_specs = []
|
|
503
466
|
|
|
504
467
|
# noinspection PyTypeChecker
|
|
505
|
-
rasa.shared.utils.io.
|
|
468
|
+
rasa.shared.utils.io.write_text_file(
|
|
469
|
+
str(jsonpickle.encode(self)), featurizer_file
|
|
470
|
+
)
|
|
506
471
|
|
|
507
472
|
@staticmethod
|
|
508
473
|
def load(path: Union[Text, Path]) -> Optional[TrackerFeaturizer]:
|
|
@@ -516,17 +481,7 @@ class TrackerFeaturizer:
|
|
|
516
481
|
"""
|
|
517
482
|
featurizer_file = Path(path) / FEATURIZER_FILE
|
|
518
483
|
if featurizer_file.is_file():
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
if "type" not in data:
|
|
522
|
-
logger.error(
|
|
523
|
-
f"Couldn't load featurizer for policy. "
|
|
524
|
-
f"File '{featurizer_file}' does not contain all "
|
|
525
|
-
f"necessary information. 'type' is missing."
|
|
526
|
-
)
|
|
527
|
-
return None
|
|
528
|
-
|
|
529
|
-
return TrackerFeaturizer.from_dict(data)
|
|
484
|
+
return jsonpickle.decode(rasa.shared.utils.io.read_file(featurizer_file))
|
|
530
485
|
|
|
531
486
|
logger.error(
|
|
532
487
|
f"Couldn't load featurizer for policy. "
|
|
@@ -553,16 +508,7 @@ class TrackerFeaturizer:
|
|
|
553
508
|
)
|
|
554
509
|
]
|
|
555
510
|
|
|
556
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
557
|
-
return {
|
|
558
|
-
"type": self.__class__._featurizer_type,
|
|
559
|
-
"state_featurizer": (
|
|
560
|
-
self.state_featurizer.to_dict() if self.state_featurizer else None
|
|
561
|
-
),
|
|
562
|
-
}
|
|
563
|
-
|
|
564
511
|
|
|
565
|
-
@TrackerFeaturizer.register("FullDialogueTrackerFeaturizer")
|
|
566
512
|
class FullDialogueTrackerFeaturizer(TrackerFeaturizer):
|
|
567
513
|
"""Creates full dialogue training data for time distributed architectures.
|
|
568
514
|
|
|
@@ -700,20 +646,7 @@ class FullDialogueTrackerFeaturizer(TrackerFeaturizer):
|
|
|
700
646
|
|
|
701
647
|
return trackers_as_states
|
|
702
648
|
|
|
703
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
704
|
-
return super().to_dict()
|
|
705
649
|
|
|
706
|
-
@classmethod
|
|
707
|
-
def create_from_dict(cls, data: Dict[str, Any]) -> "FullDialogueTrackerFeaturizer":
|
|
708
|
-
state_featurizer = SingleStateFeaturizer.create_from_dict(
|
|
709
|
-
data["state_featurizer"]
|
|
710
|
-
)
|
|
711
|
-
return cls(
|
|
712
|
-
state_featurizer,
|
|
713
|
-
)
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
@TrackerFeaturizer.register("MaxHistoryTrackerFeaturizer")
|
|
717
650
|
class MaxHistoryTrackerFeaturizer(TrackerFeaturizer):
|
|
718
651
|
"""Truncates the tracker history into `max_history` long sequences.
|
|
719
652
|
|
|
@@ -951,25 +884,7 @@ class MaxHistoryTrackerFeaturizer(TrackerFeaturizer):
|
|
|
951
884
|
|
|
952
885
|
return trackers_as_states
|
|
953
886
|
|
|
954
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
955
|
-
data = super().to_dict()
|
|
956
|
-
data.update(
|
|
957
|
-
{
|
|
958
|
-
"remove_duplicates": self.remove_duplicates,
|
|
959
|
-
"max_history": self.max_history,
|
|
960
|
-
}
|
|
961
|
-
)
|
|
962
|
-
return data
|
|
963
|
-
|
|
964
|
-
@classmethod
|
|
965
|
-
def create_from_dict(cls, data: Dict[str, Any]) -> "MaxHistoryTrackerFeaturizer":
|
|
966
|
-
state_featurizer = SingleStateFeaturizer.create_from_dict(
|
|
967
|
-
data["state_featurizer"]
|
|
968
|
-
)
|
|
969
|
-
return cls(state_featurizer, data["max_history"], data["remove_duplicates"])
|
|
970
887
|
|
|
971
|
-
|
|
972
|
-
@TrackerFeaturizer.register("IntentMaxHistoryTrackerFeaturizer")
|
|
973
888
|
class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer):
|
|
974
889
|
"""Truncates the tracker history into `max_history` long sequences.
|
|
975
890
|
|
|
@@ -1244,18 +1159,6 @@ class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer):
|
|
|
1244
1159
|
|
|
1245
1160
|
return trackers_as_states
|
|
1246
1161
|
|
|
1247
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
1248
|
-
return super().to_dict()
|
|
1249
|
-
|
|
1250
|
-
@classmethod
|
|
1251
|
-
def create_from_dict(
|
|
1252
|
-
cls, data: Dict[str, Any]
|
|
1253
|
-
) -> "IntentMaxHistoryTrackerFeaturizer":
|
|
1254
|
-
state_featurizer = SingleStateFeaturizer.create_from_dict(
|
|
1255
|
-
data["state_featurizer"]
|
|
1256
|
-
)
|
|
1257
|
-
return cls(state_featurizer, data["max_history"], data["remove_duplicates"])
|
|
1258
|
-
|
|
1259
1162
|
|
|
1260
1163
|
def _is_prev_action_unlikely_intent_in_state(state: State) -> bool:
|
|
1261
1164
|
prev_action_name = state.get(PREVIOUS_ACTION, {}).get(ACTION_NAME)
|
rasa/core/policies/ted_policy.py
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
3
2
|
import logging
|
|
3
|
+
|
|
4
|
+
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from collections import defaultdict
|
|
6
7
|
import contextlib
|
|
7
|
-
from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
|
|
8
8
|
|
|
9
9
|
import numpy as np
|
|
10
10
|
import tensorflow as tf
|
|
11
|
+
from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
|
|
11
12
|
|
|
12
|
-
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
13
13
|
from rasa.engine.graph import ExecutionContext
|
|
14
14
|
from rasa.engine.storage.resource import Resource
|
|
15
15
|
from rasa.engine.storage.storage import ModelStorage
|
|
@@ -49,22 +49,18 @@ from rasa.shared.core.generator import TrackerWithCachedStates
|
|
|
49
49
|
from rasa.shared.core.events import EntitiesAdded, Event
|
|
50
50
|
from rasa.shared.core.domain import Domain
|
|
51
51
|
from rasa.shared.nlu.training_data.message import Message
|
|
52
|
-
from rasa.shared.nlu.training_data.features import
|
|
53
|
-
Features,
|
|
54
|
-
save_features,
|
|
55
|
-
load_features,
|
|
56
|
-
)
|
|
52
|
+
from rasa.shared.nlu.training_data.features import Features
|
|
57
53
|
import rasa.shared.utils.io
|
|
58
54
|
import rasa.utils.io
|
|
59
55
|
from rasa.utils import train_utils
|
|
60
|
-
from rasa.utils.tensorflow.feature_array import (
|
|
61
|
-
FeatureArray,
|
|
62
|
-
serialize_nested_feature_arrays,
|
|
63
|
-
deserialize_nested_feature_arrays,
|
|
64
|
-
)
|
|
65
56
|
from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
|
|
66
57
|
from rasa.utils.tensorflow import rasa_layers
|
|
67
|
-
from rasa.utils.tensorflow.model_data import
|
|
58
|
+
from rasa.utils.tensorflow.model_data import (
|
|
59
|
+
RasaModelData,
|
|
60
|
+
FeatureSignature,
|
|
61
|
+
FeatureArray,
|
|
62
|
+
Data,
|
|
63
|
+
)
|
|
68
64
|
from rasa.utils.tensorflow.model_data_utils import convert_to_data_format
|
|
69
65
|
from rasa.utils.tensorflow.constants import (
|
|
70
66
|
LABEL,
|
|
@@ -965,32 +961,22 @@ class TEDPolicy(Policy):
|
|
|
965
961
|
model_path: Path where model is to be persisted
|
|
966
962
|
"""
|
|
967
963
|
model_filename = self._metadata_filename()
|
|
968
|
-
rasa.
|
|
969
|
-
model_path / f"{model_filename}.priority.
|
|
970
|
-
)
|
|
971
|
-
rasa.shared.utils.io.dump_obj_as_json_to_file(
|
|
972
|
-
model_path / f"{model_filename}.meta.json", self.config
|
|
964
|
+
rasa.utils.io.json_pickle(
|
|
965
|
+
model_path / f"{model_filename}.priority.pkl", self.priority
|
|
973
966
|
)
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
self.data_example,
|
|
977
|
-
str(model_path / f"{model_filename}.data_example.st"),
|
|
978
|
-
str(model_path / f"{model_filename}.data_example_metadata.json"),
|
|
967
|
+
rasa.utils.io.pickle_dump(
|
|
968
|
+
model_path / f"{model_filename}.meta.pkl", self.config
|
|
979
969
|
)
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
dict(self._label_data.data) if self._label_data is not None else {},
|
|
983
|
-
str(model_path / f"{model_filename}.label_data.st"),
|
|
984
|
-
str(model_path / f"{model_filename}.label_data_metadata.json"),
|
|
970
|
+
rasa.utils.io.pickle_dump(
|
|
971
|
+
model_path / f"{model_filename}.data_example.pkl", self.data_example
|
|
985
972
|
)
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
self.fake_features, str(model_path / f"{model_filename}.fake_features.st")
|
|
973
|
+
rasa.utils.io.pickle_dump(
|
|
974
|
+
model_path / f"{model_filename}.fake_features.pkl", self.fake_features
|
|
989
975
|
)
|
|
990
|
-
rasa.
|
|
991
|
-
model_path / f"{model_filename}.
|
|
976
|
+
rasa.utils.io.pickle_dump(
|
|
977
|
+
model_path / f"{model_filename}.label_data.pkl",
|
|
978
|
+
dict(self._label_data.data) if self._label_data is not None else {},
|
|
992
979
|
)
|
|
993
|
-
|
|
994
980
|
entity_tag_specs = (
|
|
995
981
|
[tag_spec._asdict() for tag_spec in self._entity_tag_specs]
|
|
996
982
|
if self._entity_tag_specs
|
|
@@ -1008,29 +994,18 @@ class TEDPolicy(Policy):
|
|
|
1008
994
|
model_path: Path where model is to be persisted.
|
|
1009
995
|
"""
|
|
1010
996
|
tf_model_file = model_path / f"{cls._metadata_filename()}.tf_model"
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
loaded_data = deserialize_nested_feature_arrays(
|
|
1014
|
-
str(model_path / f"{cls._metadata_filename()}.data_example.st"),
|
|
1015
|
-
str(model_path / f"{cls._metadata_filename()}.data_example_metadata.json"),
|
|
997
|
+
loaded_data = rasa.utils.io.pickle_load(
|
|
998
|
+
model_path / f"{cls._metadata_filename()}.data_example.pkl"
|
|
1016
999
|
)
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
str(model_path / f"{cls._metadata_filename()}.label_data.st"),
|
|
1020
|
-
str(model_path / f"{cls._metadata_filename()}.label_data_metadata.json"),
|
|
1000
|
+
label_data = rasa.utils.io.pickle_load(
|
|
1001
|
+
model_path / f"{cls._metadata_filename()}.label_data.pkl"
|
|
1021
1002
|
)
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
# load fake features
|
|
1025
|
-
metadata = rasa.shared.utils.io.read_json_file(
|
|
1026
|
-
model_path / f"{cls._metadata_filename()}.fake_features_metadata.json"
|
|
1003
|
+
fake_features = rasa.utils.io.pickle_load(
|
|
1004
|
+
model_path / f"{cls._metadata_filename()}.fake_features.pkl"
|
|
1027
1005
|
)
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
priority = rasa.shared.utils.io.read_json_file(
|
|
1033
|
-
model_path / f"{cls._metadata_filename()}.priority.json"
|
|
1006
|
+
label_data = RasaModelData(data=label_data)
|
|
1007
|
+
priority = rasa.utils.io.json_unpickle(
|
|
1008
|
+
model_path / f"{cls._metadata_filename()}.priority.pkl"
|
|
1034
1009
|
)
|
|
1035
1010
|
entity_tag_specs = rasa.shared.utils.io.read_json_file(
|
|
1036
1011
|
model_path / f"{cls._metadata_filename()}.entity_tag_specs.json"
|
|
@@ -1048,8 +1023,8 @@ class TEDPolicy(Policy):
|
|
|
1048
1023
|
)
|
|
1049
1024
|
for tag_spec in entity_tag_specs
|
|
1050
1025
|
]
|
|
1051
|
-
model_config = rasa.
|
|
1052
|
-
model_path / f"{cls._metadata_filename()}.meta.
|
|
1026
|
+
model_config = rasa.utils.io.pickle_load(
|
|
1027
|
+
model_path / f"{cls._metadata_filename()}.meta.pkl"
|
|
1053
1028
|
)
|
|
1054
1029
|
|
|
1055
1030
|
return {
|
|
@@ -1095,7 +1070,7 @@ class TEDPolicy(Policy):
|
|
|
1095
1070
|
) -> TEDPolicy:
|
|
1096
1071
|
featurizer = TrackerFeaturizer.load(model_path)
|
|
1097
1072
|
|
|
1098
|
-
if not (model_path / f"{cls._metadata_filename()}.data_example.
|
|
1073
|
+
if not (model_path / f"{cls._metadata_filename()}.data_example.pkl").is_file():
|
|
1099
1074
|
return cls(
|
|
1100
1075
|
config,
|
|
1101
1076
|
model_storage,
|
|
@@ -5,7 +5,6 @@ from typing import Any, List, Optional, Text, Dict, Type, Union
|
|
|
5
5
|
|
|
6
6
|
import numpy as np
|
|
7
7
|
import tensorflow as tf
|
|
8
|
-
|
|
9
8
|
import rasa.utils.common
|
|
10
9
|
from rasa.engine.graph import ExecutionContext
|
|
11
10
|
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
@@ -17,7 +16,6 @@ from rasa.shared.core.domain import Domain
|
|
|
17
16
|
from rasa.shared.core.trackers import DialogueStateTracker
|
|
18
17
|
from rasa.shared.core.constants import SLOTS, ACTIVE_LOOP, ACTION_UNLIKELY_INTENT_NAME
|
|
19
18
|
from rasa.shared.core.events import UserUttered, ActionExecuted
|
|
20
|
-
import rasa.shared.utils.io
|
|
21
19
|
from rasa.shared.nlu.constants import (
|
|
22
20
|
INTENT,
|
|
23
21
|
TEXT,
|
|
@@ -105,6 +103,8 @@ from rasa.utils.tensorflow.constants import (
|
|
|
105
103
|
)
|
|
106
104
|
from rasa.utils.tensorflow import layers
|
|
107
105
|
from rasa.utils.tensorflow.model_data import RasaModelData, FeatureArray, Data
|
|
106
|
+
|
|
107
|
+
import rasa.utils.io as io_utils
|
|
108
108
|
from rasa.core.exceptions import RasaCoreException
|
|
109
109
|
from rasa.shared.utils import common
|
|
110
110
|
|
|
@@ -881,12 +881,9 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
|
|
|
881
881
|
model_path: Path where model is to be persisted
|
|
882
882
|
"""
|
|
883
883
|
super().persist_model_utilities(model_path)
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
save_file(
|
|
888
|
-
{str(k): np.array(v) for k, v in self.label_quantiles.items()},
|
|
889
|
-
model_path / f"{self._metadata_filename()}.label_quantiles.st",
|
|
884
|
+
io_utils.pickle_dump(
|
|
885
|
+
model_path / f"{self._metadata_filename()}.label_quantiles.pkl",
|
|
886
|
+
self.label_quantiles,
|
|
890
887
|
)
|
|
891
888
|
|
|
892
889
|
@classmethod
|
|
@@ -897,14 +894,9 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
|
|
|
897
894
|
model_path: Path where model is to be persisted.
|
|
898
895
|
"""
|
|
899
896
|
model_utilties = super()._load_model_utilities(model_path)
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
loaded_label_quantiles = load_file(
|
|
904
|
-
model_path / f"{cls._metadata_filename()}.label_quantiles.st"
|
|
897
|
+
label_quantiles = io_utils.pickle_load(
|
|
898
|
+
model_path / f"{cls._metadata_filename()}.label_quantiles.pkl"
|
|
905
899
|
)
|
|
906
|
-
label_quantiles = {int(k): list(v) for k, v in loaded_label_quantiles.items()}
|
|
907
|
-
|
|
908
900
|
model_utilties.update({"label_quantiles": label_quantiles})
|
|
909
901
|
return model_utilties
|
|
910
902
|
|
rasa/e2e_test/e2e_test_runner.py
CHANGED
|
@@ -1155,6 +1155,8 @@ class E2ETestRunner:
|
|
|
1155
1155
|
flow_paths_stack
|
|
1156
1156
|
and self.agent.domain
|
|
1157
1157
|
and self.agent.domain.is_custom_action(event.action_name)
|
|
1158
|
+
and STEP_ID_METADATA_KEY in event.metadata
|
|
1159
|
+
and ACTIVE_FLOW_METADATA_KEY in event.metadata
|
|
1158
1160
|
):
|
|
1159
1161
|
flow_paths_stack[-1].nodes.append(self._create_path_node(event))
|
|
1160
1162
|
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
3
2
|
import copy
|
|
4
3
|
import logging
|
|
5
4
|
from collections import defaultdict
|
|
6
5
|
from pathlib import Path
|
|
7
|
-
|
|
6
|
+
|
|
7
|
+
from rasa.exceptions import ModelNotFound
|
|
8
|
+
from rasa.nlu.featurizers.featurizer import Featurizer
|
|
8
9
|
|
|
9
10
|
import numpy as np
|
|
10
11
|
import scipy.sparse
|
|
11
12
|
import tensorflow as tf
|
|
12
13
|
|
|
13
|
-
from
|
|
14
|
-
|
|
14
|
+
from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type
|
|
15
|
+
|
|
15
16
|
from rasa.engine.graph import ExecutionContext, GraphComponent
|
|
16
17
|
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
17
18
|
from rasa.engine.storage.resource import Resource
|
|
@@ -19,21 +20,18 @@ from rasa.engine.storage.storage import ModelStorage
|
|
|
19
20
|
from rasa.nlu.extractors.extractor import EntityExtractorMixin
|
|
20
21
|
from rasa.nlu.classifiers.classifier import IntentClassifier
|
|
21
22
|
import rasa.shared.utils.io
|
|
23
|
+
import rasa.utils.io as io_utils
|
|
22
24
|
import rasa.nlu.utils.bilou_utils as bilou_utils
|
|
23
25
|
from rasa.shared.constants import DIAGNOSTIC_DATA
|
|
24
26
|
from rasa.nlu.extractors.extractor import EntityTagSpec
|
|
25
27
|
from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
|
|
26
28
|
from rasa.utils import train_utils
|
|
27
29
|
from rasa.utils.tensorflow import rasa_layers
|
|
28
|
-
from rasa.utils.tensorflow.feature_array import (
|
|
29
|
-
FeatureArray,
|
|
30
|
-
serialize_nested_feature_arrays,
|
|
31
|
-
deserialize_nested_feature_arrays,
|
|
32
|
-
)
|
|
33
30
|
from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
|
|
34
31
|
from rasa.utils.tensorflow.model_data import (
|
|
35
32
|
RasaModelData,
|
|
36
33
|
FeatureSignature,
|
|
34
|
+
FeatureArray,
|
|
37
35
|
)
|
|
38
36
|
from rasa.nlu.constants import TOKENS_NAMES, DEFAULT_TRANSFORMER_SIZE
|
|
39
37
|
from rasa.shared.nlu.constants import (
|
|
@@ -120,6 +118,7 @@ LABEL_SUB_KEY = IDS
|
|
|
120
118
|
|
|
121
119
|
POSSIBLE_TAGS = [ENTITY_ATTRIBUTE_TYPE, ENTITY_ATTRIBUTE_ROLE, ENTITY_ATTRIBUTE_GROUP]
|
|
122
120
|
|
|
121
|
+
|
|
123
122
|
DIETClassifierT = TypeVar("DIETClassifierT", bound="DIETClassifier")
|
|
124
123
|
|
|
125
124
|
|
|
@@ -1084,24 +1083,18 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
|
|
|
1084
1083
|
|
|
1085
1084
|
self.model.save(str(tf_model_file))
|
|
1086
1085
|
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
self._data_example,
|
|
1090
|
-
model_path / f"{file_name}.data_example.st",
|
|
1091
|
-
model_path / f"{file_name}.data_example_metadata.json",
|
|
1092
|
-
)
|
|
1093
|
-
# save label data
|
|
1094
|
-
serialize_nested_feature_arrays(
|
|
1095
|
-
dict(self._label_data.data) if self._label_data is not None else {},
|
|
1096
|
-
model_path / f"{file_name}.label_data.st",
|
|
1097
|
-
model_path / f"{file_name}.label_data_metadata.json",
|
|
1086
|
+
io_utils.pickle_dump(
|
|
1087
|
+
model_path / f"{file_name}.data_example.pkl", self._data_example
|
|
1098
1088
|
)
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
model_path / f"{file_name}.sparse_feature_sizes.json",
|
|
1089
|
+
io_utils.pickle_dump(
|
|
1090
|
+
model_path / f"{file_name}.sparse_feature_sizes.pkl",
|
|
1102
1091
|
self._sparse_feature_sizes,
|
|
1103
1092
|
)
|
|
1104
|
-
|
|
1093
|
+
io_utils.pickle_dump(
|
|
1094
|
+
model_path / f"{file_name}.label_data.pkl",
|
|
1095
|
+
dict(self._label_data.data) if self._label_data is not None else {},
|
|
1096
|
+
)
|
|
1097
|
+
io_utils.json_pickle(
|
|
1105
1098
|
model_path / f"{file_name}.index_label_id_mapping.json",
|
|
1106
1099
|
self.index_label_id_mapping,
|
|
1107
1100
|
)
|
|
@@ -1190,22 +1183,15 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
|
|
|
1190
1183
|
]:
|
|
1191
1184
|
file_name = cls.__name__
|
|
1192
1185
|
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
str(model_path / f"{file_name}.data_example.st"),
|
|
1196
|
-
str(model_path / f"{file_name}.data_example_metadata.json"),
|
|
1186
|
+
data_example = io_utils.pickle_load(
|
|
1187
|
+
model_path / f"{file_name}.data_example.pkl"
|
|
1197
1188
|
)
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
)
|
|
1203
|
-
label_data = RasaModelData(data=loaded_label_data)
|
|
1204
|
-
|
|
1205
|
-
sparse_feature_sizes = rasa.shared.utils.io.read_json_file(
|
|
1206
|
-
model_path / f"{file_name}.sparse_feature_sizes.json"
|
|
1189
|
+
label_data = io_utils.pickle_load(model_path / f"{file_name}.label_data.pkl")
|
|
1190
|
+
label_data = RasaModelData(data=label_data)
|
|
1191
|
+
sparse_feature_sizes = io_utils.pickle_load(
|
|
1192
|
+
model_path / f"{file_name}.sparse_feature_sizes.pkl"
|
|
1207
1193
|
)
|
|
1208
|
-
index_label_id_mapping =
|
|
1194
|
+
index_label_id_mapping = io_utils.json_unpickle(
|
|
1209
1195
|
model_path / f"{file_name}.index_label_id_mapping.json"
|
|
1210
1196
|
)
|
|
1211
1197
|
entity_tag_specs = rasa.shared.utils.io.read_json_file(
|
|
@@ -1225,6 +1211,7 @@ class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
|
|
|
1225
1211
|
for tag_spec in entity_tag_specs
|
|
1226
1212
|
]
|
|
1227
1213
|
|
|
1214
|
+
# jsonpickle converts dictionary keys to strings
|
|
1228
1215
|
index_label_id_mapping = {
|
|
1229
1216
|
int(key): value for key, value in index_label_id_mapping.items()
|
|
1230
1217
|
}
|
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
from typing import Any, Text, Dict, List, Type, Tuple
|
|
2
2
|
|
|
3
|
+
import joblib
|
|
3
4
|
import structlog
|
|
4
5
|
from scipy.sparse import hstack, vstack, csr_matrix
|
|
5
6
|
from sklearn.exceptions import NotFittedError
|
|
6
7
|
from sklearn.linear_model import LogisticRegression
|
|
7
8
|
from sklearn.utils.validation import check_is_fitted
|
|
8
9
|
|
|
9
|
-
from rasa.engine.graph import ExecutionContext, GraphComponent
|
|
10
|
-
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
11
10
|
from rasa.engine.storage.resource import Resource
|
|
12
11
|
from rasa.engine.storage.storage import ModelStorage
|
|
12
|
+
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
13
|
+
from rasa.engine.graph import ExecutionContext, GraphComponent
|
|
13
14
|
from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
|
|
14
|
-
from rasa.nlu.classifiers.classifier import IntentClassifier
|
|
15
15
|
from rasa.nlu.featurizers.featurizer import Featurizer
|
|
16
|
-
from rasa.
|
|
17
|
-
from rasa.shared.nlu.training_data.message import Message
|
|
16
|
+
from rasa.nlu.classifiers.classifier import IntentClassifier
|
|
18
17
|
from rasa.shared.nlu.training_data.training_data import TrainingData
|
|
18
|
+
from rasa.shared.nlu.training_data.message import Message
|
|
19
|
+
from rasa.shared.nlu.constants import TEXT, INTENT
|
|
19
20
|
from rasa.utils.tensorflow.constants import RANKING_LENGTH
|
|
20
21
|
|
|
21
22
|
structlogger = structlog.get_logger()
|
|
@@ -183,11 +184,9 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
|
|
|
183
184
|
|
|
184
185
|
def persist(self) -> None:
|
|
185
186
|
"""Persist this model into the passed directory."""
|
|
186
|
-
import skops.io as sio
|
|
187
|
-
|
|
188
187
|
with self._model_storage.write_to(self._resource) as model_dir:
|
|
189
|
-
path = model_dir / f"{self._resource.name}.
|
|
190
|
-
|
|
188
|
+
path = model_dir / f"{self._resource.name}.joblib"
|
|
189
|
+
joblib.dump(self.clf, path)
|
|
191
190
|
structlogger.debug(
|
|
192
191
|
"logistic_regression_classifier.persist",
|
|
193
192
|
event_info=f"Saved intent classifier to '{path}'.",
|
|
@@ -203,21 +202,9 @@ class LogisticRegressionClassifier(IntentClassifier, GraphComponent):
|
|
|
203
202
|
**kwargs: Any,
|
|
204
203
|
) -> "LogisticRegressionClassifier":
|
|
205
204
|
"""Loads trained component (see parent class for full docstring)."""
|
|
206
|
-
import skops.io as sio
|
|
207
|
-
|
|
208
205
|
try:
|
|
209
206
|
with model_storage.read_from(resource) as model_dir:
|
|
210
|
-
|
|
211
|
-
unknown_types = sio.get_untrusted_types(file=classifier_file)
|
|
212
|
-
|
|
213
|
-
if unknown_types:
|
|
214
|
-
structlogger.error(
|
|
215
|
-
f"Untrusted types found when loading {classifier_file}!",
|
|
216
|
-
unknown_types=unknown_types,
|
|
217
|
-
)
|
|
218
|
-
raise ValueError()
|
|
219
|
-
|
|
220
|
-
classifier = sio.load(classifier_file, trusted=unknown_types)
|
|
207
|
+
classifier = joblib.load(model_dir / f"{resource.name}.joblib")
|
|
221
208
|
component = cls(
|
|
222
209
|
config, execution_context.node_name, model_storage, resource
|
|
223
210
|
)
|