rasa-pro 3.10.10__py3-none-any.whl → 3.10.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rasa-pro might be problematic. Click here for more details.
- README.md +17 -396
- rasa/cli/arguments/train.py +9 -3
- rasa/cli/train.py +40 -2
- rasa/cli/utils.py +7 -5
- rasa/constants.py +1 -1
- rasa/core/featurizers/single_state_featurizer.py +22 -1
- rasa/core/featurizers/tracker_featurizers.py +115 -18
- rasa/core/policies/ted_policy.py +58 -33
- rasa/core/policies/unexpected_intent_policy.py +15 -7
- rasa/dialogue_understanding/commands/change_flow_command.py +6 -0
- rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py +20 -3
- rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py +29 -4
- rasa/e2e_test/e2e_test_runner.py +2 -2
- rasa/engine/storage/local_model_storage.py +41 -12
- rasa/model_training.py +10 -3
- rasa/nlu/classifiers/diet_classifier.py +38 -25
- rasa/nlu/classifiers/logistic_regression_classifier.py +22 -9
- rasa/nlu/classifiers/sklearn_intent_classifier.py +37 -16
- rasa/nlu/extractors/crf_entity_extractor.py +93 -50
- rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py +45 -16
- rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py +52 -17
- rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py +5 -3
- rasa/nlu/persistor.py +37 -15
- rasa/shared/constants.py +4 -1
- rasa/shared/importers/importer.py +7 -8
- rasa/shared/nlu/training_data/features.py +120 -2
- rasa/shared/utils/io.py +1 -0
- rasa/utils/io.py +0 -66
- rasa/utils/tensorflow/feature_array.py +366 -0
- rasa/utils/tensorflow/model_data.py +2 -193
- rasa/version.py +1 -1
- rasa_pro-3.10.12.dist-info/METADATA +196 -0
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.12.dist-info}/RECORD +36 -36
- rasa/shared/importers/remote_importer.py +0 -196
- rasa_pro-3.10.10.dist-info/METADATA +0 -575
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.12.dist-info}/NOTICE +0 -0
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.12.dist-info}/WHEEL +0 -0
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.12.dist-info}/entry_points.txt +0 -0
|
@@ -4,9 +4,9 @@ from collections import OrderedDict
|
|
|
4
4
|
from enum import Enum
|
|
5
5
|
import logging
|
|
6
6
|
import typing
|
|
7
|
+
from typing import Any, Dict, List, Optional, Text, Tuple, Callable, Type
|
|
7
8
|
|
|
8
9
|
import numpy as np
|
|
9
|
-
from typing import Any, Dict, List, Optional, Text, Tuple, Callable, Type
|
|
10
10
|
|
|
11
11
|
import rasa.nlu.utils.bilou_utils as bilou_utils
|
|
12
12
|
import rasa.shared.utils.io
|
|
@@ -41,6 +41,9 @@ if typing.TYPE_CHECKING:
|
|
|
41
41
|
from sklearn_crfsuite import CRF
|
|
42
42
|
|
|
43
43
|
|
|
44
|
+
CONFIG_FEATURES = "features"
|
|
45
|
+
|
|
46
|
+
|
|
44
47
|
class CRFToken:
|
|
45
48
|
def __init__(
|
|
46
49
|
self,
|
|
@@ -60,6 +63,29 @@ class CRFToken:
|
|
|
60
63
|
self.entity_role_tag = entity_role_tag
|
|
61
64
|
self.entity_group_tag = entity_group_tag
|
|
62
65
|
|
|
66
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
67
|
+
return {
|
|
68
|
+
"text": self.text,
|
|
69
|
+
"pos_tag": self.pos_tag,
|
|
70
|
+
"pattern": self.pattern,
|
|
71
|
+
"dense_features": [str(x) for x in list(self.dense_features)],
|
|
72
|
+
"entity_tag": self.entity_tag,
|
|
73
|
+
"entity_role_tag": self.entity_role_tag,
|
|
74
|
+
"entity_group_tag": self.entity_group_tag,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
@classmethod
|
|
78
|
+
def create_from_dict(cls, data: Dict[str, Any]) -> "CRFToken":
|
|
79
|
+
return cls(
|
|
80
|
+
data["text"],
|
|
81
|
+
data["pos_tag"],
|
|
82
|
+
data["pattern"],
|
|
83
|
+
np.array([float(x) for x in data["dense_features"]]),
|
|
84
|
+
data["entity_tag"],
|
|
85
|
+
data["entity_role_tag"],
|
|
86
|
+
data["entity_group_tag"],
|
|
87
|
+
)
|
|
88
|
+
|
|
63
89
|
|
|
64
90
|
class CRFEntityExtractorOptions(str, Enum):
|
|
65
91
|
"""Features that can be used for the 'CRFEntityExtractor'."""
|
|
@@ -88,8 +114,6 @@ class CRFEntityExtractorOptions(str, Enum):
|
|
|
88
114
|
class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
89
115
|
"""Implements conditional random fields (CRF) to do named entity recognition."""
|
|
90
116
|
|
|
91
|
-
CONFIG_FEATURES = "features"
|
|
92
|
-
|
|
93
117
|
function_dict: Dict[Text, Callable[[CRFToken], Any]] = { # noqa: RUF012
|
|
94
118
|
CRFEntityExtractorOptions.LOW: lambda crf_token: crf_token.text.lower(),
|
|
95
119
|
CRFEntityExtractorOptions.TITLE: lambda crf_token: crf_token.text.istitle(),
|
|
@@ -137,7 +161,7 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
137
161
|
# "is the preceding token in title case?"
|
|
138
162
|
# POS features require SpacyTokenizer
|
|
139
163
|
# pattern feature require RegexFeaturizer
|
|
140
|
-
|
|
164
|
+
CONFIG_FEATURES: [
|
|
141
165
|
[
|
|
142
166
|
CRFEntityExtractorOptions.LOW,
|
|
143
167
|
CRFEntityExtractorOptions.TITLE,
|
|
@@ -200,7 +224,7 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
200
224
|
)
|
|
201
225
|
|
|
202
226
|
def _validate_configuration(self) -> None:
|
|
203
|
-
if len(self.component_config.get(
|
|
227
|
+
if len(self.component_config.get(CONFIG_FEATURES, [])) % 2 != 1:
|
|
204
228
|
raise ValueError(
|
|
205
229
|
"Need an odd number of crf feature lists to have a center word."
|
|
206
230
|
)
|
|
@@ -251,9 +275,11 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
251
275
|
]
|
|
252
276
|
dataset = [self._convert_to_crf_tokens(example) for example in entity_examples]
|
|
253
277
|
|
|
254
|
-
self.
|
|
278
|
+
self.entity_taggers = self.train_model(
|
|
279
|
+
dataset, self.component_config, self.crf_order
|
|
280
|
+
)
|
|
255
281
|
|
|
256
|
-
self.persist()
|
|
282
|
+
self.persist(dataset)
|
|
257
283
|
|
|
258
284
|
return self._resource
|
|
259
285
|
|
|
@@ -299,7 +325,9 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
299
325
|
if include_tag_features:
|
|
300
326
|
self._add_tag_to_crf_token(crf_tokens, predictions)
|
|
301
327
|
|
|
302
|
-
features = self._crf_tokens_to_features(
|
|
328
|
+
features = self._crf_tokens_to_features(
|
|
329
|
+
crf_tokens, self.component_config, include_tag_features
|
|
330
|
+
)
|
|
303
331
|
predictions[tag_name] = entity_tagger.predict_marginals_single(features)
|
|
304
332
|
|
|
305
333
|
# convert predictions into a list of tags and a list of confidences
|
|
@@ -389,27 +417,25 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
389
417
|
**kwargs: Any,
|
|
390
418
|
) -> CRFEntityExtractor:
|
|
391
419
|
"""Loads trained component (see parent class for full docstring)."""
|
|
392
|
-
import joblib
|
|
393
|
-
|
|
394
420
|
try:
|
|
395
|
-
entity_taggers = OrderedDict()
|
|
396
421
|
with model_storage.read_from(resource) as model_dir:
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
"Maybe you did not provide enough training data and "
|
|
404
|
-
"no model was trained."
|
|
405
|
-
)
|
|
406
|
-
return cls(config, model_storage, resource)
|
|
422
|
+
dataset = rasa.shared.utils.io.read_json_file(
|
|
423
|
+
model_dir / "crf_dataset.json"
|
|
424
|
+
)
|
|
425
|
+
crf_order = rasa.shared.utils.io.read_json_file(
|
|
426
|
+
model_dir / "crf_order.json"
|
|
427
|
+
)
|
|
407
428
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
429
|
+
dataset = [
|
|
430
|
+
[CRFToken.create_from_dict(token_data) for token_data in sub_list]
|
|
431
|
+
for sub_list in dataset
|
|
432
|
+
]
|
|
433
|
+
|
|
434
|
+
entity_taggers = cls.train_model(dataset, config, crf_order)
|
|
411
435
|
|
|
412
|
-
|
|
436
|
+
entity_extractor = cls(config, model_storage, resource, entity_taggers)
|
|
437
|
+
entity_extractor.crf_order = crf_order
|
|
438
|
+
return entity_extractor
|
|
413
439
|
except ValueError:
|
|
414
440
|
logger.warning(
|
|
415
441
|
f"Failed to load {cls.__name__} from model storage. Resource "
|
|
@@ -417,23 +443,29 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
417
443
|
)
|
|
418
444
|
return cls(config, model_storage, resource)
|
|
419
445
|
|
|
420
|
-
def persist(self) -> None:
|
|
446
|
+
def persist(self, dataset: List[List[CRFToken]]) -> None:
|
|
421
447
|
"""Persist this model into the passed directory."""
|
|
422
|
-
import joblib
|
|
423
|
-
|
|
424
448
|
with self._model_storage.write_to(self._resource) as model_dir:
|
|
425
|
-
|
|
426
|
-
for
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
449
|
+
data_to_store = [
|
|
450
|
+
[token.to_dict() for token in sub_list] for sub_list in dataset
|
|
451
|
+
]
|
|
452
|
+
|
|
453
|
+
rasa.shared.utils.io.dump_obj_as_json_to_file(
|
|
454
|
+
model_dir / "crf_dataset.json", data_to_store
|
|
455
|
+
)
|
|
456
|
+
rasa.shared.utils.io.dump_obj_as_json_to_file(
|
|
457
|
+
model_dir / "crf_order.json", self.crf_order
|
|
458
|
+
)
|
|
431
459
|
|
|
460
|
+
@classmethod
|
|
432
461
|
def _crf_tokens_to_features(
|
|
433
|
-
|
|
462
|
+
cls,
|
|
463
|
+
crf_tokens: List[CRFToken],
|
|
464
|
+
config: Dict[str, Any],
|
|
465
|
+
include_tag_features: bool = False,
|
|
434
466
|
) -> List[Dict[Text, Any]]:
|
|
435
467
|
"""Convert the list of tokens into discrete features."""
|
|
436
|
-
configured_features =
|
|
468
|
+
configured_features = config[CONFIG_FEATURES]
|
|
437
469
|
sentence_features = []
|
|
438
470
|
|
|
439
471
|
for token_idx in range(len(crf_tokens)):
|
|
@@ -444,28 +476,31 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
444
476
|
half_window_size = window_size // 2
|
|
445
477
|
window_range = range(-half_window_size, half_window_size + 1)
|
|
446
478
|
|
|
447
|
-
token_features =
|
|
479
|
+
token_features = cls._create_features_for_token(
|
|
448
480
|
crf_tokens,
|
|
449
481
|
token_idx,
|
|
450
482
|
half_window_size,
|
|
451
483
|
window_range,
|
|
452
484
|
include_tag_features,
|
|
485
|
+
config,
|
|
453
486
|
)
|
|
454
487
|
|
|
455
488
|
sentence_features.append(token_features)
|
|
456
489
|
|
|
457
490
|
return sentence_features
|
|
458
491
|
|
|
492
|
+
@classmethod
|
|
459
493
|
def _create_features_for_token(
|
|
460
|
-
|
|
494
|
+
cls,
|
|
461
495
|
crf_tokens: List[CRFToken],
|
|
462
496
|
token_idx: int,
|
|
463
497
|
half_window_size: int,
|
|
464
498
|
window_range: range,
|
|
465
499
|
include_tag_features: bool,
|
|
500
|
+
config: Dict[str, Any],
|
|
466
501
|
) -> Dict[Text, Any]:
|
|
467
502
|
"""Convert a token into discrete features including words before and after."""
|
|
468
|
-
configured_features =
|
|
503
|
+
configured_features = config[CONFIG_FEATURES]
|
|
469
504
|
prefixes = [str(i) for i in window_range]
|
|
470
505
|
|
|
471
506
|
token_features = {}
|
|
@@ -505,13 +540,13 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
505
540
|
# set in the training data, 'matched' is either 'True' or
|
|
506
541
|
# 'False' depending on whether the token actually matches the
|
|
507
542
|
# pattern or not
|
|
508
|
-
regex_patterns =
|
|
543
|
+
regex_patterns = cls.function_dict[feature](token)
|
|
509
544
|
for pattern_name, matched in regex_patterns.items():
|
|
510
545
|
token_features[f"{prefix}:{feature}:{pattern_name}"] = (
|
|
511
546
|
matched
|
|
512
547
|
)
|
|
513
548
|
else:
|
|
514
|
-
value =
|
|
549
|
+
value = cls.function_dict[feature](token)
|
|
515
550
|
token_features[f"{prefix}:{feature}"] = value
|
|
516
551
|
|
|
517
552
|
return token_features
|
|
@@ -635,38 +670,46 @@ class CRFEntityExtractor(GraphComponent, EntityExtractorMixin):
|
|
|
635
670
|
|
|
636
671
|
return tags
|
|
637
672
|
|
|
638
|
-
|
|
673
|
+
@classmethod
|
|
674
|
+
def train_model(
|
|
675
|
+
cls,
|
|
676
|
+
df_train: List[List[CRFToken]],
|
|
677
|
+
config: Dict[str, Any],
|
|
678
|
+
crf_order: List[str],
|
|
679
|
+
) -> OrderedDict[str, CRF]:
|
|
639
680
|
"""Train the crf tagger based on the training data."""
|
|
640
681
|
import sklearn_crfsuite
|
|
641
682
|
|
|
642
|
-
|
|
683
|
+
entity_taggers = OrderedDict()
|
|
643
684
|
|
|
644
|
-
for tag_name in
|
|
685
|
+
for tag_name in crf_order:
|
|
645
686
|
logger.debug(f"Training CRF for '{tag_name}'.")
|
|
646
687
|
|
|
647
688
|
# add entity tag features for second level CRFs
|
|
648
689
|
include_tag_features = tag_name != ENTITY_ATTRIBUTE_TYPE
|
|
649
690
|
X_train = (
|
|
650
|
-
|
|
691
|
+
cls._crf_tokens_to_features(sentence, config, include_tag_features)
|
|
651
692
|
for sentence in df_train
|
|
652
693
|
)
|
|
653
694
|
y_train = (
|
|
654
|
-
|
|
695
|
+
cls._crf_tokens_to_tags(sentence, tag_name) for sentence in df_train
|
|
655
696
|
)
|
|
656
697
|
|
|
657
698
|
entity_tagger = sklearn_crfsuite.CRF(
|
|
658
699
|
algorithm="lbfgs",
|
|
659
700
|
# coefficient for L1 penalty
|
|
660
|
-
c1=
|
|
701
|
+
c1=config["L1_c"],
|
|
661
702
|
# coefficient for L2 penalty
|
|
662
|
-
c2=
|
|
703
|
+
c2=config["L2_c"],
|
|
663
704
|
# stop earlier
|
|
664
|
-
max_iterations=
|
|
705
|
+
max_iterations=config["max_iterations"],
|
|
665
706
|
# include transitions that are possible, but not observed
|
|
666
707
|
all_possible_transitions=True,
|
|
667
708
|
)
|
|
668
709
|
entity_tagger.fit(X_train, y_train)
|
|
669
710
|
|
|
670
|
-
|
|
711
|
+
entity_taggers[tag_name] = entity_tagger
|
|
671
712
|
|
|
672
713
|
logger.debug("Training finished.")
|
|
714
|
+
|
|
715
|
+
return entity_taggers
|
|
@@ -1,30 +1,32 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
|
|
2
3
|
import logging
|
|
3
4
|
import re
|
|
5
|
+
from typing import Any, Dict, List, Optional, Text, Tuple, Set, Type, Union
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
4
8
|
import scipy.sparse
|
|
5
|
-
from
|
|
6
|
-
from
|
|
9
|
+
from sklearn.exceptions import NotFittedError
|
|
10
|
+
from sklearn.feature_extraction.text import CountVectorizer
|
|
7
11
|
|
|
8
12
|
import rasa.shared.utils.io
|
|
9
13
|
from rasa.engine.graph import GraphComponent, ExecutionContext
|
|
10
14
|
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
11
15
|
from rasa.engine.storage.resource import Resource
|
|
12
16
|
from rasa.engine.storage.storage import ModelStorage
|
|
13
|
-
from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer
|
|
14
|
-
from rasa.nlu.utils.spacy_utils import SpacyModel
|
|
15
|
-
from rasa.shared.constants import DOCS_URL_COMPONENTS
|
|
16
|
-
import rasa.utils.io as io_utils
|
|
17
|
-
from sklearn.exceptions import NotFittedError
|
|
18
|
-
from sklearn.feature_extraction.text import CountVectorizer
|
|
19
|
-
from rasa.shared.nlu.training_data.training_data import TrainingData
|
|
20
|
-
from rasa.shared.nlu.training_data.message import Message
|
|
21
|
-
from rasa.shared.exceptions import RasaException, FileIOException
|
|
22
17
|
from rasa.nlu.constants import (
|
|
23
18
|
TOKENS_NAMES,
|
|
24
19
|
MESSAGE_ATTRIBUTES,
|
|
25
20
|
DENSE_FEATURIZABLE_ATTRIBUTES,
|
|
26
21
|
)
|
|
22
|
+
from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer
|
|
23
|
+
from rasa.nlu.tokenizers.tokenizer import Tokenizer
|
|
24
|
+
from rasa.nlu.utils.spacy_utils import SpacyModel
|
|
25
|
+
from rasa.shared.constants import DOCS_URL_COMPONENTS
|
|
26
|
+
from rasa.shared.exceptions import RasaException, FileIOException
|
|
27
27
|
from rasa.shared.nlu.constants import TEXT, INTENT, INTENT_RESPONSE_KEY, ACTION_NAME
|
|
28
|
+
from rasa.shared.nlu.training_data.message import Message
|
|
29
|
+
from rasa.shared.nlu.training_data.training_data import TrainingData
|
|
28
30
|
|
|
29
31
|
BUFFER_SLOTS_PREFIX = "buf_"
|
|
30
32
|
|
|
@@ -688,6 +690,31 @@ class CountVectorsFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
688
690
|
"""Check if any model got trained."""
|
|
689
691
|
return any(value is not None for value in attribute_vocabularies.values())
|
|
690
692
|
|
|
693
|
+
@staticmethod
|
|
694
|
+
def convert_vocab(
|
|
695
|
+
vocab: Dict[str, Union[int, Optional[Dict[str, int]]]], to_int: bool
|
|
696
|
+
) -> Dict[str, Union[None, int, np.int64, Dict[str, Union[int, np.int64]]]]:
|
|
697
|
+
"""Converts numpy integers in the vocabulary to Python integers."""
|
|
698
|
+
|
|
699
|
+
def convert_value(value: int) -> Union[int, np.int64]:
|
|
700
|
+
"""Helper function to convert a single value based on to_int flag."""
|
|
701
|
+
return int(value) if to_int else np.int64(value)
|
|
702
|
+
|
|
703
|
+
result_dict: Dict[
|
|
704
|
+
str, Union[None, int, np.int64, Dict[str, Union[int, np.int64]]]
|
|
705
|
+
] = {}
|
|
706
|
+
for key, sub_dict in vocab.items():
|
|
707
|
+
if isinstance(sub_dict, int):
|
|
708
|
+
result_dict[key] = convert_value(sub_dict)
|
|
709
|
+
elif not sub_dict:
|
|
710
|
+
result_dict[key] = None
|
|
711
|
+
else:
|
|
712
|
+
result_dict[key] = {
|
|
713
|
+
sub_key: convert_value(value) for sub_key, value in sub_dict.items()
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
return result_dict
|
|
717
|
+
|
|
691
718
|
def persist(self) -> None:
|
|
692
719
|
"""Persist this model into the passed directory.
|
|
693
720
|
|
|
@@ -701,17 +728,18 @@ class CountVectorsFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
701
728
|
attribute_vocabularies = self._collect_vectorizer_vocabularies()
|
|
702
729
|
if self._is_any_model_trained(attribute_vocabularies):
|
|
703
730
|
# Definitely need to persist some vocabularies
|
|
704
|
-
featurizer_file = model_dir / "vocabularies.
|
|
731
|
+
featurizer_file = model_dir / "vocabularies.json"
|
|
705
732
|
|
|
706
733
|
# Only persist vocabulary from one attribute if `use_shared_vocab`.
|
|
707
734
|
# Can be loaded and distributed to all attributes.
|
|
708
|
-
|
|
735
|
+
loaded_vocab = (
|
|
709
736
|
attribute_vocabularies[TEXT]
|
|
710
737
|
if self.use_shared_vocab
|
|
711
738
|
else attribute_vocabularies
|
|
712
739
|
)
|
|
740
|
+
vocab = self.convert_vocab(loaded_vocab, to_int=True)
|
|
713
741
|
|
|
714
|
-
|
|
742
|
+
rasa.shared.utils.io.dump_obj_as_json_to_file(featurizer_file, vocab)
|
|
715
743
|
|
|
716
744
|
# Dump OOV words separately as they might have been modified during
|
|
717
745
|
# training
|
|
@@ -786,8 +814,9 @@ class CountVectorsFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
786
814
|
"""Loads trained component (see parent class for full docstring)."""
|
|
787
815
|
try:
|
|
788
816
|
with model_storage.read_from(resource) as model_dir:
|
|
789
|
-
featurizer_file = model_dir / "vocabularies.
|
|
790
|
-
vocabulary =
|
|
817
|
+
featurizer_file = model_dir / "vocabularies.json"
|
|
818
|
+
vocabulary = rasa.shared.utils.io.read_json_file(featurizer_file)
|
|
819
|
+
vocabulary = cls.convert_vocab(vocabulary, to_int=False)
|
|
791
820
|
|
|
792
821
|
share_vocabulary = config["use_shared_vocab"]
|
|
793
822
|
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
|
|
2
3
|
import logging
|
|
3
4
|
from collections import OrderedDict
|
|
4
|
-
|
|
5
|
-
import scipy.sparse
|
|
6
|
-
import numpy as np
|
|
7
5
|
from typing import (
|
|
8
6
|
Any,
|
|
9
7
|
Dict,
|
|
@@ -17,30 +15,34 @@ from typing import (
|
|
|
17
15
|
Union,
|
|
18
16
|
)
|
|
19
17
|
|
|
18
|
+
import numpy as np
|
|
19
|
+
import scipy.sparse
|
|
20
|
+
|
|
21
|
+
import rasa.shared.utils.io
|
|
22
|
+
import rasa.utils.io
|
|
20
23
|
from rasa.engine.graph import ExecutionContext, GraphComponent
|
|
21
24
|
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
22
25
|
from rasa.engine.storage.resource import Resource
|
|
23
26
|
from rasa.engine.storage.storage import ModelStorage
|
|
27
|
+
from rasa.nlu.constants import TOKENS_NAMES
|
|
28
|
+
from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer
|
|
24
29
|
from rasa.nlu.tokenizers.spacy_tokenizer import POS_TAG_KEY, SpacyTokenizer
|
|
25
30
|
from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
|
|
26
|
-
from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer
|
|
27
|
-
from rasa.nlu.constants import TOKENS_NAMES
|
|
28
31
|
from rasa.shared.constants import DOCS_URL_COMPONENTS
|
|
29
|
-
from rasa.shared.nlu.training_data.training_data import TrainingData
|
|
30
|
-
from rasa.shared.nlu.training_data.message import Message
|
|
31
|
-
from rasa.shared.nlu.constants import TEXT
|
|
32
32
|
from rasa.shared.exceptions import InvalidConfigException
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
from rasa.shared.nlu.constants import TEXT
|
|
34
|
+
from rasa.shared.nlu.training_data.message import Message
|
|
35
|
+
from rasa.shared.nlu.training_data.training_data import TrainingData
|
|
35
36
|
|
|
36
37
|
logger = logging.getLogger(__name__)
|
|
37
38
|
|
|
38
|
-
|
|
39
39
|
END_OF_SENTENCE = "EOS"
|
|
40
40
|
BEGIN_OF_SENTENCE = "BOS"
|
|
41
41
|
|
|
42
42
|
FEATURES = "features"
|
|
43
43
|
|
|
44
|
+
SEPERATOR = "###"
|
|
45
|
+
|
|
44
46
|
|
|
45
47
|
@DefaultV1Recipe.register(
|
|
46
48
|
DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=True
|
|
@@ -72,7 +74,7 @@ class LexicalSyntacticFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
72
74
|
of the token at position `t+1`.
|
|
73
75
|
"""
|
|
74
76
|
|
|
75
|
-
FILENAME_FEATURE_TO_IDX_DICT = "feature_to_idx_dict.
|
|
77
|
+
FILENAME_FEATURE_TO_IDX_DICT = "feature_to_idx_dict.json"
|
|
76
78
|
|
|
77
79
|
# NOTE: "suffix5" of the token "is" will be "is". Hence, when combining multiple
|
|
78
80
|
# prefixes, short words will be represented/encoded repeatedly.
|
|
@@ -488,6 +490,32 @@ class LexicalSyntacticFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
488
490
|
"""Creates a new untrained component (see parent class for full docstring)."""
|
|
489
491
|
return cls(config, model_storage, resource, execution_context)
|
|
490
492
|
|
|
493
|
+
@staticmethod
|
|
494
|
+
def _restructure_feature_to_idx_dict(
|
|
495
|
+
loaded_data: Dict[str, Dict[str, int]],
|
|
496
|
+
) -> Dict[Tuple[int, str], Dict[str, int]]:
|
|
497
|
+
"""Reconstructs the feature to idx dict.
|
|
498
|
+
|
|
499
|
+
When storing the feature_to_idx_dict to disk, we need to convert the tuple (key)
|
|
500
|
+
into a string to be able to store it via json. When loading the data
|
|
501
|
+
we need to reconstruct the tuple from the stored string.
|
|
502
|
+
|
|
503
|
+
Args:
|
|
504
|
+
loaded_data: The loaded feature to idx dict from file.
|
|
505
|
+
|
|
506
|
+
Returns:
|
|
507
|
+
The reconstructed feature_to_idx_dict
|
|
508
|
+
"""
|
|
509
|
+
feature_to_idx_dict = {}
|
|
510
|
+
for tuple_string, feature_value in loaded_data.items():
|
|
511
|
+
# Example of tuple_string: "1###low"
|
|
512
|
+
index, feature_name = tuple_string.split(SEPERATOR)
|
|
513
|
+
|
|
514
|
+
feature_key = (int(index), feature_name)
|
|
515
|
+
feature_to_idx_dict[feature_key] = feature_value
|
|
516
|
+
|
|
517
|
+
return feature_to_idx_dict
|
|
518
|
+
|
|
491
519
|
@classmethod
|
|
492
520
|
def load(
|
|
493
521
|
cls,
|
|
@@ -500,10 +528,13 @@ class LexicalSyntacticFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
500
528
|
"""Loads trained component (see parent class for full docstring)."""
|
|
501
529
|
try:
|
|
502
530
|
with model_storage.read_from(resource) as model_path:
|
|
503
|
-
|
|
531
|
+
loaded_data = rasa.shared.utils.io.read_json_file(
|
|
504
532
|
model_path / cls.FILENAME_FEATURE_TO_IDX_DICT,
|
|
505
|
-
encode_non_string_keys=True,
|
|
506
533
|
)
|
|
534
|
+
|
|
535
|
+
# convert the key back into tuple
|
|
536
|
+
feature_to_idx_dict = cls._restructure_feature_to_idx_dict(loaded_data)
|
|
537
|
+
|
|
507
538
|
return cls(
|
|
508
539
|
config=config,
|
|
509
540
|
model_storage=model_storage,
|
|
@@ -528,9 +559,13 @@ class LexicalSyntacticFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
528
559
|
if not self._feature_to_idx_dict:
|
|
529
560
|
return None
|
|
530
561
|
|
|
562
|
+
# as we cannot dump tuples, convert the tuple into a string
|
|
563
|
+
restructured_feature_dict = {
|
|
564
|
+
f"{k[0]}{SEPERATOR}{k[1]}": v for k, v in self._feature_to_idx_dict.items()
|
|
565
|
+
}
|
|
566
|
+
|
|
531
567
|
with self._model_storage.write_to(self._resource) as model_path:
|
|
532
|
-
rasa.utils.io.
|
|
568
|
+
rasa.shared.utils.io.dump_obj_as_json_to_file(
|
|
533
569
|
model_path / self.FILENAME_FEATURE_TO_IDX_DICT,
|
|
534
|
-
|
|
535
|
-
encode_non_string_keys=True,
|
|
570
|
+
restructured_feature_dict,
|
|
536
571
|
)
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
|
|
2
3
|
import logging
|
|
3
4
|
import re
|
|
4
5
|
from typing import Any, Dict, List, Optional, Text, Tuple, Type
|
|
6
|
+
|
|
5
7
|
import numpy as np
|
|
6
8
|
import scipy.sparse
|
|
7
|
-
from rasa.nlu.tokenizers.tokenizer import Tokenizer
|
|
8
9
|
|
|
10
|
+
from rasa.nlu.tokenizers.tokenizer import Tokenizer
|
|
9
11
|
import rasa.shared.utils.io
|
|
10
12
|
import rasa.utils.io
|
|
11
13
|
import rasa.nlu.utils.pattern_utils as pattern_utils
|
|
@@ -240,7 +242,7 @@ class RegexFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
240
242
|
|
|
241
243
|
try:
|
|
242
244
|
with model_storage.read_from(resource) as model_dir:
|
|
243
|
-
patterns_file_name = model_dir / "patterns.
|
|
245
|
+
patterns_file_name = model_dir / "patterns.json"
|
|
244
246
|
known_patterns = rasa.shared.utils.io.read_json_file(patterns_file_name)
|
|
245
247
|
except (ValueError, FileNotFoundError):
|
|
246
248
|
logger.warning(
|
|
@@ -258,7 +260,7 @@ class RegexFeaturizer(SparseFeaturizer, GraphComponent):
|
|
|
258
260
|
|
|
259
261
|
def _persist(self) -> None:
|
|
260
262
|
with self._model_storage.write_to(self._resource) as model_dir:
|
|
261
|
-
regex_file = model_dir / "patterns.
|
|
263
|
+
regex_file = model_dir / "patterns.json"
|
|
262
264
|
rasa.shared.utils.io.dump_obj_as_json_to_file(
|
|
263
265
|
regex_file, self.known_patterns
|
|
264
266
|
)
|
rasa/nlu/persistor.py
CHANGED
|
@@ -82,16 +82,14 @@ def get_persistor(storage: StorageType) -> Optional[Persistor]:
|
|
|
82
82
|
Currently, `aws`, `gcs`, `azure` and providing module paths are supported remote
|
|
83
83
|
storages.
|
|
84
84
|
"""
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
if storage == RemoteStorageType.AWS.value:
|
|
85
|
+
if storage == RemoteStorageType.AWS:
|
|
88
86
|
return AWSPersistor(
|
|
89
87
|
os.environ.get(BUCKET_NAME_ENV), os.environ.get(AWS_ENDPOINT_URL_ENV)
|
|
90
88
|
)
|
|
91
|
-
if storage == RemoteStorageType.GCS
|
|
89
|
+
if storage == RemoteStorageType.GCS:
|
|
92
90
|
return GCSPersistor(os.environ.get(BUCKET_NAME_ENV))
|
|
93
91
|
|
|
94
|
-
if storage == RemoteStorageType.AZURE
|
|
92
|
+
if storage == RemoteStorageType.AZURE:
|
|
95
93
|
return AzurePersistor(
|
|
96
94
|
os.environ.get(AZURE_CONTAINER_ENV),
|
|
97
95
|
os.environ.get(AZURE_ACCOUNT_NAME_ENV),
|
|
@@ -125,24 +123,36 @@ class Persistor(abc.ABC):
|
|
|
125
123
|
"""Downloads a model that has been persisted to cloud storage.
|
|
126
124
|
|
|
127
125
|
Downloaded model will be saved to the `target_path`.
|
|
128
|
-
If `target_path` is a directory, the model will be
|
|
129
|
-
If `target_path` is a file, the model will be
|
|
126
|
+
If `target_path` is a directory, the model will be downloaded to that directory.
|
|
127
|
+
If `target_path` is a file, the model will be downloaded to that file.
|
|
130
128
|
|
|
131
129
|
Args:
|
|
132
130
|
model_name: The name of the model to retrieve.
|
|
133
|
-
target_path: The path to which the model should be
|
|
131
|
+
target_path: The path to which the model should be downloaded.
|
|
134
132
|
"""
|
|
135
133
|
tar_name = model_name
|
|
136
134
|
if not model_name.endswith(MODEL_ARCHIVE_EXTENSION):
|
|
137
135
|
# ensure backward compatibility
|
|
138
136
|
tar_name = self._tar_name(model_name)
|
|
139
|
-
|
|
140
|
-
self._retrieve_tar(
|
|
141
|
-
self._copy(os.path.basename(tar_name), target_path)
|
|
137
|
+
remote_object_path = self._create_file_key(tar_name)
|
|
138
|
+
self._retrieve_tar(remote_object_path)
|
|
142
139
|
|
|
140
|
+
target_tar_file_name = os.path.basename(tar_name)
|
|
143
141
|
if os.path.isdir(target_path):
|
|
144
|
-
|
|
142
|
+
target_path = os.path.join(target_path, target_tar_file_name)
|
|
143
|
+
|
|
144
|
+
if not os.path.exists(target_path):
|
|
145
|
+
structlogger.debug(
|
|
146
|
+
"persistor.retrieve.copy_model",
|
|
147
|
+
event_info=f"Copying model '{target_tar_file_name}' to "
|
|
148
|
+
f"'{target_path}'.",
|
|
149
|
+
)
|
|
150
|
+
self._copy(target_tar_file_name, target_path)
|
|
145
151
|
|
|
152
|
+
structlogger.debug(
|
|
153
|
+
"persistor.retrieve.model_retrieved",
|
|
154
|
+
event_info=f"Model retrieved and saved to '{target_path}'.",
|
|
155
|
+
)
|
|
146
156
|
return target_path
|
|
147
157
|
|
|
148
158
|
@abc.abstractmethod
|
|
@@ -262,6 +272,11 @@ class AWSPersistor(Persistor):
|
|
|
262
272
|
|
|
263
273
|
def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
|
|
264
274
|
"""Uploads a model persisted in the `target_dir` to s3."""
|
|
275
|
+
structlogger.debug(
|
|
276
|
+
"aws_persistor.persist_tar.uploading_model",
|
|
277
|
+
event_info=f"Uploading tar archive {file_key} to "
|
|
278
|
+
f"s3 bucket '{self.bucket_name}'.",
|
|
279
|
+
)
|
|
265
280
|
with open(tar_path, "rb") as f:
|
|
266
281
|
self.s3.Object(self.bucket_name, file_key).put(Body=f)
|
|
267
282
|
|
|
@@ -329,7 +344,7 @@ class GCSPersistor(Persistor):
|
|
|
329
344
|
def _retrieve_tar(self, target_filename: Text) -> None:
|
|
330
345
|
"""Downloads a model that has previously been persisted to GCS."""
|
|
331
346
|
blob = self.bucket.blob(target_filename)
|
|
332
|
-
blob.download_to_filename(target_filename)
|
|
347
|
+
blob.download_to_filename(os.path.basename(target_filename))
|
|
333
348
|
|
|
334
349
|
|
|
335
350
|
class AzurePersistor(Persistor):
|
|
@@ -370,12 +385,19 @@ class AzurePersistor(Persistor):
|
|
|
370
385
|
def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
|
|
371
386
|
"""Uploads a model persisted in the `target_dir` to Azure."""
|
|
372
387
|
with open(tar_path, "rb") as data:
|
|
373
|
-
self._container_client().upload_blob(
|
|
388
|
+
self._container_client().upload_blob(
|
|
389
|
+
name=file_key,
|
|
390
|
+
data=data,
|
|
391
|
+
# overwrite is set to True to keep in line with
|
|
392
|
+
# how GCS and AWS APIs work this enables easy
|
|
393
|
+
# updating of models in the cloud
|
|
394
|
+
overwrite=True,
|
|
395
|
+
)
|
|
374
396
|
|
|
375
397
|
def _retrieve_tar(self, target_filename: Text) -> None:
|
|
376
398
|
"""Downloads a model that has previously been persisted to Azure."""
|
|
377
399
|
blob_client = self._container_client().get_blob_client(target_filename)
|
|
378
400
|
|
|
379
|
-
with open(target_filename, "wb") as blob:
|
|
401
|
+
with open(os.path.basename(target_filename), "wb") as blob:
|
|
380
402
|
download_stream = blob_client.download_blob()
|
|
381
403
|
blob.write(download_stream.readall())
|