rasa-pro 3.10.9.dev1__py3-none-any.whl → 3.10.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rasa-pro might be problematic. Click here for more details.
- rasa/cli/arguments/train.py +9 -3
- rasa/cli/train.py +40 -2
- rasa/cli/utils.py +7 -5
- rasa/constants.py +1 -1
- rasa/core/featurizers/single_state_featurizer.py +1 -22
- rasa/core/featurizers/tracker_featurizers.py +18 -115
- rasa/core/policies/ted_policy.py +33 -58
- rasa/core/policies/unexpected_intent_policy.py +7 -15
- rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py +20 -3
- rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py +29 -4
- rasa/e2e_test/e2e_test_runner.py +4 -2
- rasa/engine/storage/local_model_storage.py +41 -12
- rasa/model_training.py +10 -3
- rasa/nlu/classifiers/diet_classifier.py +25 -38
- rasa/nlu/classifiers/logistic_regression_classifier.py +9 -22
- rasa/nlu/classifiers/sklearn_intent_classifier.py +16 -37
- rasa/nlu/extractors/crf_entity_extractor.py +50 -93
- rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py +16 -45
- rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py +17 -52
- rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py +3 -5
- rasa/nlu/persistor.py +37 -15
- rasa/shared/constants.py +4 -1
- rasa/shared/importers/importer.py +7 -8
- rasa/shared/nlu/training_data/features.py +2 -120
- rasa/shared/utils/io.py +0 -1
- rasa/utils/io.py +66 -0
- rasa/utils/tensorflow/model_data.py +193 -2
- rasa/version.py +1 -1
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.11.dist-info}/METADATA +6 -6
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.11.dist-info}/RECORD +33 -35
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.11.dist-info}/WHEEL +1 -1
- rasa/shared/importers/remote_importer.py +0 -196
- rasa/utils/tensorflow/feature_array.py +0 -366
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.11.dist-info}/NOTICE +0 -0
- {rasa_pro-3.10.9.dev1.dist-info → rasa_pro-3.10.11.dist-info}/entry_points.txt +0 -0
rasa/cli/arguments/train.py
CHANGED
|
@@ -3,12 +3,12 @@ from typing import Union
|
|
|
3
3
|
|
|
4
4
|
from rasa.cli.arguments.default_arguments import (
|
|
5
5
|
add_config_param,
|
|
6
|
-
add_stories_param,
|
|
7
|
-
add_nlu_data_param,
|
|
8
|
-
add_out_param,
|
|
9
6
|
add_domain_param,
|
|
10
7
|
add_endpoint_param,
|
|
8
|
+
add_nlu_data_param,
|
|
9
|
+
add_out_param,
|
|
11
10
|
add_remote_storage_param,
|
|
11
|
+
add_stories_param,
|
|
12
12
|
)
|
|
13
13
|
from rasa.graph_components.providers.training_tracker_provider import (
|
|
14
14
|
TrainingTrackerProvider,
|
|
@@ -40,6 +40,12 @@ def set_train_arguments(parser: argparse.ArgumentParser) -> None:
|
|
|
40
40
|
parser, help_text="Configuration file for the connectors as a yml file."
|
|
41
41
|
)
|
|
42
42
|
add_remote_storage_param(parser)
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--remote-bot-config-path",
|
|
45
|
+
help="Path to the bot configuration file in the remote storage.",
|
|
46
|
+
required=False,
|
|
47
|
+
type=str,
|
|
48
|
+
)
|
|
43
49
|
|
|
44
50
|
|
|
45
51
|
def set_train_core_arguments(parser: argparse.ArgumentParser) -> None:
|
rasa/cli/train.py
CHANGED
|
@@ -1,18 +1,22 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import asyncio
|
|
3
|
+
import os
|
|
3
4
|
import sys
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from typing import Dict, List, Optional, Text, Union
|
|
6
7
|
|
|
7
8
|
import structlog
|
|
9
|
+
from tarsafe import TarSafe
|
|
8
10
|
|
|
9
11
|
import rasa.cli.arguments.train as train_arguments
|
|
10
12
|
import rasa.cli.utils
|
|
11
13
|
import rasa.core.utils
|
|
12
14
|
import rasa.utils.common
|
|
15
|
+
from rasa.api import train as train_all
|
|
13
16
|
from rasa.cli import SubParsersAction
|
|
14
17
|
from rasa.core.nlg.generator import NaturalLanguageGenerator
|
|
15
18
|
from rasa.core.train import do_compare_training
|
|
19
|
+
from rasa.nlu.persistor import get_persistor
|
|
16
20
|
from rasa.shared.constants import (
|
|
17
21
|
CONFIG_MANDATORY_KEYS,
|
|
18
22
|
CONFIG_MANDATORY_KEYS_CORE,
|
|
@@ -20,6 +24,7 @@ from rasa.shared.constants import (
|
|
|
20
24
|
DEFAULT_DATA_PATH,
|
|
21
25
|
DEFAULT_DOMAIN_PATHS,
|
|
22
26
|
)
|
|
27
|
+
from rasa.shared.exceptions import RasaException
|
|
23
28
|
from rasa.shared.importers.importer import TrainingDataImporter
|
|
24
29
|
|
|
25
30
|
structlogger = structlog.getLogger(__name__)
|
|
@@ -36,7 +41,7 @@ def add_subparser(
|
|
|
36
41
|
"""
|
|
37
42
|
train_parser = subparsers.add_parser(
|
|
38
43
|
"train",
|
|
39
|
-
help="Trains a Rasa model using your NLU data and stories.",
|
|
44
|
+
help="Trains a Rasa model using your CALM flows, NLU data and stories.",
|
|
40
45
|
parents=parents,
|
|
41
46
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
42
47
|
)
|
|
@@ -83,6 +88,37 @@ def _check_nlg_endpoint_validity(endpoint: Union[Path, str]) -> None:
|
|
|
83
88
|
sys.exit(1)
|
|
84
89
|
|
|
85
90
|
|
|
91
|
+
def retrieve_and_unpack_bot_config_from_remote_storage(
|
|
92
|
+
args: argparse.Namespace,
|
|
93
|
+
) -> None:
|
|
94
|
+
"""Retrieve and unpack bot config from remote storage.
|
|
95
|
+
|
|
96
|
+
Bot config is retrieved from remote storage and unpacked
|
|
97
|
+
to the current working directory.
|
|
98
|
+
"""
|
|
99
|
+
persistor = get_persistor(args.remote_storage)
|
|
100
|
+
if persistor is None:
|
|
101
|
+
raise RasaException(
|
|
102
|
+
f"Could not find a persistor for "
|
|
103
|
+
f"the storage type '{args.remote_storage}'."
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
current_working_directory = os.getcwd()
|
|
107
|
+
|
|
108
|
+
persistor.retrieve(args.remote_bot_config_path, current_working_directory)
|
|
109
|
+
|
|
110
|
+
remote_bot_config_tar_file_name = os.path.basename(args.remote_bot_config_path)
|
|
111
|
+
|
|
112
|
+
with TarSafe.open(remote_bot_config_tar_file_name, "r:gz") as tar:
|
|
113
|
+
tar.extractall(path=current_working_directory)
|
|
114
|
+
|
|
115
|
+
structlogger.debug(
|
|
116
|
+
"rasa.train.retrieve_and_unpack_bot_config.remove_downloaded_archive",
|
|
117
|
+
training_data_path=args.remote_bot_config_path,
|
|
118
|
+
)
|
|
119
|
+
os.remove(Path(current_working_directory).joinpath(remote_bot_config_tar_file_name))
|
|
120
|
+
|
|
121
|
+
|
|
86
122
|
def run_training(args: argparse.Namespace, can_exit: bool = False) -> Optional[Text]:
|
|
87
123
|
"""Trains a model.
|
|
88
124
|
|
|
@@ -94,7 +130,9 @@ def run_training(args: argparse.Namespace, can_exit: bool = False) -> Optional[T
|
|
|
94
130
|
Returns:
|
|
95
131
|
Path to a trained model or `None` if training was not successful.
|
|
96
132
|
"""
|
|
97
|
-
|
|
133
|
+
# retrieve and unpack bot_config from remote storage
|
|
134
|
+
if hasattr(args, "remote_bot_config_path") and args.remote_bot_config_path:
|
|
135
|
+
retrieve_and_unpack_bot_config_from_remote_storage(args)
|
|
98
136
|
|
|
99
137
|
domain = rasa.cli.utils.get_validated_path(
|
|
100
138
|
args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=True
|
rasa/cli/utils.py
CHANGED
|
@@ -1,30 +1,32 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import argparse
|
|
3
|
-
import structlog
|
|
4
2
|
import importlib
|
|
3
|
+
import json
|
|
5
4
|
import os
|
|
6
5
|
import sys
|
|
7
6
|
import time
|
|
8
7
|
from pathlib import Path
|
|
9
8
|
from types import FrameType
|
|
10
|
-
from typing import Any, Dict, List, Optional,
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text, Union, overload
|
|
10
|
+
|
|
11
11
|
import randomname
|
|
12
|
+
import structlog
|
|
12
13
|
|
|
13
14
|
import rasa.shared.utils.cli
|
|
14
15
|
import rasa.shared.utils.io
|
|
15
|
-
from rasa
|
|
16
|
+
from rasa import telemetry
|
|
16
17
|
from rasa.shared.constants import (
|
|
17
18
|
ASSISTANT_ID_DEFAULT_VALUE,
|
|
18
19
|
ASSISTANT_ID_KEY,
|
|
19
20
|
DEFAULT_CONFIG_PATH,
|
|
20
21
|
)
|
|
21
|
-
from rasa import
|
|
22
|
+
from rasa.shared.importers.importer import TrainingDataImporter
|
|
22
23
|
from rasa.shared.utils.yaml import read_config_file
|
|
23
24
|
from rasa.utils.io import write_yaml
|
|
24
25
|
|
|
25
26
|
if TYPE_CHECKING:
|
|
26
27
|
from questionary import Question
|
|
27
28
|
from typing_extensions import Literal
|
|
29
|
+
|
|
28
30
|
from rasa.validator import Validator
|
|
29
31
|
|
|
30
32
|
structlogger = structlog.get_logger()
|
rasa/constants.py
CHANGED
|
@@ -18,7 +18,7 @@ CONFIG_TELEMETRY_ID = "rasa_user_id"
|
|
|
18
18
|
CONFIG_TELEMETRY_ENABLED = "enabled"
|
|
19
19
|
CONFIG_TELEMETRY_DATE = "date"
|
|
20
20
|
|
|
21
|
-
MINIMUM_COMPATIBLE_VERSION = "3.10.
|
|
21
|
+
MINIMUM_COMPATIBLE_VERSION = "3.10.0rc1"
|
|
22
22
|
|
|
23
23
|
GLOBAL_USER_CONFIG_PATH = os.path.expanduser("~/.config/rasa/global.yml")
|
|
24
24
|
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import List, Optional, Dict, Text, Set, Any
|
|
3
|
-
|
|
4
2
|
import numpy as np
|
|
5
3
|
import scipy.sparse
|
|
4
|
+
from typing import List, Optional, Dict, Text, Set, Any
|
|
6
5
|
|
|
7
6
|
from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization
|
|
8
7
|
from rasa.nlu.extractors.extractor import EntityTagSpec
|
|
@@ -361,26 +360,6 @@ class SingleStateFeaturizer:
|
|
|
361
360
|
for action in domain.action_names_or_texts
|
|
362
361
|
]
|
|
363
362
|
|
|
364
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
365
|
-
return {
|
|
366
|
-
"action_texts": self.action_texts,
|
|
367
|
-
"entity_tag_specs": self.entity_tag_specs,
|
|
368
|
-
"feature_states": self._default_feature_states,
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
@classmethod
|
|
372
|
-
def create_from_dict(
|
|
373
|
-
cls, data: Dict[str, Any]
|
|
374
|
-
) -> Optional["SingleStateFeaturizer"]:
|
|
375
|
-
if not data:
|
|
376
|
-
return None
|
|
377
|
-
|
|
378
|
-
featurizer = SingleStateFeaturizer()
|
|
379
|
-
featurizer.action_texts = data["action_texts"]
|
|
380
|
-
featurizer._default_feature_states = data["feature_states"]
|
|
381
|
-
featurizer.entity_tag_specs = data["entity_tag_specs"]
|
|
382
|
-
return featurizer
|
|
383
|
-
|
|
384
363
|
|
|
385
364
|
class IntentTokenizerSingleStateFeaturizer(SingleStateFeaturizer):
|
|
386
365
|
"""A SingleStateFeaturizer for use with policies that predict intent labels."""
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import logging
|
|
4
|
-
from abc import abstractmethod
|
|
5
|
-
from collections import defaultdict
|
|
6
2
|
from pathlib import Path
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from abc import abstractmethod
|
|
5
|
+
import jsonpickle
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from tqdm import tqdm
|
|
7
9
|
from typing import (
|
|
8
10
|
Tuple,
|
|
9
11
|
List,
|
|
@@ -16,30 +18,25 @@ from typing import (
|
|
|
16
18
|
Set,
|
|
17
19
|
DefaultDict,
|
|
18
20
|
cast,
|
|
19
|
-
Type,
|
|
20
|
-
Callable,
|
|
21
|
-
ClassVar,
|
|
22
21
|
)
|
|
23
|
-
|
|
24
22
|
import numpy as np
|
|
25
|
-
from tqdm import tqdm
|
|
26
23
|
|
|
24
|
+
from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer
|
|
25
|
+
from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization
|
|
26
|
+
from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError
|
|
27
27
|
import rasa.shared.core.trackers
|
|
28
28
|
import rasa.shared.utils.io
|
|
29
|
-
from rasa.
|
|
30
|
-
from rasa.
|
|
31
|
-
from rasa.core.
|
|
29
|
+
from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME
|
|
30
|
+
from rasa.shared.nlu.training_data.features import Features
|
|
31
|
+
from rasa.shared.core.trackers import DialogueStateTracker
|
|
32
|
+
from rasa.shared.core.domain import State, Domain
|
|
33
|
+
from rasa.shared.core.events import Event, ActionExecuted, UserUttered
|
|
32
34
|
from rasa.shared.core.constants import (
|
|
33
35
|
USER,
|
|
34
36
|
ACTION_UNLIKELY_INTENT_NAME,
|
|
35
37
|
PREVIOUS_ACTION,
|
|
36
38
|
)
|
|
37
|
-
from rasa.shared.core.domain import State, Domain
|
|
38
|
-
from rasa.shared.core.events import Event, ActionExecuted, UserUttered
|
|
39
|
-
from rasa.shared.core.trackers import DialogueStateTracker
|
|
40
39
|
from rasa.shared.exceptions import RasaException
|
|
41
|
-
from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME
|
|
42
|
-
from rasa.shared.nlu.training_data.features import Features
|
|
43
40
|
from rasa.utils.tensorflow.constants import LABEL_PAD_ID
|
|
44
41
|
from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
|
|
45
42
|
|
|
@@ -67,10 +64,6 @@ class InvalidStory(RasaException):
|
|
|
67
64
|
class TrackerFeaturizer:
|
|
68
65
|
"""Base class for actual tracker featurizers."""
|
|
69
66
|
|
|
70
|
-
# Class registry to store all subclasses
|
|
71
|
-
_registry: ClassVar[Dict[str, Type["TrackerFeaturizer"]]] = {}
|
|
72
|
-
_featurizer_type: str = "TrackerFeaturizer"
|
|
73
|
-
|
|
74
67
|
def __init__(
|
|
75
68
|
self, state_featurizer: Optional[SingleStateFeaturizer] = None
|
|
76
69
|
) -> None:
|
|
@@ -81,36 +74,6 @@ class TrackerFeaturizer:
|
|
|
81
74
|
"""
|
|
82
75
|
self.state_featurizer = state_featurizer
|
|
83
76
|
|
|
84
|
-
@classmethod
|
|
85
|
-
def register(cls, featurizer_type: str) -> Callable:
|
|
86
|
-
"""Decorator to register featurizer subclasses."""
|
|
87
|
-
|
|
88
|
-
def wrapper(subclass: Type["TrackerFeaturizer"]) -> Type["TrackerFeaturizer"]:
|
|
89
|
-
cls._registry[featurizer_type] = subclass
|
|
90
|
-
# Store the type identifier in the class for serialization
|
|
91
|
-
subclass._featurizer_type = featurizer_type
|
|
92
|
-
return subclass
|
|
93
|
-
|
|
94
|
-
return wrapper
|
|
95
|
-
|
|
96
|
-
@classmethod
|
|
97
|
-
def from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer":
|
|
98
|
-
"""Create featurizer instance from dictionary."""
|
|
99
|
-
featurizer_type = data.pop("type")
|
|
100
|
-
|
|
101
|
-
if featurizer_type not in cls._registry:
|
|
102
|
-
raise ValueError(f"Unknown featurizer type: {featurizer_type}")
|
|
103
|
-
|
|
104
|
-
# Get the correct subclass and instantiate it
|
|
105
|
-
subclass = cls._registry[featurizer_type]
|
|
106
|
-
return subclass.create_from_dict(data)
|
|
107
|
-
|
|
108
|
-
@classmethod
|
|
109
|
-
@abstractmethod
|
|
110
|
-
def create_from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer":
|
|
111
|
-
"""Each subclass must implement its own creation from dict method."""
|
|
112
|
-
pass
|
|
113
|
-
|
|
114
77
|
@staticmethod
|
|
115
78
|
def _create_states(
|
|
116
79
|
tracker: DialogueStateTracker,
|
|
@@ -502,7 +465,9 @@ class TrackerFeaturizer:
|
|
|
502
465
|
self.state_featurizer.entity_tag_specs = []
|
|
503
466
|
|
|
504
467
|
# noinspection PyTypeChecker
|
|
505
|
-
rasa.shared.utils.io.
|
|
468
|
+
rasa.shared.utils.io.write_text_file(
|
|
469
|
+
str(jsonpickle.encode(self)), featurizer_file
|
|
470
|
+
)
|
|
506
471
|
|
|
507
472
|
@staticmethod
|
|
508
473
|
def load(path: Union[Text, Path]) -> Optional[TrackerFeaturizer]:
|
|
@@ -516,17 +481,7 @@ class TrackerFeaturizer:
|
|
|
516
481
|
"""
|
|
517
482
|
featurizer_file = Path(path) / FEATURIZER_FILE
|
|
518
483
|
if featurizer_file.is_file():
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
if "type" not in data:
|
|
522
|
-
logger.error(
|
|
523
|
-
f"Couldn't load featurizer for policy. "
|
|
524
|
-
f"File '{featurizer_file}' does not contain all "
|
|
525
|
-
f"necessary information. 'type' is missing."
|
|
526
|
-
)
|
|
527
|
-
return None
|
|
528
|
-
|
|
529
|
-
return TrackerFeaturizer.from_dict(data)
|
|
484
|
+
return jsonpickle.decode(rasa.shared.utils.io.read_file(featurizer_file))
|
|
530
485
|
|
|
531
486
|
logger.error(
|
|
532
487
|
f"Couldn't load featurizer for policy. "
|
|
@@ -553,16 +508,7 @@ class TrackerFeaturizer:
|
|
|
553
508
|
)
|
|
554
509
|
]
|
|
555
510
|
|
|
556
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
557
|
-
return {
|
|
558
|
-
"type": self.__class__._featurizer_type,
|
|
559
|
-
"state_featurizer": (
|
|
560
|
-
self.state_featurizer.to_dict() if self.state_featurizer else None
|
|
561
|
-
),
|
|
562
|
-
}
|
|
563
|
-
|
|
564
511
|
|
|
565
|
-
@TrackerFeaturizer.register("FullDialogueTrackerFeaturizer")
|
|
566
512
|
class FullDialogueTrackerFeaturizer(TrackerFeaturizer):
|
|
567
513
|
"""Creates full dialogue training data for time distributed architectures.
|
|
568
514
|
|
|
@@ -700,20 +646,7 @@ class FullDialogueTrackerFeaturizer(TrackerFeaturizer):
|
|
|
700
646
|
|
|
701
647
|
return trackers_as_states
|
|
702
648
|
|
|
703
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
704
|
-
return super().to_dict()
|
|
705
649
|
|
|
706
|
-
@classmethod
|
|
707
|
-
def create_from_dict(cls, data: Dict[str, Any]) -> "FullDialogueTrackerFeaturizer":
|
|
708
|
-
state_featurizer = SingleStateFeaturizer.create_from_dict(
|
|
709
|
-
data["state_featurizer"]
|
|
710
|
-
)
|
|
711
|
-
return cls(
|
|
712
|
-
state_featurizer,
|
|
713
|
-
)
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
@TrackerFeaturizer.register("MaxHistoryTrackerFeaturizer")
|
|
717
650
|
class MaxHistoryTrackerFeaturizer(TrackerFeaturizer):
|
|
718
651
|
"""Truncates the tracker history into `max_history` long sequences.
|
|
719
652
|
|
|
@@ -951,25 +884,7 @@ class MaxHistoryTrackerFeaturizer(TrackerFeaturizer):
|
|
|
951
884
|
|
|
952
885
|
return trackers_as_states
|
|
953
886
|
|
|
954
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
955
|
-
data = super().to_dict()
|
|
956
|
-
data.update(
|
|
957
|
-
{
|
|
958
|
-
"remove_duplicates": self.remove_duplicates,
|
|
959
|
-
"max_history": self.max_history,
|
|
960
|
-
}
|
|
961
|
-
)
|
|
962
|
-
return data
|
|
963
|
-
|
|
964
|
-
@classmethod
|
|
965
|
-
def create_from_dict(cls, data: Dict[str, Any]) -> "MaxHistoryTrackerFeaturizer":
|
|
966
|
-
state_featurizer = SingleStateFeaturizer.create_from_dict(
|
|
967
|
-
data["state_featurizer"]
|
|
968
|
-
)
|
|
969
|
-
return cls(state_featurizer, data["max_history"], data["remove_duplicates"])
|
|
970
887
|
|
|
971
|
-
|
|
972
|
-
@TrackerFeaturizer.register("IntentMaxHistoryTrackerFeaturizer")
|
|
973
888
|
class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer):
|
|
974
889
|
"""Truncates the tracker history into `max_history` long sequences.
|
|
975
890
|
|
|
@@ -1244,18 +1159,6 @@ class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer):
|
|
|
1244
1159
|
|
|
1245
1160
|
return trackers_as_states
|
|
1246
1161
|
|
|
1247
|
-
def to_dict(self) -> Dict[str, Any]:
|
|
1248
|
-
return super().to_dict()
|
|
1249
|
-
|
|
1250
|
-
@classmethod
|
|
1251
|
-
def create_from_dict(
|
|
1252
|
-
cls, data: Dict[str, Any]
|
|
1253
|
-
) -> "IntentMaxHistoryTrackerFeaturizer":
|
|
1254
|
-
state_featurizer = SingleStateFeaturizer.create_from_dict(
|
|
1255
|
-
data["state_featurizer"]
|
|
1256
|
-
)
|
|
1257
|
-
return cls(state_featurizer, data["max_history"], data["remove_duplicates"])
|
|
1258
|
-
|
|
1259
1162
|
|
|
1260
1163
|
def _is_prev_action_unlikely_intent_in_state(state: State) -> bool:
|
|
1261
1164
|
prev_action_name = state.get(PREVIOUS_ACTION, {}).get(ACTION_NAME)
|
rasa/core/policies/ted_policy.py
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
3
2
|
import logging
|
|
3
|
+
|
|
4
|
+
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from collections import defaultdict
|
|
6
7
|
import contextlib
|
|
7
|
-
from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
|
|
8
8
|
|
|
9
9
|
import numpy as np
|
|
10
10
|
import tensorflow as tf
|
|
11
|
+
from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
|
|
11
12
|
|
|
12
|
-
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
13
13
|
from rasa.engine.graph import ExecutionContext
|
|
14
14
|
from rasa.engine.storage.resource import Resource
|
|
15
15
|
from rasa.engine.storage.storage import ModelStorage
|
|
@@ -49,22 +49,18 @@ from rasa.shared.core.generator import TrackerWithCachedStates
|
|
|
49
49
|
from rasa.shared.core.events import EntitiesAdded, Event
|
|
50
50
|
from rasa.shared.core.domain import Domain
|
|
51
51
|
from rasa.shared.nlu.training_data.message import Message
|
|
52
|
-
from rasa.shared.nlu.training_data.features import
|
|
53
|
-
Features,
|
|
54
|
-
save_features,
|
|
55
|
-
load_features,
|
|
56
|
-
)
|
|
52
|
+
from rasa.shared.nlu.training_data.features import Features
|
|
57
53
|
import rasa.shared.utils.io
|
|
58
54
|
import rasa.utils.io
|
|
59
55
|
from rasa.utils import train_utils
|
|
60
|
-
from rasa.utils.tensorflow.feature_array import (
|
|
61
|
-
FeatureArray,
|
|
62
|
-
serialize_nested_feature_arrays,
|
|
63
|
-
deserialize_nested_feature_arrays,
|
|
64
|
-
)
|
|
65
56
|
from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
|
|
66
57
|
from rasa.utils.tensorflow import rasa_layers
|
|
67
|
-
from rasa.utils.tensorflow.model_data import
|
|
58
|
+
from rasa.utils.tensorflow.model_data import (
|
|
59
|
+
RasaModelData,
|
|
60
|
+
FeatureSignature,
|
|
61
|
+
FeatureArray,
|
|
62
|
+
Data,
|
|
63
|
+
)
|
|
68
64
|
from rasa.utils.tensorflow.model_data_utils import convert_to_data_format
|
|
69
65
|
from rasa.utils.tensorflow.constants import (
|
|
70
66
|
LABEL,
|
|
@@ -965,32 +961,22 @@ class TEDPolicy(Policy):
|
|
|
965
961
|
model_path: Path where model is to be persisted
|
|
966
962
|
"""
|
|
967
963
|
model_filename = self._metadata_filename()
|
|
968
|
-
rasa.
|
|
969
|
-
model_path / f"{model_filename}.priority.
|
|
970
|
-
)
|
|
971
|
-
rasa.shared.utils.io.dump_obj_as_json_to_file(
|
|
972
|
-
model_path / f"{model_filename}.meta.json", self.config
|
|
964
|
+
rasa.utils.io.json_pickle(
|
|
965
|
+
model_path / f"{model_filename}.priority.pkl", self.priority
|
|
973
966
|
)
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
self.data_example,
|
|
977
|
-
str(model_path / f"{model_filename}.data_example.st"),
|
|
978
|
-
str(model_path / f"{model_filename}.data_example_metadata.json"),
|
|
967
|
+
rasa.utils.io.pickle_dump(
|
|
968
|
+
model_path / f"{model_filename}.meta.pkl", self.config
|
|
979
969
|
)
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
dict(self._label_data.data) if self._label_data is not None else {},
|
|
983
|
-
str(model_path / f"{model_filename}.label_data.st"),
|
|
984
|
-
str(model_path / f"{model_filename}.label_data_metadata.json"),
|
|
970
|
+
rasa.utils.io.pickle_dump(
|
|
971
|
+
model_path / f"{model_filename}.data_example.pkl", self.data_example
|
|
985
972
|
)
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
self.fake_features, str(model_path / f"{model_filename}.fake_features.st")
|
|
973
|
+
rasa.utils.io.pickle_dump(
|
|
974
|
+
model_path / f"{model_filename}.fake_features.pkl", self.fake_features
|
|
989
975
|
)
|
|
990
|
-
rasa.
|
|
991
|
-
model_path / f"{model_filename}.
|
|
976
|
+
rasa.utils.io.pickle_dump(
|
|
977
|
+
model_path / f"{model_filename}.label_data.pkl",
|
|
978
|
+
dict(self._label_data.data) if self._label_data is not None else {},
|
|
992
979
|
)
|
|
993
|
-
|
|
994
980
|
entity_tag_specs = (
|
|
995
981
|
[tag_spec._asdict() for tag_spec in self._entity_tag_specs]
|
|
996
982
|
if self._entity_tag_specs
|
|
@@ -1008,29 +994,18 @@ class TEDPolicy(Policy):
|
|
|
1008
994
|
model_path: Path where model is to be persisted.
|
|
1009
995
|
"""
|
|
1010
996
|
tf_model_file = model_path / f"{cls._metadata_filename()}.tf_model"
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
loaded_data = deserialize_nested_feature_arrays(
|
|
1014
|
-
str(model_path / f"{cls._metadata_filename()}.data_example.st"),
|
|
1015
|
-
str(model_path / f"{cls._metadata_filename()}.data_example_metadata.json"),
|
|
997
|
+
loaded_data = rasa.utils.io.pickle_load(
|
|
998
|
+
model_path / f"{cls._metadata_filename()}.data_example.pkl"
|
|
1016
999
|
)
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
str(model_path / f"{cls._metadata_filename()}.label_data.st"),
|
|
1020
|
-
str(model_path / f"{cls._metadata_filename()}.label_data_metadata.json"),
|
|
1000
|
+
label_data = rasa.utils.io.pickle_load(
|
|
1001
|
+
model_path / f"{cls._metadata_filename()}.label_data.pkl"
|
|
1021
1002
|
)
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
# load fake features
|
|
1025
|
-
metadata = rasa.shared.utils.io.read_json_file(
|
|
1026
|
-
model_path / f"{cls._metadata_filename()}.fake_features_metadata.json"
|
|
1003
|
+
fake_features = rasa.utils.io.pickle_load(
|
|
1004
|
+
model_path / f"{cls._metadata_filename()}.fake_features.pkl"
|
|
1027
1005
|
)
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
priority = rasa.shared.utils.io.read_json_file(
|
|
1033
|
-
model_path / f"{cls._metadata_filename()}.priority.json"
|
|
1006
|
+
label_data = RasaModelData(data=label_data)
|
|
1007
|
+
priority = rasa.utils.io.json_unpickle(
|
|
1008
|
+
model_path / f"{cls._metadata_filename()}.priority.pkl"
|
|
1034
1009
|
)
|
|
1035
1010
|
entity_tag_specs = rasa.shared.utils.io.read_json_file(
|
|
1036
1011
|
model_path / f"{cls._metadata_filename()}.entity_tag_specs.json"
|
|
@@ -1048,8 +1023,8 @@ class TEDPolicy(Policy):
|
|
|
1048
1023
|
)
|
|
1049
1024
|
for tag_spec in entity_tag_specs
|
|
1050
1025
|
]
|
|
1051
|
-
model_config = rasa.
|
|
1052
|
-
model_path / f"{cls._metadata_filename()}.meta.
|
|
1026
|
+
model_config = rasa.utils.io.pickle_load(
|
|
1027
|
+
model_path / f"{cls._metadata_filename()}.meta.pkl"
|
|
1053
1028
|
)
|
|
1054
1029
|
|
|
1055
1030
|
return {
|
|
@@ -1095,7 +1070,7 @@ class TEDPolicy(Policy):
|
|
|
1095
1070
|
) -> TEDPolicy:
|
|
1096
1071
|
featurizer = TrackerFeaturizer.load(model_path)
|
|
1097
1072
|
|
|
1098
|
-
if not (model_path / f"{cls._metadata_filename()}.data_example.
|
|
1073
|
+
if not (model_path / f"{cls._metadata_filename()}.data_example.pkl").is_file():
|
|
1099
1074
|
return cls(
|
|
1100
1075
|
config,
|
|
1101
1076
|
model_storage,
|
|
@@ -5,7 +5,6 @@ from typing import Any, List, Optional, Text, Dict, Type, Union
|
|
|
5
5
|
|
|
6
6
|
import numpy as np
|
|
7
7
|
import tensorflow as tf
|
|
8
|
-
|
|
9
8
|
import rasa.utils.common
|
|
10
9
|
from rasa.engine.graph import ExecutionContext
|
|
11
10
|
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
@@ -17,7 +16,6 @@ from rasa.shared.core.domain import Domain
|
|
|
17
16
|
from rasa.shared.core.trackers import DialogueStateTracker
|
|
18
17
|
from rasa.shared.core.constants import SLOTS, ACTIVE_LOOP, ACTION_UNLIKELY_INTENT_NAME
|
|
19
18
|
from rasa.shared.core.events import UserUttered, ActionExecuted
|
|
20
|
-
import rasa.shared.utils.io
|
|
21
19
|
from rasa.shared.nlu.constants import (
|
|
22
20
|
INTENT,
|
|
23
21
|
TEXT,
|
|
@@ -105,6 +103,8 @@ from rasa.utils.tensorflow.constants import (
|
|
|
105
103
|
)
|
|
106
104
|
from rasa.utils.tensorflow import layers
|
|
107
105
|
from rasa.utils.tensorflow.model_data import RasaModelData, FeatureArray, Data
|
|
106
|
+
|
|
107
|
+
import rasa.utils.io as io_utils
|
|
108
108
|
from rasa.core.exceptions import RasaCoreException
|
|
109
109
|
from rasa.shared.utils import common
|
|
110
110
|
|
|
@@ -881,12 +881,9 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
|
|
|
881
881
|
model_path: Path where model is to be persisted
|
|
882
882
|
"""
|
|
883
883
|
super().persist_model_utilities(model_path)
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
save_file(
|
|
888
|
-
{str(k): np.array(v) for k, v in self.label_quantiles.items()},
|
|
889
|
-
model_path / f"{self._metadata_filename()}.label_quantiles.st",
|
|
884
|
+
io_utils.pickle_dump(
|
|
885
|
+
model_path / f"{self._metadata_filename()}.label_quantiles.pkl",
|
|
886
|
+
self.label_quantiles,
|
|
890
887
|
)
|
|
891
888
|
|
|
892
889
|
@classmethod
|
|
@@ -897,14 +894,9 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
|
|
|
897
894
|
model_path: Path where model is to be persisted.
|
|
898
895
|
"""
|
|
899
896
|
model_utilties = super()._load_model_utilities(model_path)
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
loaded_label_quantiles = load_file(
|
|
904
|
-
model_path / f"{cls._metadata_filename()}.label_quantiles.st"
|
|
897
|
+
label_quantiles = io_utils.pickle_load(
|
|
898
|
+
model_path / f"{cls._metadata_filename()}.label_quantiles.pkl"
|
|
905
899
|
)
|
|
906
|
-
label_quantiles = {int(k): list(v) for k, v in loaded_label_quantiles.items()}
|
|
907
|
-
|
|
908
900
|
model_utilties.update({"label_quantiles": label_quantiles})
|
|
909
901
|
return model_utilties
|
|
910
902
|
|
|
@@ -229,9 +229,9 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
229
229
|
commands: List[Command] = []
|
|
230
230
|
|
|
231
231
|
slot_set_re = re.compile(
|
|
232
|
-
r"""SetSlot\(
|
|
232
|
+
r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
|
|
233
233
|
)
|
|
234
|
-
start_flow_re = re.compile(r"StartFlow\(([a-zA-Z0-9_-]
|
|
234
|
+
start_flow_re = re.compile(r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)")
|
|
235
235
|
change_flow_re = re.compile(r"ChangeFlow\(\)")
|
|
236
236
|
cancel_flow_re = re.compile(r"CancelFlow\(\)")
|
|
237
237
|
chitchat_re = re.compile(r"ChitChat\(\)")
|
|
@@ -280,9 +280,19 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
280
280
|
commands.append(HumanHandoffCommand())
|
|
281
281
|
elif match := clarify_re.search(action):
|
|
282
282
|
options = sorted([opt.strip() for opt in match.group(1).split(",")])
|
|
283
|
+
# Remove surrounding quotes if present
|
|
284
|
+
cleaned_options = []
|
|
285
|
+
for flow in options:
|
|
286
|
+
if (flow.startswith('"') and flow.endswith('"')) or (
|
|
287
|
+
flow.startswith("'") and flow.endswith("'")
|
|
288
|
+
):
|
|
289
|
+
cleaned_options.append(flow[1:-1])
|
|
290
|
+
else:
|
|
291
|
+
cleaned_options.append(flow)
|
|
292
|
+
# check if flow is valid
|
|
283
293
|
valid_options = [
|
|
284
294
|
flow
|
|
285
|
-
for flow in
|
|
295
|
+
for flow in cleaned_options
|
|
286
296
|
if flow in flows.user_flow_ids
|
|
287
297
|
and flow not in user_flows_on_the_stack(tracker.stack)
|
|
288
298
|
]
|
|
@@ -293,6 +303,13 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
293
303
|
elif change_flow_re.search(action):
|
|
294
304
|
commands.append(ChangeFlowCommand())
|
|
295
305
|
|
|
306
|
+
if not commands:
|
|
307
|
+
structlogger.debug(
|
|
308
|
+
"multi_step_llm_command_generator.parse_commands",
|
|
309
|
+
message="No commands were parsed from the LLM actions.",
|
|
310
|
+
actions=actions,
|
|
311
|
+
)
|
|
312
|
+
|
|
296
313
|
return commands
|
|
297
314
|
|
|
298
315
|
### Helper methods
|