rasa-pro 3.10.10__py3-none-any.whl → 3.10.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rasa-pro might be problematic. Click here for more details.
- rasa/cli/arguments/train.py +9 -3
- rasa/cli/train.py +40 -2
- rasa/cli/utils.py +7 -5
- rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py +20 -3
- rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py +29 -4
- rasa/e2e_test/e2e_test_runner.py +2 -2
- rasa/engine/storage/local_model_storage.py +41 -12
- rasa/model_training.py +10 -3
- rasa/nlu/persistor.py +37 -15
- rasa/shared/constants.py +4 -1
- rasa/shared/importers/importer.py +7 -8
- rasa/version.py +1 -1
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.11.dist-info}/METADATA +2 -2
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.11.dist-info}/RECORD +17 -18
- rasa/shared/importers/remote_importer.py +0 -196
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.11.dist-info}/NOTICE +0 -0
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.11.dist-info}/WHEEL +0 -0
- {rasa_pro-3.10.10.dist-info → rasa_pro-3.10.11.dist-info}/entry_points.txt +0 -0
rasa/cli/arguments/train.py
CHANGED
|
@@ -3,12 +3,12 @@ from typing import Union
|
|
|
3
3
|
|
|
4
4
|
from rasa.cli.arguments.default_arguments import (
|
|
5
5
|
add_config_param,
|
|
6
|
-
add_stories_param,
|
|
7
|
-
add_nlu_data_param,
|
|
8
|
-
add_out_param,
|
|
9
6
|
add_domain_param,
|
|
10
7
|
add_endpoint_param,
|
|
8
|
+
add_nlu_data_param,
|
|
9
|
+
add_out_param,
|
|
11
10
|
add_remote_storage_param,
|
|
11
|
+
add_stories_param,
|
|
12
12
|
)
|
|
13
13
|
from rasa.graph_components.providers.training_tracker_provider import (
|
|
14
14
|
TrainingTrackerProvider,
|
|
@@ -40,6 +40,12 @@ def set_train_arguments(parser: argparse.ArgumentParser) -> None:
|
|
|
40
40
|
parser, help_text="Configuration file for the connectors as a yml file."
|
|
41
41
|
)
|
|
42
42
|
add_remote_storage_param(parser)
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--remote-bot-config-path",
|
|
45
|
+
help="Path to the bot configuration file in the remote storage.",
|
|
46
|
+
required=False,
|
|
47
|
+
type=str,
|
|
48
|
+
)
|
|
43
49
|
|
|
44
50
|
|
|
45
51
|
def set_train_core_arguments(parser: argparse.ArgumentParser) -> None:
|
rasa/cli/train.py
CHANGED
|
@@ -1,18 +1,22 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import asyncio
|
|
3
|
+
import os
|
|
3
4
|
import sys
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from typing import Dict, List, Optional, Text, Union
|
|
6
7
|
|
|
7
8
|
import structlog
|
|
9
|
+
from tarsafe import TarSafe
|
|
8
10
|
|
|
9
11
|
import rasa.cli.arguments.train as train_arguments
|
|
10
12
|
import rasa.cli.utils
|
|
11
13
|
import rasa.core.utils
|
|
12
14
|
import rasa.utils.common
|
|
15
|
+
from rasa.api import train as train_all
|
|
13
16
|
from rasa.cli import SubParsersAction
|
|
14
17
|
from rasa.core.nlg.generator import NaturalLanguageGenerator
|
|
15
18
|
from rasa.core.train import do_compare_training
|
|
19
|
+
from rasa.nlu.persistor import get_persistor
|
|
16
20
|
from rasa.shared.constants import (
|
|
17
21
|
CONFIG_MANDATORY_KEYS,
|
|
18
22
|
CONFIG_MANDATORY_KEYS_CORE,
|
|
@@ -20,6 +24,7 @@ from rasa.shared.constants import (
|
|
|
20
24
|
DEFAULT_DATA_PATH,
|
|
21
25
|
DEFAULT_DOMAIN_PATHS,
|
|
22
26
|
)
|
|
27
|
+
from rasa.shared.exceptions import RasaException
|
|
23
28
|
from rasa.shared.importers.importer import TrainingDataImporter
|
|
24
29
|
|
|
25
30
|
structlogger = structlog.getLogger(__name__)
|
|
@@ -36,7 +41,7 @@ def add_subparser(
|
|
|
36
41
|
"""
|
|
37
42
|
train_parser = subparsers.add_parser(
|
|
38
43
|
"train",
|
|
39
|
-
help="Trains a Rasa model using your NLU data and stories.",
|
|
44
|
+
help="Trains a Rasa model using your CALM flows, NLU data and stories.",
|
|
40
45
|
parents=parents,
|
|
41
46
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
42
47
|
)
|
|
@@ -83,6 +88,37 @@ def _check_nlg_endpoint_validity(endpoint: Union[Path, str]) -> None:
|
|
|
83
88
|
sys.exit(1)
|
|
84
89
|
|
|
85
90
|
|
|
91
|
+
def retrieve_and_unpack_bot_config_from_remote_storage(
|
|
92
|
+
args: argparse.Namespace,
|
|
93
|
+
) -> None:
|
|
94
|
+
"""Retrieve and unpack bot config from remote storage.
|
|
95
|
+
|
|
96
|
+
Bot config is retrieved from remote storage and unpacked
|
|
97
|
+
to the current working directory.
|
|
98
|
+
"""
|
|
99
|
+
persistor = get_persistor(args.remote_storage)
|
|
100
|
+
if persistor is None:
|
|
101
|
+
raise RasaException(
|
|
102
|
+
f"Could not find a persistor for "
|
|
103
|
+
f"the storage type '{args.remote_storage}'."
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
current_working_directory = os.getcwd()
|
|
107
|
+
|
|
108
|
+
persistor.retrieve(args.remote_bot_config_path, current_working_directory)
|
|
109
|
+
|
|
110
|
+
remote_bot_config_tar_file_name = os.path.basename(args.remote_bot_config_path)
|
|
111
|
+
|
|
112
|
+
with TarSafe.open(remote_bot_config_tar_file_name, "r:gz") as tar:
|
|
113
|
+
tar.extractall(path=current_working_directory)
|
|
114
|
+
|
|
115
|
+
structlogger.debug(
|
|
116
|
+
"rasa.train.retrieve_and_unpack_bot_config.remove_downloaded_archive",
|
|
117
|
+
training_data_path=args.remote_bot_config_path,
|
|
118
|
+
)
|
|
119
|
+
os.remove(Path(current_working_directory).joinpath(remote_bot_config_tar_file_name))
|
|
120
|
+
|
|
121
|
+
|
|
86
122
|
def run_training(args: argparse.Namespace, can_exit: bool = False) -> Optional[Text]:
|
|
87
123
|
"""Trains a model.
|
|
88
124
|
|
|
@@ -94,7 +130,9 @@ def run_training(args: argparse.Namespace, can_exit: bool = False) -> Optional[T
|
|
|
94
130
|
Returns:
|
|
95
131
|
Path to a trained model or `None` if training was not successful.
|
|
96
132
|
"""
|
|
97
|
-
|
|
133
|
+
# retrieve and unpack bot_config from remote storage
|
|
134
|
+
if hasattr(args, "remote_bot_config_path") and args.remote_bot_config_path:
|
|
135
|
+
retrieve_and_unpack_bot_config_from_remote_storage(args)
|
|
98
136
|
|
|
99
137
|
domain = rasa.cli.utils.get_validated_path(
|
|
100
138
|
args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=True
|
rasa/cli/utils.py
CHANGED
|
@@ -1,30 +1,32 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import argparse
|
|
3
|
-
import structlog
|
|
4
2
|
import importlib
|
|
3
|
+
import json
|
|
5
4
|
import os
|
|
6
5
|
import sys
|
|
7
6
|
import time
|
|
8
7
|
from pathlib import Path
|
|
9
8
|
from types import FrameType
|
|
10
|
-
from typing import Any, Dict, List, Optional,
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text, Union, overload
|
|
10
|
+
|
|
11
11
|
import randomname
|
|
12
|
+
import structlog
|
|
12
13
|
|
|
13
14
|
import rasa.shared.utils.cli
|
|
14
15
|
import rasa.shared.utils.io
|
|
15
|
-
from rasa
|
|
16
|
+
from rasa import telemetry
|
|
16
17
|
from rasa.shared.constants import (
|
|
17
18
|
ASSISTANT_ID_DEFAULT_VALUE,
|
|
18
19
|
ASSISTANT_ID_KEY,
|
|
19
20
|
DEFAULT_CONFIG_PATH,
|
|
20
21
|
)
|
|
21
|
-
from rasa import
|
|
22
|
+
from rasa.shared.importers.importer import TrainingDataImporter
|
|
22
23
|
from rasa.shared.utils.yaml import read_config_file
|
|
23
24
|
from rasa.utils.io import write_yaml
|
|
24
25
|
|
|
25
26
|
if TYPE_CHECKING:
|
|
26
27
|
from questionary import Question
|
|
27
28
|
from typing_extensions import Literal
|
|
29
|
+
|
|
28
30
|
from rasa.validator import Validator
|
|
29
31
|
|
|
30
32
|
structlogger = structlog.get_logger()
|
|
@@ -229,9 +229,9 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
229
229
|
commands: List[Command] = []
|
|
230
230
|
|
|
231
231
|
slot_set_re = re.compile(
|
|
232
|
-
r"""SetSlot\(
|
|
232
|
+
r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
|
|
233
233
|
)
|
|
234
|
-
start_flow_re = re.compile(r"StartFlow\(([a-zA-Z0-9_-]
|
|
234
|
+
start_flow_re = re.compile(r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)")
|
|
235
235
|
change_flow_re = re.compile(r"ChangeFlow\(\)")
|
|
236
236
|
cancel_flow_re = re.compile(r"CancelFlow\(\)")
|
|
237
237
|
chitchat_re = re.compile(r"ChitChat\(\)")
|
|
@@ -280,9 +280,19 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
280
280
|
commands.append(HumanHandoffCommand())
|
|
281
281
|
elif match := clarify_re.search(action):
|
|
282
282
|
options = sorted([opt.strip() for opt in match.group(1).split(",")])
|
|
283
|
+
# Remove surrounding quotes if present
|
|
284
|
+
cleaned_options = []
|
|
285
|
+
for flow in options:
|
|
286
|
+
if (flow.startswith('"') and flow.endswith('"')) or (
|
|
287
|
+
flow.startswith("'") and flow.endswith("'")
|
|
288
|
+
):
|
|
289
|
+
cleaned_options.append(flow[1:-1])
|
|
290
|
+
else:
|
|
291
|
+
cleaned_options.append(flow)
|
|
292
|
+
# check if flow is valid
|
|
283
293
|
valid_options = [
|
|
284
294
|
flow
|
|
285
|
-
for flow in
|
|
295
|
+
for flow in cleaned_options
|
|
286
296
|
if flow in flows.user_flow_ids
|
|
287
297
|
and flow not in user_flows_on_the_stack(tracker.stack)
|
|
288
298
|
]
|
|
@@ -293,6 +303,13 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
293
303
|
elif change_flow_re.search(action):
|
|
294
304
|
commands.append(ChangeFlowCommand())
|
|
295
305
|
|
|
306
|
+
if not commands:
|
|
307
|
+
structlogger.debug(
|
|
308
|
+
"multi_step_llm_command_generator.parse_commands",
|
|
309
|
+
message="No commands were parsed from the LLM actions.",
|
|
310
|
+
actions=actions,
|
|
311
|
+
)
|
|
312
|
+
|
|
296
313
|
return commands
|
|
297
314
|
|
|
298
315
|
### Helper methods
|
|
@@ -185,6 +185,12 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
185
185
|
|
|
186
186
|
if not commands:
|
|
187
187
|
# no commands are parsed or there's an invalid command
|
|
188
|
+
structlogger.warning(
|
|
189
|
+
"single_step_llm_command_generator.predict_commands",
|
|
190
|
+
message="No commands were predicted as the LLM response could "
|
|
191
|
+
"not be parsed or the LLM responded with an invalid command."
|
|
192
|
+
"Returning a CannotHandleCommand instead.",
|
|
193
|
+
)
|
|
188
194
|
commands = [CannotHandleCommand()]
|
|
189
195
|
|
|
190
196
|
if tracker.has_coexistence_routing_slot:
|
|
@@ -285,14 +291,16 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
285
291
|
|
|
286
292
|
commands: List[Command] = []
|
|
287
293
|
|
|
288
|
-
slot_set_re = re.compile(
|
|
289
|
-
|
|
294
|
+
slot_set_re = re.compile(
|
|
295
|
+
r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
|
|
296
|
+
)
|
|
297
|
+
start_flow_re = re.compile(r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)")
|
|
290
298
|
cancel_flow_re = re.compile(r"CancelFlow\(\)")
|
|
291
299
|
chitchat_re = re.compile(r"ChitChat\(\)")
|
|
292
300
|
skip_question_re = re.compile(r"SkipQuestion\(\)")
|
|
293
301
|
knowledge_re = re.compile(r"SearchAndReply\(\)")
|
|
294
302
|
humand_handoff_re = re.compile(r"HumanHandoff\(\)")
|
|
295
|
-
clarify_re = re.compile(r"Clarify\(([a-zA-Z0-9_, ]+)\)")
|
|
303
|
+
clarify_re = re.compile(r"Clarify\(([\"\'a-zA-Z0-9_, ]+)\)")
|
|
296
304
|
|
|
297
305
|
for action in actions.strip().splitlines():
|
|
298
306
|
if match := slot_set_re.search(action):
|
|
@@ -321,14 +329,31 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
|
|
|
321
329
|
commands.append(HumanHandoffCommand())
|
|
322
330
|
elif match := clarify_re.search(action):
|
|
323
331
|
options = sorted([opt.strip() for opt in match.group(1).split(",")])
|
|
332
|
+
# Remove surrounding quotes if present
|
|
333
|
+
cleaned_options = []
|
|
334
|
+
for flow in options:
|
|
335
|
+
if (flow.startswith('"') and flow.endswith('"')) or (
|
|
336
|
+
flow.startswith("'") and flow.endswith("'")
|
|
337
|
+
):
|
|
338
|
+
cleaned_options.append(flow[1:-1])
|
|
339
|
+
else:
|
|
340
|
+
cleaned_options.append(flow)
|
|
341
|
+
# check if flow is valid
|
|
324
342
|
valid_options = [
|
|
325
|
-
flow for flow in
|
|
343
|
+
flow for flow in cleaned_options if flow in flows.user_flow_ids
|
|
326
344
|
]
|
|
327
345
|
if len(set(valid_options)) == 1:
|
|
328
346
|
commands.extend(cls.start_flow_by_name(valid_options[0], flows))
|
|
329
347
|
elif len(valid_options) > 1:
|
|
330
348
|
commands.append(ClarifyCommand(valid_options))
|
|
331
349
|
|
|
350
|
+
if not commands:
|
|
351
|
+
structlogger.debug(
|
|
352
|
+
"single_step_llm_command_generator.parse_commands",
|
|
353
|
+
message="No commands were parsed from the LLM actions.",
|
|
354
|
+
actions=actions,
|
|
355
|
+
)
|
|
356
|
+
|
|
332
357
|
return commands
|
|
333
358
|
|
|
334
359
|
@classmethod
|
rasa/e2e_test/e2e_test_runner.py
CHANGED
|
@@ -190,11 +190,11 @@ class E2ETestRunner:
|
|
|
190
190
|
error=f"Message handling timed out for user message '{step.text}'.",
|
|
191
191
|
exc_info=True,
|
|
192
192
|
)
|
|
193
|
-
except Exception:
|
|
193
|
+
except Exception as exc:
|
|
194
194
|
structlogger.error(
|
|
195
195
|
"e2e_test_runner.run_prediction_loop",
|
|
196
196
|
error=f"An exception occurred while handling "
|
|
197
|
-
f"user message '{step.text}'.",
|
|
197
|
+
f"user message '{step.text}'. Error: {exc}",
|
|
198
198
|
)
|
|
199
199
|
tracker = await self.agent.tracker_store.retrieve(sender_id) # type: ignore[assignment]
|
|
200
200
|
turns[position], event_cursor = self.get_actual_step_output(
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import logging
|
|
4
3
|
import shutil
|
|
5
4
|
import sys
|
|
6
5
|
import tempfile
|
|
@@ -8,19 +7,21 @@ import uuid
|
|
|
8
7
|
from contextlib import contextmanager
|
|
9
8
|
from datetime import datetime
|
|
10
9
|
from pathlib import Path
|
|
11
|
-
from tarsafe import TarSafe
|
|
12
10
|
from typing import Generator, Optional, Text, Tuple, Union
|
|
13
11
|
|
|
14
|
-
import
|
|
12
|
+
import structlog
|
|
13
|
+
from tarsafe import TarSafe
|
|
14
|
+
|
|
15
|
+
import rasa.model
|
|
15
16
|
import rasa.shared.utils.io
|
|
16
|
-
|
|
17
|
+
import rasa.utils.common
|
|
17
18
|
from rasa.engine.graph import GraphModelConfiguration
|
|
18
19
|
from rasa.engine.storage.resource import Resource
|
|
20
|
+
from rasa.engine.storage.storage import ModelMetadata, ModelStorage
|
|
19
21
|
from rasa.exceptions import UnsupportedModelVersionError
|
|
20
22
|
from rasa.shared.core.domain import Domain
|
|
21
|
-
import rasa.model
|
|
22
23
|
|
|
23
|
-
|
|
24
|
+
structlogger = structlog.get_logger()
|
|
24
25
|
|
|
25
26
|
# Paths within model archive
|
|
26
27
|
MODEL_ARCHIVE_COMPONENTS_DIR = "components"
|
|
@@ -86,7 +87,14 @@ class LocalModelStorage(ModelStorage):
|
|
|
86
87
|
cls._extract_archive_to_directory(
|
|
87
88
|
model_archive_path, temporary_directory_path
|
|
88
89
|
)
|
|
89
|
-
|
|
90
|
+
structlogger.debug(
|
|
91
|
+
"local_model_storage.from_model_archive",
|
|
92
|
+
event_info=(
|
|
93
|
+
f"Extracted model '{model_archive_path}' to "
|
|
94
|
+
f"'{temporary_directory_path}'."
|
|
95
|
+
),
|
|
96
|
+
)
|
|
97
|
+
|
|
90
98
|
cls._initialize_model_storage_from_model_archive(
|
|
91
99
|
temporary_directory_path, storage_path
|
|
92
100
|
)
|
|
@@ -142,6 +150,10 @@ class LocalModelStorage(ModelStorage):
|
|
|
142
150
|
temporary_directory: Path, storage_path: Path
|
|
143
151
|
) -> None:
|
|
144
152
|
for path in (temporary_directory / MODEL_ARCHIVE_COMPONENTS_DIR).glob("*"):
|
|
153
|
+
structlogger.debug(
|
|
154
|
+
"local_model_storage._initialize_model_storage_from_model_archive",
|
|
155
|
+
event_info=f"Moving '{path}' to '{storage_path}'.",
|
|
156
|
+
)
|
|
145
157
|
shutil.move(str(path), str(storage_path))
|
|
146
158
|
|
|
147
159
|
@staticmethod
|
|
@@ -155,7 +167,10 @@ class LocalModelStorage(ModelStorage):
|
|
|
155
167
|
@contextmanager
|
|
156
168
|
def write_to(self, resource: Resource) -> Generator[Path, None, None]:
|
|
157
169
|
"""Persists data for a resource (see parent class for full docstring)."""
|
|
158
|
-
|
|
170
|
+
structlogger.debug(
|
|
171
|
+
"local_model_storage.write_to.resource_write_requested",
|
|
172
|
+
event_info=f"Resource '{resource.name}' was requested for writing.",
|
|
173
|
+
)
|
|
159
174
|
directory = self._directory_for_resource(resource)
|
|
160
175
|
|
|
161
176
|
if not directory.exists():
|
|
@@ -163,7 +178,10 @@ class LocalModelStorage(ModelStorage):
|
|
|
163
178
|
|
|
164
179
|
yield directory
|
|
165
180
|
|
|
166
|
-
|
|
181
|
+
structlogger.debug(
|
|
182
|
+
"local_model_storage.write_to.resource_persisted",
|
|
183
|
+
event_info=f"Resource '{resource.name}' was persisted.",
|
|
184
|
+
)
|
|
167
185
|
|
|
168
186
|
def _directory_for_resource(self, resource: Resource) -> Path:
|
|
169
187
|
return self._storage_path / resource.name
|
|
@@ -171,7 +189,10 @@ class LocalModelStorage(ModelStorage):
|
|
|
171
189
|
@contextmanager
|
|
172
190
|
def read_from(self, resource: Resource) -> Generator[Path, None, None]:
|
|
173
191
|
"""Provides the data of a `Resource` (see parent class for full docstring)."""
|
|
174
|
-
|
|
192
|
+
structlogger.debug(
|
|
193
|
+
"local_model_storage.read_from",
|
|
194
|
+
event_info=f"Resource '{resource.name}' was requested for reading.",
|
|
195
|
+
)
|
|
175
196
|
directory = self._directory_for_resource(resource)
|
|
176
197
|
|
|
177
198
|
if not directory.exists():
|
|
@@ -193,7 +214,12 @@ class LocalModelStorage(ModelStorage):
|
|
|
193
214
|
domain: Domain,
|
|
194
215
|
) -> ModelMetadata:
|
|
195
216
|
"""Creates model package (see parent class for full docstring)."""
|
|
196
|
-
|
|
217
|
+
structlogger.debug(
|
|
218
|
+
"local_model_storage.create_model_package.started",
|
|
219
|
+
event_info=(
|
|
220
|
+
f"Start to created model " f"package for path '{model_archive_path}'.",
|
|
221
|
+
),
|
|
222
|
+
)
|
|
197
223
|
|
|
198
224
|
with windows_safe_temporary_directory() as temp_dir:
|
|
199
225
|
temporary_directory = Path(temp_dir)
|
|
@@ -214,7 +240,10 @@ class LocalModelStorage(ModelStorage):
|
|
|
214
240
|
with TarSafe.open(model_archive_path, "w:gz") as tar:
|
|
215
241
|
tar.add(temporary_directory, arcname="")
|
|
216
242
|
|
|
217
|
-
|
|
243
|
+
structlogger.debug(
|
|
244
|
+
"local_model_storage.create_model_package.finished",
|
|
245
|
+
event_info=f"Model package created in path '{model_archive_path}'.",
|
|
246
|
+
)
|
|
218
247
|
|
|
219
248
|
return model_metadata
|
|
220
249
|
|
rasa/model_training.py
CHANGED
|
@@ -22,7 +22,7 @@ from rasa.engine.storage.local_model_storage import LocalModelStorage
|
|
|
22
22
|
from rasa.engine.storage.storage import ModelStorage
|
|
23
23
|
from rasa.engine.training.components import FingerprintStatus
|
|
24
24
|
from rasa.engine.training.graph_trainer import GraphTrainer
|
|
25
|
-
from rasa.nlu.persistor import StorageType
|
|
25
|
+
from rasa.nlu.persistor import RemoteStorageType, StorageType
|
|
26
26
|
from rasa.shared.core.domain import Domain
|
|
27
27
|
from rasa.shared.core.events import SlotSet
|
|
28
28
|
from rasa.shared.core.training_data.structures import StoryGraph
|
|
@@ -350,18 +350,25 @@ async def _train_graph(
|
|
|
350
350
|
if remote_storage:
|
|
351
351
|
push_model_to_remote_storage(full_model_path, remote_storage)
|
|
352
352
|
full_model_path.unlink()
|
|
353
|
+
remote_storage_string = (
|
|
354
|
+
remote_storage.value
|
|
355
|
+
if isinstance(remote_storage, RemoteStorageType)
|
|
356
|
+
else remote_storage
|
|
357
|
+
)
|
|
353
358
|
structlogger.info(
|
|
354
359
|
"model_training.train.finished_training",
|
|
355
360
|
event_info=(
|
|
356
361
|
f"Your Rasa model {model_name} is trained "
|
|
357
|
-
f"and saved at remote storage provider
|
|
362
|
+
f"and saved at remote storage provider "
|
|
363
|
+
f"'{remote_storage_string}'."
|
|
358
364
|
),
|
|
359
365
|
)
|
|
360
366
|
else:
|
|
361
367
|
structlogger.info(
|
|
362
368
|
"model_training.train.finished_training",
|
|
363
369
|
event_info=(
|
|
364
|
-
f"Your Rasa model is trained and saved at
|
|
370
|
+
f"Your Rasa model is trained and saved at "
|
|
371
|
+
f"'{full_model_path}'."
|
|
365
372
|
),
|
|
366
373
|
)
|
|
367
374
|
|
rasa/nlu/persistor.py
CHANGED
|
@@ -82,16 +82,14 @@ def get_persistor(storage: StorageType) -> Optional[Persistor]:
|
|
|
82
82
|
Currently, `aws`, `gcs`, `azure` and providing module paths are supported remote
|
|
83
83
|
storages.
|
|
84
84
|
"""
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
if storage == RemoteStorageType.AWS.value:
|
|
85
|
+
if storage == RemoteStorageType.AWS:
|
|
88
86
|
return AWSPersistor(
|
|
89
87
|
os.environ.get(BUCKET_NAME_ENV), os.environ.get(AWS_ENDPOINT_URL_ENV)
|
|
90
88
|
)
|
|
91
|
-
if storage == RemoteStorageType.GCS
|
|
89
|
+
if storage == RemoteStorageType.GCS:
|
|
92
90
|
return GCSPersistor(os.environ.get(BUCKET_NAME_ENV))
|
|
93
91
|
|
|
94
|
-
if storage == RemoteStorageType.AZURE
|
|
92
|
+
if storage == RemoteStorageType.AZURE:
|
|
95
93
|
return AzurePersistor(
|
|
96
94
|
os.environ.get(AZURE_CONTAINER_ENV),
|
|
97
95
|
os.environ.get(AZURE_ACCOUNT_NAME_ENV),
|
|
@@ -125,24 +123,36 @@ class Persistor(abc.ABC):
|
|
|
125
123
|
"""Downloads a model that has been persisted to cloud storage.
|
|
126
124
|
|
|
127
125
|
Downloaded model will be saved to the `target_path`.
|
|
128
|
-
If `target_path` is a directory, the model will be
|
|
129
|
-
If `target_path` is a file, the model will be
|
|
126
|
+
If `target_path` is a directory, the model will be downloaded to that directory.
|
|
127
|
+
If `target_path` is a file, the model will be downloaded to that file.
|
|
130
128
|
|
|
131
129
|
Args:
|
|
132
130
|
model_name: The name of the model to retrieve.
|
|
133
|
-
target_path: The path to which the model should be
|
|
131
|
+
target_path: The path to which the model should be downloaded.
|
|
134
132
|
"""
|
|
135
133
|
tar_name = model_name
|
|
136
134
|
if not model_name.endswith(MODEL_ARCHIVE_EXTENSION):
|
|
137
135
|
# ensure backward compatibility
|
|
138
136
|
tar_name = self._tar_name(model_name)
|
|
139
|
-
|
|
140
|
-
self._retrieve_tar(
|
|
141
|
-
self._copy(os.path.basename(tar_name), target_path)
|
|
137
|
+
remote_object_path = self._create_file_key(tar_name)
|
|
138
|
+
self._retrieve_tar(remote_object_path)
|
|
142
139
|
|
|
140
|
+
target_tar_file_name = os.path.basename(tar_name)
|
|
143
141
|
if os.path.isdir(target_path):
|
|
144
|
-
|
|
142
|
+
target_path = os.path.join(target_path, target_tar_file_name)
|
|
143
|
+
|
|
144
|
+
if not os.path.exists(target_path):
|
|
145
|
+
structlogger.debug(
|
|
146
|
+
"persistor.retrieve.copy_model",
|
|
147
|
+
event_info=f"Copying model '{target_tar_file_name}' to "
|
|
148
|
+
f"'{target_path}'.",
|
|
149
|
+
)
|
|
150
|
+
self._copy(target_tar_file_name, target_path)
|
|
145
151
|
|
|
152
|
+
structlogger.debug(
|
|
153
|
+
"persistor.retrieve.model_retrieved",
|
|
154
|
+
event_info=f"Model retrieved and saved to '{target_path}'.",
|
|
155
|
+
)
|
|
146
156
|
return target_path
|
|
147
157
|
|
|
148
158
|
@abc.abstractmethod
|
|
@@ -262,6 +272,11 @@ class AWSPersistor(Persistor):
|
|
|
262
272
|
|
|
263
273
|
def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
|
|
264
274
|
"""Uploads a model persisted in the `target_dir` to s3."""
|
|
275
|
+
structlogger.debug(
|
|
276
|
+
"aws_persistor.persist_tar.uploading_model",
|
|
277
|
+
event_info=f"Uploading tar archive {file_key} to "
|
|
278
|
+
f"s3 bucket '{self.bucket_name}'.",
|
|
279
|
+
)
|
|
265
280
|
with open(tar_path, "rb") as f:
|
|
266
281
|
self.s3.Object(self.bucket_name, file_key).put(Body=f)
|
|
267
282
|
|
|
@@ -329,7 +344,7 @@ class GCSPersistor(Persistor):
|
|
|
329
344
|
def _retrieve_tar(self, target_filename: Text) -> None:
|
|
330
345
|
"""Downloads a model that has previously been persisted to GCS."""
|
|
331
346
|
blob = self.bucket.blob(target_filename)
|
|
332
|
-
blob.download_to_filename(target_filename)
|
|
347
|
+
blob.download_to_filename(os.path.basename(target_filename))
|
|
333
348
|
|
|
334
349
|
|
|
335
350
|
class AzurePersistor(Persistor):
|
|
@@ -370,12 +385,19 @@ class AzurePersistor(Persistor):
|
|
|
370
385
|
def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
|
|
371
386
|
"""Uploads a model persisted in the `target_dir` to Azure."""
|
|
372
387
|
with open(tar_path, "rb") as data:
|
|
373
|
-
self._container_client().upload_blob(
|
|
388
|
+
self._container_client().upload_blob(
|
|
389
|
+
name=file_key,
|
|
390
|
+
data=data,
|
|
391
|
+
# overwrite is set to True to keep in line with
|
|
392
|
+
# how GCS and AWS APIs work this enables easy
|
|
393
|
+
# updating of models in the cloud
|
|
394
|
+
overwrite=True,
|
|
395
|
+
)
|
|
374
396
|
|
|
375
397
|
def _retrieve_tar(self, target_filename: Text) -> None:
|
|
376
398
|
"""Downloads a model that has previously been persisted to Azure."""
|
|
377
399
|
blob_client = self._container_client().get_blob_client(target_filename)
|
|
378
400
|
|
|
379
|
-
with open(target_filename, "wb") as blob:
|
|
401
|
+
with open(os.path.basename(target_filename), "wb") as blob:
|
|
380
402
|
download_stream = blob_client.download_blob()
|
|
381
403
|
blob.write(download_stream.readall())
|
rasa/shared/constants.py
CHANGED
|
@@ -111,7 +111,10 @@ CONFIG_KEYS_NLU = ["language", "pipeline"] + CONFIG_MANDATORY_COMMON_KEYS
|
|
|
111
111
|
CONFIG_KEYS = CONFIG_KEYS_CORE + CONFIG_KEYS_NLU
|
|
112
112
|
CONFIG_MANDATORY_KEYS_CORE: List[Text] = [] + CONFIG_MANDATORY_COMMON_KEYS
|
|
113
113
|
CONFIG_MANDATORY_KEYS_NLU = ["language"] + CONFIG_MANDATORY_COMMON_KEYS
|
|
114
|
-
|
|
114
|
+
# we need the list to contain unique values
|
|
115
|
+
CONFIG_MANDATORY_KEYS = list(
|
|
116
|
+
set(CONFIG_MANDATORY_KEYS_CORE + CONFIG_MANDATORY_KEYS_NLU)
|
|
117
|
+
)
|
|
115
118
|
|
|
116
119
|
# Keys related to Forms (in the Domain)
|
|
117
120
|
REQUIRED_SLOTS_KEY = "required_slots"
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import logging
|
|
2
4
|
from abc import ABC, abstractmethod
|
|
3
5
|
from functools import reduce
|
|
@@ -114,7 +116,7 @@ class TrainingDataImporter(ABC):
|
|
|
114
116
|
domain_path: Optional[Text] = None,
|
|
115
117
|
training_data_paths: Optional[List[Text]] = None,
|
|
116
118
|
args: Optional[Dict[Text, Any]] = {},
|
|
117
|
-
) ->
|
|
119
|
+
) -> TrainingDataImporter:
|
|
118
120
|
"""Loads a `TrainingDataImporter` instance from a configuration file."""
|
|
119
121
|
config = read_config_file(config_path)
|
|
120
122
|
return TrainingDataImporter.load_from_dict(
|
|
@@ -127,7 +129,7 @@ class TrainingDataImporter(ABC):
|
|
|
127
129
|
domain_path: Optional[Text] = None,
|
|
128
130
|
training_data_paths: Optional[List[Text]] = None,
|
|
129
131
|
args: Optional[Dict[Text, Any]] = {},
|
|
130
|
-
) ->
|
|
132
|
+
) -> TrainingDataImporter:
|
|
131
133
|
"""Loads core `TrainingDataImporter` instance.
|
|
132
134
|
|
|
133
135
|
Instance loaded from configuration file will only read Core training data.
|
|
@@ -143,7 +145,7 @@ class TrainingDataImporter(ABC):
|
|
|
143
145
|
domain_path: Optional[Text] = None,
|
|
144
146
|
training_data_paths: Optional[List[Text]] = None,
|
|
145
147
|
args: Optional[Dict[Text, Any]] = {},
|
|
146
|
-
) ->
|
|
148
|
+
) -> TrainingDataImporter:
|
|
147
149
|
"""Loads nlu `TrainingDataImporter` instance.
|
|
148
150
|
|
|
149
151
|
Instance loaded from configuration file will only read NLU training data.
|
|
@@ -166,7 +168,7 @@ class TrainingDataImporter(ABC):
|
|
|
166
168
|
domain_path: Optional[Text] = None,
|
|
167
169
|
training_data_paths: Optional[List[Text]] = None,
|
|
168
170
|
args: Optional[Dict[Text, Any]] = None,
|
|
169
|
-
) ->
|
|
171
|
+
) -> TrainingDataImporter:
|
|
170
172
|
"""Loads a `TrainingDataImporter` instance from a dictionary."""
|
|
171
173
|
from rasa.shared.importers.rasa import RasaFileImporter
|
|
172
174
|
|
|
@@ -195,18 +197,15 @@ class TrainingDataImporter(ABC):
|
|
|
195
197
|
domain_path: Optional[Text] = None,
|
|
196
198
|
training_data_paths: Optional[List[Text]] = None,
|
|
197
199
|
args: Optional[Dict[Text, Any]] = None,
|
|
198
|
-
) -> Optional[
|
|
200
|
+
) -> Optional[TrainingDataImporter]:
|
|
199
201
|
from rasa.shared.importers.multi_project import MultiProjectImporter
|
|
200
202
|
from rasa.shared.importers.rasa import RasaFileImporter
|
|
201
|
-
from rasa.shared.importers.remote_importer import RemoteTrainingDataImporter
|
|
202
203
|
|
|
203
204
|
module_path = importer_config.pop("name", None)
|
|
204
205
|
if module_path == RasaFileImporter.__name__:
|
|
205
206
|
importer_class: Type[TrainingDataImporter] = RasaFileImporter
|
|
206
207
|
elif module_path == MultiProjectImporter.__name__:
|
|
207
208
|
importer_class = MultiProjectImporter
|
|
208
|
-
elif module_path == RemoteTrainingDataImporter.__name__:
|
|
209
|
-
importer_class = RemoteTrainingDataImporter
|
|
210
209
|
else:
|
|
211
210
|
try:
|
|
212
211
|
importer_class = rasa.shared.utils.common.class_from_module_path(
|
rasa/version.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rasa-pro
|
|
3
|
-
Version: 3.10.
|
|
3
|
+
Version: 3.10.11
|
|
4
4
|
Summary: State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development.
|
|
5
5
|
Home-page: https://rasa.com
|
|
6
6
|
Keywords: nlp,machine-learning,machine-learning-library,bot,bots,botkit,rasa conversational-agents,conversational-ai,chatbot,chatbot-framework,bot-framework
|
|
@@ -103,7 +103,7 @@ Requires-Dist: pyyaml (>=6.0)
|
|
|
103
103
|
Requires-Dist: qdrant-client (>=1.9.0,<2.0.0)
|
|
104
104
|
Requires-Dist: questionary (>=1.10.0,<2.1.0)
|
|
105
105
|
Requires-Dist: randomname (>=0.2.1,<0.3.0)
|
|
106
|
-
Requires-Dist: rasa-sdk (
|
|
106
|
+
Requires-Dist: rasa-sdk (>=3.10.0,<3.11.0)
|
|
107
107
|
Requires-Dist: redis (>=4.6.0,<6.0)
|
|
108
108
|
Requires-Dist: regex (>=2022.10.31,<2022.11)
|
|
109
109
|
Requires-Dist: requests (>=2.31.0,<2.32.0)
|
|
@@ -19,7 +19,7 @@ rasa/cli/arguments/interactive.py,sha256=S-cY9XUOg7vzgVh_1mlQWObTmIO9pDQ0OdWVVCx
|
|
|
19
19
|
rasa/cli/arguments/run.py,sha256=G7Ldj5pEsfhXMhhHOqTO1lgRG0939VZ6SRVpwBYaP_M,6340
|
|
20
20
|
rasa/cli/arguments/shell.py,sha256=Vyt3XizJPxKAMo4NIcpdSOs73rD-q-yjWH1Qzc15xCs,367
|
|
21
21
|
rasa/cli/arguments/test.py,sha256=2g6OvwkTlwCK85_nxYvKiFUMICBUMjfttd6Qif2Hme8,6854
|
|
22
|
-
rasa/cli/arguments/train.py,sha256=
|
|
22
|
+
rasa/cli/arguments/train.py,sha256=CMf6h9oCDYS29UXEdVDuSYCkLJTobURhplmvMW4wlNQ,8539
|
|
23
23
|
rasa/cli/arguments/visualize.py,sha256=Su0qyXv4bEx5mrteRqEyf-K3JGQ1t2WCXOYlCpGYfAk,861
|
|
24
24
|
rasa/cli/arguments/x.py,sha256=FQkarKvNBtzZ5xrPBhHWk-ZKPgEHvgE5ItwRL1TNR3I,1027
|
|
25
25
|
rasa/cli/data.py,sha256=M33oHv3PCqDx4Gc5ifGmcZoEc6h2_uP8pRIygimjT8w,12695
|
|
@@ -85,8 +85,8 @@ rasa/cli/studio/train.py,sha256=ruXA5UkaRbO3femk8w3I2cXKs06-34FYtB_9MKdP6hw,1572
|
|
|
85
85
|
rasa/cli/studio/upload.py,sha256=kdHqrVGsEbbqH5fz_HusWwJEycB31SHaPlXer8lXAE0,2069
|
|
86
86
|
rasa/cli/telemetry.py,sha256=ZywhlOpp0l2Yz9oEcOGA2ej3SEkSTisKPpBhn_fS7tc,3538
|
|
87
87
|
rasa/cli/test.py,sha256=Ub7Cm9rFQ_tkB310jPYzVwU0Di88Z7IE0nLi1o-aYbA,8901
|
|
88
|
-
rasa/cli/train.py,sha256=
|
|
89
|
-
rasa/cli/utils.py,sha256=
|
|
88
|
+
rasa/cli/train.py,sha256=a8KB-zc9mFZrWYztpiC7g5Ab3O86KJcTlo_CBLyB7Tk,9934
|
|
89
|
+
rasa/cli/utils.py,sha256=Q4WFdSYrqQvMY2nZY4i2P-vBimUh_cS08KEN-PGkJlg,15662
|
|
90
90
|
rasa/cli/visualize.py,sha256=YmRAATAfxHpgE8_PknGyM-oIujwICNzVftTzz6iLNNc,1256
|
|
91
91
|
rasa/cli/x.py,sha256=1w-H6kb_3OG3zVPJ1isX67BTb_T-x2MJo4OGffCD4Vc,6827
|
|
92
92
|
rasa/constants.py,sha256=BuDQ59uM3GhxmShZR-1IuDHh5VHfEneA9P9HUQagZ9M,1311
|
|
@@ -364,11 +364,11 @@ rasa/dialogue_understanding/generator/llm_command_generator.py,sha256=P1LA0eJxf3
|
|
|
364
364
|
rasa/dialogue_understanding/generator/multi_step/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
365
365
|
rasa/dialogue_understanding/generator/multi_step/fill_slots_prompt.jinja2,sha256=Y0m673tAML3cFPaLM-urMXDsBYUUcXIw9YUpkAhGUuA,2933
|
|
366
366
|
rasa/dialogue_understanding/generator/multi_step/handle_flows_prompt.jinja2,sha256=8l93_QBKBYnqLICVdiTu5ejZDE8F36BU8-qwba0px44,1927
|
|
367
|
-
rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py,sha256=
|
|
367
|
+
rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py,sha256=133WGlwv49PF4_DKM53u0J_pWrDs88dgUYc5fq1m6NQ,31568
|
|
368
368
|
rasa/dialogue_understanding/generator/nlu_command_adapter.py,sha256=-oc3lxAdQEcX25f4IalxOhEw9XhA_6HxGpItlY1RUvY,8412
|
|
369
369
|
rasa/dialogue_understanding/generator/single_step/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
370
370
|
rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2,sha256=qVTuas5XgAv2M_hsihyXl-wAnBDEpg_uhVvNrR5m-h0,3751
|
|
371
|
-
rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py,sha256=
|
|
371
|
+
rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py,sha256=lzSGEqsKtRPWK9hP3GmaZazN39icII-MZp_0sykvpNQ,16031
|
|
372
372
|
rasa/dialogue_understanding/patterns/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
373
373
|
rasa/dialogue_understanding/patterns/cancel.py,sha256=IQ4GVHNnNCqwKRLlAqBtLsgolcbPPnHsHdb3aOAFhEs,3868
|
|
374
374
|
rasa/dialogue_understanding/patterns/cannot_handle.py,sha256=pg0zJHl-hDBnl6y9IyxZzW57yuMdfD8xI8eiK6EVrG8,1406
|
|
@@ -410,7 +410,7 @@ rasa/e2e_test/e2e_test_converter.py,sha256=VxIx7uk36HzLIyEumJiR6G6-CyyqkV_lYoX-X
|
|
|
410
410
|
rasa/e2e_test/e2e_test_converter_prompt.jinja2,sha256=EMy-aCd7jLARHmwAuZUGT5ABnNHjR872_pexRIMGA7c,2791
|
|
411
411
|
rasa/e2e_test/e2e_test_coverage_report.py,sha256=Cv5bMtoOC240231YMNHz10ibSqm_UD1-eskQVdjPUsw,11326
|
|
412
412
|
rasa/e2e_test/e2e_test_result.py,sha256=9LlH6vIQeK_dxDwMQ5RzlNHoxCNXpWC9S527-ch8kUA,1649
|
|
413
|
-
rasa/e2e_test/e2e_test_runner.py,sha256=
|
|
413
|
+
rasa/e2e_test/e2e_test_runner.py,sha256=o-vCDTMV3xoElNV-Cj2_vnuDwhTxnQBXdMjRE5z1cko,44021
|
|
414
414
|
rasa/e2e_test/e2e_test_schema.yml,sha256=0deWjuKRHNo6e_LSCnUoiw9NLIYf6dj1-zFPl_AqLYA,5632
|
|
415
415
|
rasa/e2e_test/pykwalify_extensions.py,sha256=OGYKIKYJXd2S0NrWknoQuijyBQaE-oMLkfV_eMRkGSM,1331
|
|
416
416
|
rasa/e2e_test/stub_custom_action.py,sha256=teq8c5I6IuUsFX4lPdeBLY3j0SLSMCC95KmKx7GrE8I,2369
|
|
@@ -434,7 +434,7 @@ rasa/engine/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
|
|
|
434
434
|
rasa/engine/runner/dask.py,sha256=npWZXpYtYjdbQOH4OVCTsA2F0TNK43g8201y2wCS6U8,9540
|
|
435
435
|
rasa/engine/runner/interface.py,sha256=4mbJSMbXPwnITDl5maYZjq8sUcbWw1ToGjYVQtnWCkc,1678
|
|
436
436
|
rasa/engine/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
437
|
-
rasa/engine/storage/local_model_storage.py,sha256=
|
|
437
|
+
rasa/engine/storage/local_model_storage.py,sha256=gybPSv4HuG6JdbU6Blxs3PEPTeY-VkOFyzWUJSGcCSM,10517
|
|
438
438
|
rasa/engine/storage/resource.py,sha256=1ecgZbDw7y6CLLFLdi5cYRfdk0yTmD7V1seWEzdtzpU,3931
|
|
439
439
|
rasa/engine/storage/storage.py,sha256=Qx1kp8LwX6Oe0W6rfTd0qzo9jWhNVttXWhVXRy_qW4w,6923
|
|
440
440
|
rasa/engine/training/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -483,7 +483,7 @@ rasa/markers/upload.py,sha256=Ot1s_O-CEIB9c4CKUlfOldiJo92pdqxFUHOPCU7E_NU,2518
|
|
|
483
483
|
rasa/markers/validate.py,sha256=YypXKRS87xxrMMEz9HpAQzYJUwg0OPbczMXBRNjlJq4,709
|
|
484
484
|
rasa/model.py,sha256=GH1-N6Po3gL3nwfa9eGoN2bMRNMrn4f3mi17-osW3T0,3491
|
|
485
485
|
rasa/model_testing.py,sha256=h0QUpJu6p_TDse3aHjCfYwI6OGH47b3Iuo5Ot0HQADM,14959
|
|
486
|
-
rasa/model_training.py,sha256=
|
|
486
|
+
rasa/model_training.py,sha256=1LoW9TMgzpXeXedLtzxmGHxOl1NBTbNDaqLzu2XayxE,20631
|
|
487
487
|
rasa/nlu/__init__.py,sha256=D0IYuTK_ZQ_F_9xsy0bXxVCAtU62Fzvp8S7J9tmfI_c,123
|
|
488
488
|
rasa/nlu/classifiers/__init__.py,sha256=Qvrf7_rfiMxm2Vt2fClb56R3QFExf7WPdFdL-AOvgsk,118
|
|
489
489
|
rasa/nlu/classifiers/classifier.py,sha256=9fm1mORuFf1vowYIXmqE9yLRKdSC4nGQW7UqNZQipKY,133
|
|
@@ -524,7 +524,7 @@ rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py,sha256=aw
|
|
|
524
524
|
rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py,sha256=PhzJ17lNv3I5h8WrCvjzjjcUvbu_MJBxY6k3pQTDCac,10289
|
|
525
525
|
rasa/nlu/featurizers/sparse_featurizer/sparse_featurizer.py,sha256=m6qpixorfTDFWSfGVmLImTOHM6zKdgydPaP_wVxCQ-w,220
|
|
526
526
|
rasa/nlu/model.py,sha256=r6StZb4Dmum_3dRoocxZWo2M5KVNV20_yKNvYZNvpOc,557
|
|
527
|
-
rasa/nlu/persistor.py,sha256=
|
|
527
|
+
rasa/nlu/persistor.py,sha256=QniAoBRjk9CPzFJOdkJIpgc_eXXan1cKC61L_xWQjGk,14702
|
|
528
528
|
rasa/nlu/run.py,sha256=WumXqNn2PEyab463geNnOu3IPwgaCtBai-x685BYCNw,803
|
|
529
529
|
rasa/nlu/selectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
530
530
|
rasa/nlu/selectors/response_selector.py,sha256=gwffu9zLniMseOzX-SuqaZ2CQiGi4JUbDcwUe-BsThI,39021
|
|
@@ -546,7 +546,7 @@ rasa/nlu/utils/spacy_utils.py,sha256=pBvsCVKVuZ3b2Pjn-XuOVZ6lzZu9Voc2R4N1VczwtCM
|
|
|
546
546
|
rasa/plugin.py,sha256=H_OZcHy_U3eAK-JHr43TSxcPqS0JEGcZkFvmumeeJEs,2670
|
|
547
547
|
rasa/server.py,sha256=pUdhi5dkhzEj4bngj2bTUKJohepjpY-aZ4MGKHYZRH0,57775
|
|
548
548
|
rasa/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
549
|
-
rasa/shared/constants.py,sha256=
|
|
549
|
+
rasa/shared/constants.py,sha256=MfO5vD2BRZMKWt_V3XiT7Yap1XhFWf5k13epuMG-P2s,9327
|
|
550
550
|
rasa/shared/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
551
551
|
rasa/shared/core/command_payload_reader.py,sha256=Vhiop9LWFawaEruRifBBrVmoEJ-fj1Tli1wBvsYu2_I,3563
|
|
552
552
|
rasa/shared/core/constants.py,sha256=U1q0-oGMQ7aDsP5p6H4CRaBi-H-KZpm2IXpTzIbaEv8,5169
|
|
@@ -597,10 +597,9 @@ rasa/shared/engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
|
|
|
597
597
|
rasa/shared/engine/caching.py,sha256=PiBV4vQ5c5z5sopxK1XpzmmaF4V42QQxP1_MX0Yw9PI,743
|
|
598
598
|
rasa/shared/exceptions.py,sha256=bUTjjIH4PDykWJqApxYXdpPvfo03EVQ6xo1sOc3yjH4,5118
|
|
599
599
|
rasa/shared/importers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
600
|
-
rasa/shared/importers/importer.py,sha256=
|
|
600
|
+
rasa/shared/importers/importer.py,sha256=wBHncehjhIdNpzdVaekKz2e7OJTJOWgWIdD2Vpqqv6o,27190
|
|
601
601
|
rasa/shared/importers/multi_project.py,sha256=b5LypQf0wdiDaZyAihqItQXSuzDjMibypbUySDXff7g,7863
|
|
602
602
|
rasa/shared/importers/rasa.py,sha256=9Ie86LgcS9ZtGWR5-NLylxSfK_I329SLvb_UTC71vu4,3853
|
|
603
|
-
rasa/shared/importers/remote_importer.py,sha256=SCmwWDKb7GczVEJhAdH8_s2dq55P4f31qDwQ8y18ndk,7694
|
|
604
603
|
rasa/shared/importers/utils.py,sha256=-EzZ0yH_YnJCSaC4JVx7xrRrXyr8CV6tAKlbASeMtJg,1242
|
|
605
604
|
rasa/shared/nlu/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
606
605
|
rasa/shared/nlu/constants.py,sha256=rf628BT4r6hnvN6QWyh_t2UFKOD7PR5APspi6igmeCU,1643
|
|
@@ -720,9 +719,9 @@ rasa/utils/train_utils.py,sha256=f1NWpp5y6al0dzoQyyio4hc4Nf73DRoRSHDzEK6-C4E,212
|
|
|
720
719
|
rasa/utils/url_tools.py,sha256=JQcHL2aLqLHu82k7_d9imUoETCm2bmlHaDpOJ-dKqBc,1218
|
|
721
720
|
rasa/utils/yaml.py,sha256=KjbZq5C94ZP7Jdsw8bYYF7HASI6K4-C_kdHfrnPLpSI,2000
|
|
722
721
|
rasa/validator.py,sha256=ToRaa4dS859CJO3H2VGqS943O5qWOg45ypbDfFMKECU,62699
|
|
723
|
-
rasa/version.py,sha256=
|
|
724
|
-
rasa_pro-3.10.
|
|
725
|
-
rasa_pro-3.10.
|
|
726
|
-
rasa_pro-3.10.
|
|
727
|
-
rasa_pro-3.10.
|
|
728
|
-
rasa_pro-3.10.
|
|
722
|
+
rasa/version.py,sha256=0PycS7JWF6zF5RV14lJkREbKvYdYXHB_CbCKkQiYfP0,118
|
|
723
|
+
rasa_pro-3.10.11.dist-info/METADATA,sha256=DoqzzlFIc1_OXg5TQimVBnOhWchKq-_c6vblqlOOeH0,30901
|
|
724
|
+
rasa_pro-3.10.11.dist-info/NOTICE,sha256=7HlBoMHJY9CL2GlYSfTQ-PZsVmLmVkYmMiPlTjhuCqA,218
|
|
725
|
+
rasa_pro-3.10.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
726
|
+
rasa_pro-3.10.11.dist-info/entry_points.txt,sha256=ckJ2SfEyTPgBqj_I6vm_tqY9dZF_LAPJZA335Xp0Q9U,43
|
|
727
|
+
rasa_pro-3.10.11.dist-info/RECORD,,
|
|
@@ -1,196 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from typing import Dict, List, Optional, Text, Union
|
|
3
|
-
|
|
4
|
-
import structlog
|
|
5
|
-
from tarsafe import TarSafe
|
|
6
|
-
|
|
7
|
-
import rasa.shared.core.flows.yaml_flows_io
|
|
8
|
-
import rasa.shared.data
|
|
9
|
-
import rasa.shared.utils.common
|
|
10
|
-
import rasa.shared.utils.io
|
|
11
|
-
from rasa.nlu.persistor import StorageType
|
|
12
|
-
from rasa.shared.core.domain import Domain, InvalidDomain
|
|
13
|
-
from rasa.shared.core.flows import FlowsList
|
|
14
|
-
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
|
|
15
|
-
YAMLStoryReader,
|
|
16
|
-
)
|
|
17
|
-
from rasa.shared.core.training_data.structures import StoryGraph
|
|
18
|
-
from rasa.shared.exceptions import RasaException
|
|
19
|
-
from rasa.shared.importers import utils
|
|
20
|
-
from rasa.shared.importers.importer import TrainingDataImporter
|
|
21
|
-
from rasa.shared.nlu.training_data.training_data import TrainingData
|
|
22
|
-
from rasa.shared.utils.yaml import read_model_configuration
|
|
23
|
-
|
|
24
|
-
structlogger = structlog.get_logger()
|
|
25
|
-
|
|
26
|
-
TRAINING_DATA_ARCHIVE = "training_data.tar.gz"
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class RemoteTrainingDataImporter(TrainingDataImporter):
|
|
30
|
-
"""Remote `TrainingFileImporter` implementation.
|
|
31
|
-
|
|
32
|
-
Fetches training data from a remote storage and extracts it to a local directory.
|
|
33
|
-
Extracted training data is then used to load flows, NLU, stories,
|
|
34
|
-
domain, and config files.
|
|
35
|
-
"""
|
|
36
|
-
|
|
37
|
-
def __init__(
|
|
38
|
-
self,
|
|
39
|
-
config_file: Optional[Text] = None,
|
|
40
|
-
domain_path: Optional[Text] = None,
|
|
41
|
-
training_data_paths: Optional[Union[List[Text], Text]] = None,
|
|
42
|
-
project_directory: Optional[Text] = None,
|
|
43
|
-
remote_storage: Optional[StorageType] = None,
|
|
44
|
-
training_data_path: Optional[Text] = None,
|
|
45
|
-
):
|
|
46
|
-
"""Initializes `RemoteTrainingDataImporter`.
|
|
47
|
-
|
|
48
|
-
Args:
|
|
49
|
-
config_file: Path to the model configuration file.
|
|
50
|
-
domain_path: Path to the domain file.
|
|
51
|
-
training_data_paths: List of paths to the training data files.
|
|
52
|
-
project_directory: Path to the project directory.
|
|
53
|
-
remote_storage: Storage to use to load the training data.
|
|
54
|
-
training_data_path: Path to the training data.
|
|
55
|
-
"""
|
|
56
|
-
self.remote_storage = remote_storage
|
|
57
|
-
self.training_data_path = training_data_path
|
|
58
|
-
|
|
59
|
-
self.extracted_path = self._fetch_and_extract_training_archive(
|
|
60
|
-
TRAINING_DATA_ARCHIVE, self.training_data_path
|
|
61
|
-
)
|
|
62
|
-
|
|
63
|
-
self._nlu_files = rasa.shared.data.get_data_files(
|
|
64
|
-
self.extracted_path, rasa.shared.data.is_nlu_file
|
|
65
|
-
)
|
|
66
|
-
self._story_files = rasa.shared.data.get_data_files(
|
|
67
|
-
self.extracted_path, YAMLStoryReader.is_stories_file
|
|
68
|
-
)
|
|
69
|
-
self._flow_files = rasa.shared.data.get_data_files(
|
|
70
|
-
self.extracted_path, rasa.shared.core.flows.yaml_flows_io.is_flows_file
|
|
71
|
-
)
|
|
72
|
-
self._conversation_test_files = rasa.shared.data.get_data_files(
|
|
73
|
-
self.extracted_path, YAMLStoryReader.is_test_stories_file
|
|
74
|
-
)
|
|
75
|
-
|
|
76
|
-
self.config_file = config_file
|
|
77
|
-
|
|
78
|
-
def _fetch_training_archive(
|
|
79
|
-
self, training_file: str, training_data_path: Optional[str] = None
|
|
80
|
-
) -> str:
|
|
81
|
-
"""Fetches training files from remote storage."""
|
|
82
|
-
from rasa.nlu.persistor import get_persistor
|
|
83
|
-
|
|
84
|
-
persistor = get_persistor(self.remote_storage)
|
|
85
|
-
if persistor is None:
|
|
86
|
-
raise RasaException(
|
|
87
|
-
f"Could not find a persistor for "
|
|
88
|
-
f"the storage type '{self.remote_storage}'."
|
|
89
|
-
)
|
|
90
|
-
|
|
91
|
-
return persistor.retrieve(training_file, training_data_path)
|
|
92
|
-
|
|
93
|
-
def _fetch_and_extract_training_archive(
|
|
94
|
-
self, training_file: str, training_data_path: Optional[Text] = None
|
|
95
|
-
) -> Optional[str]:
|
|
96
|
-
"""Fetches and extracts training files from remote storage.
|
|
97
|
-
|
|
98
|
-
If the `training_data_path` is not provided, the training
|
|
99
|
-
data is extracted to the current working directory.
|
|
100
|
-
|
|
101
|
-
Args:
|
|
102
|
-
training_file: Name of the training data archive file.
|
|
103
|
-
training_data_path: Path to the training data.
|
|
104
|
-
|
|
105
|
-
Returns:
|
|
106
|
-
Path to the extracted training data.
|
|
107
|
-
"""
|
|
108
|
-
|
|
109
|
-
if training_data_path is None:
|
|
110
|
-
training_data_path = os.path.join(os.getcwd(), "data")
|
|
111
|
-
|
|
112
|
-
if os.path.isfile(training_data_path):
|
|
113
|
-
raise ValueError(
|
|
114
|
-
f"Training data path '{training_data_path}' is a file. "
|
|
115
|
-
f"Please provide a directory path."
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
structlogger.debug(
|
|
119
|
-
"rasa.importers.remote_training_data_importer.fetch_training_archive",
|
|
120
|
-
training_data_path=training_data_path,
|
|
121
|
-
)
|
|
122
|
-
training_archive_file_path = self._fetch_training_archive(
|
|
123
|
-
training_file, training_data_path
|
|
124
|
-
)
|
|
125
|
-
|
|
126
|
-
if not os.path.isfile(training_archive_file_path):
|
|
127
|
-
raise FileNotFoundError(
|
|
128
|
-
f"Training data archive '{training_archive_file_path}' not found. "
|
|
129
|
-
f"Please make sure to provide the correct path."
|
|
130
|
-
)
|
|
131
|
-
|
|
132
|
-
structlogger.debug(
|
|
133
|
-
"rasa.importers.remote_training_data_importer.extract_training_archive",
|
|
134
|
-
training_archive_file_path=training_archive_file_path,
|
|
135
|
-
training_data_path=training_data_path,
|
|
136
|
-
)
|
|
137
|
-
with TarSafe.open(training_archive_file_path, "r:gz") as tar:
|
|
138
|
-
tar.extractall(path=training_data_path)
|
|
139
|
-
|
|
140
|
-
structlogger.debug(
|
|
141
|
-
"rasa.importers.remote_training_data_importer.remove_downloaded_archive",
|
|
142
|
-
training_data_path=training_data_path,
|
|
143
|
-
)
|
|
144
|
-
os.remove(training_archive_file_path)
|
|
145
|
-
return training_data_path
|
|
146
|
-
|
|
147
|
-
def get_config(self) -> Dict:
|
|
148
|
-
"""Retrieves model config (see parent class for full docstring)."""
|
|
149
|
-
if not self.config_file or not os.path.exists(self.config_file):
|
|
150
|
-
structlogger.debug(
|
|
151
|
-
"rasa.importers.remote_training_data_importer.no_config_file",
|
|
152
|
-
message="No configuration file was provided to the RasaFileImporter.",
|
|
153
|
-
)
|
|
154
|
-
return {}
|
|
155
|
-
|
|
156
|
-
config = read_model_configuration(self.config_file)
|
|
157
|
-
return config
|
|
158
|
-
|
|
159
|
-
@rasa.shared.utils.common.cached_method
|
|
160
|
-
def get_config_file_for_auto_config(self) -> Optional[Text]:
|
|
161
|
-
"""Returns config file path for auto-config only if there is a single one."""
|
|
162
|
-
return self.config_file
|
|
163
|
-
|
|
164
|
-
def get_stories(self, exclusion_percentage: Optional[int] = None) -> StoryGraph:
|
|
165
|
-
"""Retrieves training stories / rules (see parent class for full docstring)."""
|
|
166
|
-
return utils.story_graph_from_paths(
|
|
167
|
-
self._story_files, self.get_domain(), exclusion_percentage
|
|
168
|
-
)
|
|
169
|
-
|
|
170
|
-
def get_flows(self) -> FlowsList:
|
|
171
|
-
"""Retrieves training stories / rules (see parent class for full docstring)."""
|
|
172
|
-
return utils.flows_from_paths(self._flow_files)
|
|
173
|
-
|
|
174
|
-
def get_conversation_tests(self) -> StoryGraph:
|
|
175
|
-
"""Retrieves conversation test stories (see parent class for full docstring)."""
|
|
176
|
-
return utils.story_graph_from_paths(
|
|
177
|
-
self._conversation_test_files, self.get_domain()
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
|
|
181
|
-
"""Retrieves NLU training data (see parent class for full docstring)."""
|
|
182
|
-
return utils.training_data_from_paths(self._nlu_files, language)
|
|
183
|
-
|
|
184
|
-
def get_domain(self) -> Domain:
|
|
185
|
-
"""Retrieves model domain (see parent class for full docstring)."""
|
|
186
|
-
domain = Domain.empty()
|
|
187
|
-
domain_path = f"{self.extracted_path}"
|
|
188
|
-
try:
|
|
189
|
-
domain = Domain.load(domain_path)
|
|
190
|
-
except InvalidDomain as e:
|
|
191
|
-
rasa.shared.utils.io.raise_warning(
|
|
192
|
-
f"Loading domain from '{domain_path}' failed. Using "
|
|
193
|
-
f"empty domain. Error: '{e}'"
|
|
194
|
-
)
|
|
195
|
-
|
|
196
|
-
return domain
|
|
File without changes
|
|
File without changes
|
|
File without changes
|