rasa-pro 3.13.3__py3-none-any.whl → 3.13.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rasa-pro might be problematic. Click here for more details.
- rasa/__main__.py +8 -0
- rasa/cli/data.py +8 -3
- rasa/cli/train.py +8 -0
- rasa/cli/utils.py +31 -15
- rasa/core/channels/inspector/package.json +3 -2
- rasa/core/channels/inspector/yarn.lock +8 -8
- rasa/core/policies/enterprise_search_policy.py +15 -18
- rasa/dialogue_understanding/generator/command_parser.py +9 -13
- rasa/dialogue_understanding_test/validation.py +9 -10
- rasa/e2e_test/e2e_config.py +18 -11
- rasa/e2e_test/utils/validation.py +17 -19
- rasa/engine/validation.py +86 -91
- rasa/exceptions.py +26 -1
- rasa/shared/providers/_configs/default_litellm_client_config.py +3 -7
- rasa/shared/utils/cli.py +2 -0
- rasa/shared/utils/health_check/health_check.py +10 -14
- rasa/utils/tensorflow/transformer.py +3 -3
- rasa/validator.py +7 -5
- rasa/version.py +1 -1
- {rasa_pro-3.13.3.dist-info → rasa_pro-3.13.4.dist-info}/METADATA +1 -1
- {rasa_pro-3.13.3.dist-info → rasa_pro-3.13.4.dist-info}/RECORD +24 -24
- {rasa_pro-3.13.3.dist-info → rasa_pro-3.13.4.dist-info}/NOTICE +0 -0
- {rasa_pro-3.13.3.dist-info → rasa_pro-3.13.4.dist-info}/WHEEL +0 -0
- {rasa_pro-3.13.3.dist-info → rasa_pro-3.13.4.dist-info}/entry_points.txt +0 -0
rasa/__main__.py
CHANGED
|
@@ -34,6 +34,7 @@ from rasa.cli.utils import (
|
|
|
34
34
|
warn_if_rasa_plus_package_installed,
|
|
35
35
|
)
|
|
36
36
|
from rasa.constants import MINIMUM_COMPATIBLE_VERSION
|
|
37
|
+
from rasa.exceptions import DetailedRasaException
|
|
37
38
|
from rasa.plugin import plugin_manager
|
|
38
39
|
from rasa.shared.exceptions import RasaException
|
|
39
40
|
from rasa.utils.common import configure_logging_and_warnings
|
|
@@ -152,6 +153,13 @@ def main(raw_arguments: Optional[List[str]] = None) -> None:
|
|
|
152
153
|
structlogger.error("cli.no_command", event_info="No command specified.")
|
|
153
154
|
arg_parser.print_help()
|
|
154
155
|
sys.exit(1)
|
|
156
|
+
except DetailedRasaException as exc:
|
|
157
|
+
structlogger.error(
|
|
158
|
+
exc.code,
|
|
159
|
+
event_info=exc.info,
|
|
160
|
+
**exc.ctx,
|
|
161
|
+
)
|
|
162
|
+
sys.exit(1)
|
|
155
163
|
except RasaException as exc:
|
|
156
164
|
# these are exceptions we expect to happen (e.g. invalid training data format)
|
|
157
165
|
# it doesn't make sense to print a stacktrace for these if we are not in
|
rasa/cli/data.py
CHANGED
|
@@ -205,9 +205,14 @@ def _build_training_data_importer(args: argparse.Namespace) -> "TrainingDataImpo
|
|
|
205
205
|
args.domain, "domain", DEFAULT_DOMAIN_PATHS, none_is_valid=False
|
|
206
206
|
)
|
|
207
207
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
208
|
+
if config:
|
|
209
|
+
return TrainingDataImporter.load_from_config(
|
|
210
|
+
domain_path=domain, training_data_paths=args.data, config_path=config
|
|
211
|
+
)
|
|
212
|
+
else:
|
|
213
|
+
return TrainingDataImporter.load_from_dict(
|
|
214
|
+
domain_path=domain, training_data_paths=args.data, config_path=config
|
|
215
|
+
)
|
|
211
216
|
|
|
212
217
|
|
|
213
218
|
def _append_story_structure_arguments(parser: argparse.ArgumentParser) -> None:
|
rasa/cli/train.py
CHANGED
|
@@ -15,6 +15,7 @@ from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephra
|
|
|
15
15
|
from rasa.core.nlg.generator import NaturalLanguageGenerator
|
|
16
16
|
from rasa.core.train import do_compare_training
|
|
17
17
|
from rasa.engine.validation import validate_api_type_config_key_usage
|
|
18
|
+
from rasa.exceptions import ValidationError
|
|
18
19
|
from rasa.shared.constants import (
|
|
19
20
|
CONFIG_MANDATORY_KEYS,
|
|
20
21
|
CONFIG_MANDATORY_KEYS_CORE,
|
|
@@ -81,6 +82,13 @@ def _check_nlg_endpoint_validity(endpoint: Union[Path, str]) -> None:
|
|
|
81
82
|
ContextualResponseRephraser.__name__,
|
|
82
83
|
)
|
|
83
84
|
NaturalLanguageGenerator.create(endpoints.nlg)
|
|
85
|
+
except ValidationError as e:
|
|
86
|
+
structlogger.error(
|
|
87
|
+
e.code,
|
|
88
|
+
event_info=e.info,
|
|
89
|
+
**e.ctx,
|
|
90
|
+
)
|
|
91
|
+
sys.exit(1)
|
|
84
92
|
except Exception as e:
|
|
85
93
|
structlogger.error(
|
|
86
94
|
"cli.train.nlg_failed_to_initialise.validation_error",
|
rasa/cli/utils.py
CHANGED
|
@@ -14,7 +14,7 @@ import structlog
|
|
|
14
14
|
import rasa.shared.utils.cli
|
|
15
15
|
import rasa.shared.utils.io
|
|
16
16
|
from rasa import telemetry
|
|
17
|
-
from rasa.exceptions import ModelNotFound
|
|
17
|
+
from rasa.exceptions import ModelNotFound, ValidationError
|
|
18
18
|
from rasa.shared.constants import (
|
|
19
19
|
ASSISTANT_ID_DEFAULT_VALUE,
|
|
20
20
|
ASSISTANT_ID_KEY,
|
|
@@ -191,16 +191,15 @@ def validate_config_path(
|
|
|
191
191
|
config = rasa.cli.utils.get_validated_path(config, "config", default_config)
|
|
192
192
|
|
|
193
193
|
if not config or not os.path.exists(config):
|
|
194
|
-
|
|
195
|
-
|
|
194
|
+
display_research_study_prompt()
|
|
195
|
+
raise ValidationError(
|
|
196
|
+
code="cli.validate_config_path.does_not_exists",
|
|
196
197
|
config=config,
|
|
197
198
|
event_info=(
|
|
198
199
|
f"The config file '{config}' does not exist. "
|
|
199
200
|
f"Use '--config' to specify a valid config file."
|
|
200
201
|
),
|
|
201
202
|
)
|
|
202
|
-
display_research_study_prompt()
|
|
203
|
-
sys.exit(1)
|
|
204
203
|
|
|
205
204
|
return str(config)
|
|
206
205
|
|
|
@@ -221,8 +220,9 @@ def validate_mandatory_config_keys(
|
|
|
221
220
|
"""
|
|
222
221
|
missing_keys = set(rasa.cli.utils.missing_config_keys(config, mandatory_keys))
|
|
223
222
|
if missing_keys:
|
|
224
|
-
|
|
225
|
-
|
|
223
|
+
display_research_study_prompt()
|
|
224
|
+
raise ValidationError(
|
|
225
|
+
code="cli.validate_mandatory_config_keys.missing_keys",
|
|
226
226
|
config=config,
|
|
227
227
|
missing_keys=missing_keys,
|
|
228
228
|
event_info=(
|
|
@@ -232,8 +232,6 @@ def validate_mandatory_config_keys(
|
|
|
232
232
|
)
|
|
233
233
|
),
|
|
234
234
|
)
|
|
235
|
-
display_research_study_prompt()
|
|
236
|
-
sys.exit(1)
|
|
237
235
|
|
|
238
236
|
return str(config)
|
|
239
237
|
|
|
@@ -287,7 +285,10 @@ def validate_files(
|
|
|
287
285
|
event_info="Encountered empty domain during validation.",
|
|
288
286
|
)
|
|
289
287
|
display_research_study_prompt()
|
|
290
|
-
|
|
288
|
+
raise ValidationError(
|
|
289
|
+
code="cli.validate_files.empty_domain",
|
|
290
|
+
event_info="Encountered empty domain during validation.",
|
|
291
|
+
)
|
|
291
292
|
|
|
292
293
|
valid_domain = _validate_domain(validator)
|
|
293
294
|
valid_nlu = _validate_nlu(validator, fail_on_warnings)
|
|
@@ -295,7 +296,10 @@ def validate_files(
|
|
|
295
296
|
validator, max_history, fail_on_warnings
|
|
296
297
|
)
|
|
297
298
|
valid_flows = validator.verify_flows()
|
|
298
|
-
|
|
299
|
+
if validator.config:
|
|
300
|
+
valid_translations = validator.verify_translations(summary_mode=True)
|
|
301
|
+
else:
|
|
302
|
+
valid_translations = True
|
|
299
303
|
valid_CALM_slot_mappings = validator.validate_CALM_slot_mappings()
|
|
300
304
|
|
|
301
305
|
all_good = (
|
|
@@ -307,7 +311,8 @@ def validate_files(
|
|
|
307
311
|
and valid_CALM_slot_mappings
|
|
308
312
|
)
|
|
309
313
|
|
|
310
|
-
validator.
|
|
314
|
+
if validator.config:
|
|
315
|
+
validator.warn_if_config_mandatory_keys_are_not_set()
|
|
311
316
|
|
|
312
317
|
telemetry.track_validate_files(all_good)
|
|
313
318
|
if not all_good:
|
|
@@ -316,7 +321,10 @@ def validate_files(
|
|
|
316
321
|
event_info="Project validation completed with errors.",
|
|
317
322
|
)
|
|
318
323
|
display_research_study_prompt()
|
|
319
|
-
|
|
324
|
+
raise ValidationError(
|
|
325
|
+
code="cli.validate_files.project_validation_error",
|
|
326
|
+
event_info="Project validation completed with errors.",
|
|
327
|
+
)
|
|
320
328
|
|
|
321
329
|
|
|
322
330
|
def _validate_domain(validator: "Validator") -> bool:
|
|
@@ -354,7 +362,7 @@ def _validate_story_structure(
|
|
|
354
362
|
# Check if a valid setting for `max_history` was given
|
|
355
363
|
if isinstance(max_history, int) and max_history < 1:
|
|
356
364
|
raise argparse.ArgumentTypeError(
|
|
357
|
-
f"The value of `--max-history {max_history}`
|
|
365
|
+
f"The value of `--max-history {max_history}` is not a positive integer."
|
|
358
366
|
)
|
|
359
367
|
|
|
360
368
|
return validator.verify_story_structure(
|
|
@@ -391,7 +399,15 @@ def cancel_cause_not_found(
|
|
|
391
399
|
),
|
|
392
400
|
)
|
|
393
401
|
display_research_study_prompt()
|
|
394
|
-
|
|
402
|
+
raise ValidationError(
|
|
403
|
+
code="cli.path_does_not_exist",
|
|
404
|
+
path=current,
|
|
405
|
+
event_info=(
|
|
406
|
+
f"The path '{current}' does not exist. "
|
|
407
|
+
f"Please make sure to {default_clause} specify it "
|
|
408
|
+
f"with '--{parameter}'."
|
|
409
|
+
),
|
|
410
|
+
)
|
|
395
411
|
|
|
396
412
|
|
|
397
413
|
def parse_last_positional_argument_as_model_path() -> None:
|
|
@@ -50,13 +50,14 @@
|
|
|
50
50
|
"ts-jest": "29.1.1",
|
|
51
51
|
"ts-node": "10.9.1",
|
|
52
52
|
"typescript": "5.0.2",
|
|
53
|
-
"vite": "4.5.
|
|
53
|
+
"vite": "4.5.12"
|
|
54
54
|
},
|
|
55
55
|
"resolutions": {
|
|
56
56
|
"cross-spawn": "7.0.5",
|
|
57
57
|
"dompurify": "3.1.3",
|
|
58
58
|
"braces": "3.0.3",
|
|
59
59
|
"ws": "8.17.1",
|
|
60
|
-
"rollup": "3.29.5"
|
|
60
|
+
"rollup": "3.29.5",
|
|
61
|
+
"@adobe/css-tools": "^4.3.2"
|
|
61
62
|
}
|
|
62
63
|
}
|
|
@@ -7,10 +7,10 @@
|
|
|
7
7
|
resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf"
|
|
8
8
|
integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==
|
|
9
9
|
|
|
10
|
-
"@adobe/css-tools@^4.3.1":
|
|
11
|
-
version "4.3
|
|
12
|
-
resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.
|
|
13
|
-
integrity sha512
|
|
10
|
+
"@adobe/css-tools@^4.3.1", "@adobe/css-tools@^4.3.2":
|
|
11
|
+
version "4.4.3"
|
|
12
|
+
resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.4.3.tgz#beebbefb0264fdeb32d3052acae0e0d94315a9a2"
|
|
13
|
+
integrity sha512-VQKMkwriZbaOgVCby1UDY/LDk5fIjhQicCvVPFqfe+69fWaPWydbWJ3wRt59/YzIwda1I81loas3oCoHxnqvdA==
|
|
14
14
|
|
|
15
15
|
"@ampproject/remapping@^2.2.0":
|
|
16
16
|
version "2.2.1"
|
|
@@ -6064,10 +6064,10 @@ v8-to-istanbul@^9.0.1:
|
|
|
6064
6064
|
"@types/istanbul-lib-coverage" "^2.0.1"
|
|
6065
6065
|
convert-source-map "^2.0.0"
|
|
6066
6066
|
|
|
6067
|
-
vite@4.5.
|
|
6068
|
-
version "4.5.
|
|
6069
|
-
resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.
|
|
6070
|
-
integrity sha512-
|
|
6067
|
+
vite@4.5.12:
|
|
6068
|
+
version "4.5.12"
|
|
6069
|
+
resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.12.tgz#48f48dbcf789722765e91bc32a99cb66c628eadc"
|
|
6070
|
+
integrity sha512-qrMwavANtSz91nDy3zEiUHMtL09x0mniQsSMvDkNxuCBM1W5vriJ22hEmwTth6DhLSWsZnHBT0yHFAQXt6efGA==
|
|
6071
6071
|
dependencies:
|
|
6072
6072
|
esbuild "^0.18.10"
|
|
6073
6073
|
postcss "^8.4.27"
|
|
@@ -50,6 +50,7 @@ from rasa.engine.graph import ExecutionContext
|
|
|
50
50
|
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
|
|
51
51
|
from rasa.engine.storage.resource import Resource
|
|
52
52
|
from rasa.engine.storage.storage import ModelStorage
|
|
53
|
+
from rasa.exceptions import EnterpriseSearchPolicyError
|
|
53
54
|
from rasa.graph_components.providers.forms_provider import Forms
|
|
54
55
|
from rasa.graph_components.providers.responses_provider import Responses
|
|
55
56
|
from rasa.shared.constants import (
|
|
@@ -82,7 +83,6 @@ from rasa.shared.providers.embedding._langchain_embedding_client_adapter import
|
|
|
82
83
|
_LangchainEmbeddingClientAdapter,
|
|
83
84
|
)
|
|
84
85
|
from rasa.shared.providers.llm.llm_response import LLMResponse, measure_llm_latency
|
|
85
|
-
from rasa.shared.utils.cli import print_error_and_exit
|
|
86
86
|
from rasa.shared.utils.constants import (
|
|
87
87
|
LOG_COMPONENT_SOURCE_METHOD_FINGERPRINT_ADDON,
|
|
88
88
|
LOG_COMPONENT_SOURCE_METHOD_INIT,
|
|
@@ -343,14 +343,11 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
|
|
|
343
343
|
try:
|
|
344
344
|
embeddings = self._create_plain_embedder(self.embeddings_config)
|
|
345
345
|
except (ValidationError, Exception) as e:
|
|
346
|
-
|
|
347
|
-
"enterprise_search_policy.train
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
print_error_and_exit(
|
|
352
|
-
"Unable to create embedder. Please make sure you specified the "
|
|
353
|
-
f"required environment variables. Error: {e}"
|
|
346
|
+
raise EnterpriseSearchPolicyError(
|
|
347
|
+
code="core.policies.enterprise_search_policy.train"
|
|
348
|
+
".embedder_instantiation_failed",
|
|
349
|
+
event_info=f"Unable to create embedder. Please make sure you "
|
|
350
|
+
f"specified the required environment variables. Error: {e}",
|
|
354
351
|
)
|
|
355
352
|
|
|
356
353
|
if self.vector_store_type == DEFAULT_VECTOR_STORE_TYPE:
|
|
@@ -788,23 +785,23 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
|
|
|
788
785
|
"Please specify a valid path to the documents source directory in the "
|
|
789
786
|
"vector_store configuration."
|
|
790
787
|
)
|
|
791
|
-
|
|
792
|
-
"enterprise_search_policy.train
|
|
793
|
-
|
|
788
|
+
error_code = (
|
|
789
|
+
"core.policies.enterprise_search_policy.train"
|
|
790
|
+
".faiss.invalid_source_directory"
|
|
794
791
|
)
|
|
795
|
-
|
|
792
|
+
raise EnterpriseSearchPolicyError(code=error_code, event_info=error_message)
|
|
796
793
|
|
|
797
|
-
docs = glob.glob(os.path.join(docs_folder, "*.txt"), recursive=True)
|
|
794
|
+
docs = glob.glob(os.path.join(docs_folder, "**", "*.txt"), recursive=True)
|
|
798
795
|
if not docs or len(docs) < 1:
|
|
799
796
|
error_message = (
|
|
800
797
|
f"Document source directory is empty: '{docs_folder}'. "
|
|
801
798
|
"Please add documents to this directory or specify a different one."
|
|
802
799
|
)
|
|
803
|
-
|
|
804
|
-
"enterprise_search_policy.train
|
|
805
|
-
|
|
800
|
+
error_code = (
|
|
801
|
+
"core.policies.enterprise_search_policy.train"
|
|
802
|
+
".faiss.source_directory_empty"
|
|
806
803
|
)
|
|
807
|
-
|
|
804
|
+
raise EnterpriseSearchPolicyError(code=error_code, event_info=error_message)
|
|
808
805
|
|
|
809
806
|
@classmethod
|
|
810
807
|
def load(
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import re
|
|
2
|
-
import sys
|
|
3
2
|
from functools import lru_cache
|
|
4
3
|
from typing import Any, Callable, Dict, List, Optional, Type, Union
|
|
5
4
|
|
|
@@ -19,6 +18,7 @@ from rasa.dialogue_understanding.commands import (
|
|
|
19
18
|
)
|
|
20
19
|
from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
|
|
21
20
|
from rasa.dialogue_understanding.commands.utils import start_flow_by_name
|
|
21
|
+
from rasa.exceptions import ValidationError
|
|
22
22
|
from rasa.shared.core.flows import FlowsList
|
|
23
23
|
|
|
24
24
|
structlogger = structlog.get_logger()
|
|
@@ -87,16 +87,13 @@ def validate_custom_commands(command_classes: List[Type[PromptCommand]]) -> None
|
|
|
87
87
|
]
|
|
88
88
|
|
|
89
89
|
if clz_not_inheriting_from_command_clz:
|
|
90
|
-
|
|
91
|
-
"command_parser.validate_custom_commands.invalid_command",
|
|
90
|
+
raise ValidationError(
|
|
91
|
+
code="command_parser.validate_custom_commands.invalid_command",
|
|
92
|
+
event_info="The additional command classes must be a subclass of the "
|
|
93
|
+
"'Command' class. Please refer to the class in "
|
|
94
|
+
"`rasa.dialogue_understanding.commands.command.Command`",
|
|
92
95
|
invalid_commands=clz_not_inheriting_from_command_clz,
|
|
93
|
-
event_info=(
|
|
94
|
-
"The additional command classes must be a subclass of the 'Command' "
|
|
95
|
-
"class. Please refer to the class in "
|
|
96
|
-
"`rasa.dialogue_understanding.commands.command.Command`"
|
|
97
|
-
),
|
|
98
96
|
)
|
|
99
|
-
sys.exit(1)
|
|
100
97
|
|
|
101
98
|
clz_not_adhering_to_prompt_command_protocol = [
|
|
102
99
|
command_clz.__name__
|
|
@@ -105,16 +102,15 @@ def validate_custom_commands(command_classes: List[Type[PromptCommand]]) -> None
|
|
|
105
102
|
]
|
|
106
103
|
|
|
107
104
|
if clz_not_adhering_to_prompt_command_protocol:
|
|
108
|
-
|
|
109
|
-
"command_parser.validate_custom_commands.invalid_command",
|
|
110
|
-
invalid_commands=clz_not_adhering_to_prompt_command_protocol,
|
|
105
|
+
raise ValidationError(
|
|
106
|
+
code="command_parser.validate_custom_commands.invalid_command",
|
|
111
107
|
event_info=(
|
|
112
108
|
"The additional command classes must adhere to the 'PromptCommand' "
|
|
113
109
|
"protocol. Please refer to the protocol in "
|
|
114
110
|
"`rasa.dialogue_understanding.commands.prompt_command.PromptCommand`"
|
|
115
111
|
),
|
|
112
|
+
invalid_commands=clz_not_adhering_to_prompt_command_protocol,
|
|
116
113
|
)
|
|
117
|
-
sys.exit(1)
|
|
118
114
|
|
|
119
115
|
|
|
120
116
|
def parse_commands(
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import argparse
|
|
2
|
-
import sys
|
|
3
2
|
from typing import List, Optional
|
|
4
3
|
|
|
5
4
|
import structlog
|
|
@@ -9,6 +8,7 @@ from rasa.dialogue_understanding_test.constants import (
|
|
|
9
8
|
PLACEHOLDER_GENERATED_ANSWER_TEMPLATE,
|
|
10
9
|
)
|
|
11
10
|
from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
|
|
11
|
+
from rasa.exceptions import ValidationError
|
|
12
12
|
from rasa.shared.core.domain import Domain
|
|
13
13
|
|
|
14
14
|
structlogger = structlog.get_logger()
|
|
@@ -28,14 +28,14 @@ def validate_cli_arguments(args: argparse.Namespace) -> None:
|
|
|
28
28
|
args.remote_storage
|
|
29
29
|
and args.remote_storage.lower() not in supported_remote_storages
|
|
30
30
|
):
|
|
31
|
-
|
|
32
|
-
"dialogue_understanding_test.validate_cli_arguments
|
|
31
|
+
raise ValidationError(
|
|
32
|
+
code="dialogue_understanding_test.validate_cli_arguments"
|
|
33
|
+
".invalid_remote_storage",
|
|
33
34
|
event_info=(
|
|
34
35
|
f"Invalid remote storage option - '{args.remote_storage}'. Supported "
|
|
35
36
|
f"options are: {supported_remote_storages}"
|
|
36
37
|
),
|
|
37
38
|
)
|
|
38
|
-
sys.exit(1)
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
def validate_test_cases(
|
|
@@ -48,11 +48,10 @@ def validate_test_cases(
|
|
|
48
48
|
domain: Domain of the assistant.
|
|
49
49
|
"""
|
|
50
50
|
if not domain:
|
|
51
|
-
|
|
52
|
-
"dialogue_understanding_test.validate_test_cases.no_domain",
|
|
51
|
+
raise ValidationError(
|
|
52
|
+
code="dialogue_understanding_test.validate_test_cases.no_domain",
|
|
53
53
|
event_info="No domain found. Retrain the model with a valid domain.",
|
|
54
54
|
)
|
|
55
|
-
sys.exit(1)
|
|
56
55
|
|
|
57
56
|
# Retrieve all valid templates from the domain
|
|
58
57
|
valid_templates = domain.utterances_for_response
|
|
@@ -64,8 +63,9 @@ def validate_test_cases(
|
|
|
64
63
|
for step in test_case.steps:
|
|
65
64
|
if step.actor == ACTOR_BOT and step.template:
|
|
66
65
|
if step.template not in valid_templates:
|
|
67
|
-
|
|
68
|
-
"dialogue_understanding_test.validate_test_cases
|
|
66
|
+
raise ValidationError(
|
|
67
|
+
code="dialogue_understanding_test.validate_test_cases"
|
|
68
|
+
".invalid_template",
|
|
69
69
|
event_info=(
|
|
70
70
|
f"Invalid bot utterance template '{step.template}' in test "
|
|
71
71
|
f"case '{test_case.name}' at line {step.line}. Please "
|
|
@@ -74,4 +74,3 @@ def validate_test_cases(
|
|
|
74
74
|
test_case=test_case.name,
|
|
75
75
|
template=step.template,
|
|
76
76
|
)
|
|
77
|
-
sys.exit(1)
|
rasa/e2e_test/e2e_config.py
CHANGED
|
@@ -67,26 +67,33 @@ class LLMJudgeConfig(BaseModel):
|
|
|
67
67
|
@classmethod
|
|
68
68
|
def from_dict(cls, config_data: Dict[str, Any]) -> LLMJudgeConfig:
|
|
69
69
|
"""Loads the configuration from a dictionary."""
|
|
70
|
-
embeddings = config_data.pop(EMBEDDINGS_CONFIG_KEY,
|
|
70
|
+
embeddings = config_data.pop(EMBEDDINGS_CONFIG_KEY, {})
|
|
71
71
|
llm_config = config_data.pop("llm", {})
|
|
72
72
|
|
|
73
73
|
llm_config = resolve_model_client_config(llm_config)
|
|
74
|
-
llm_config,
|
|
74
|
+
llm_config, llm_extra_parameters = cls.extract_attributes(llm_config)
|
|
75
75
|
llm_config = combine_custom_and_default_config(
|
|
76
76
|
llm_config, cls.get_default_llm_config()
|
|
77
77
|
)
|
|
78
78
|
embeddings_config = resolve_model_client_config(embeddings)
|
|
79
|
+
embeddings_config, embeddings_extra_parameters = cls.extract_attributes(
|
|
80
|
+
embeddings_config
|
|
81
|
+
)
|
|
79
82
|
|
|
80
83
|
return LLMJudgeConfig(
|
|
81
|
-
llm_config=BaseModelConfig(
|
|
82
|
-
|
|
84
|
+
llm_config=BaseModelConfig(
|
|
85
|
+
extra_parameters=llm_extra_parameters, **llm_config
|
|
86
|
+
),
|
|
87
|
+
embeddings=BaseModelConfig(
|
|
88
|
+
extra_parameters=embeddings_extra_parameters, **embeddings_config
|
|
89
|
+
)
|
|
83
90
|
if embeddings_config
|
|
84
91
|
else None,
|
|
85
92
|
)
|
|
86
93
|
|
|
87
94
|
@classmethod
|
|
88
95
|
def extract_attributes(
|
|
89
|
-
cls,
|
|
96
|
+
cls, config: Dict[str, Any]
|
|
90
97
|
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
|
91
98
|
"""Extract the expected fields from the configuration."""
|
|
92
99
|
required_config = {}
|
|
@@ -96,22 +103,22 @@ class LLMJudgeConfig(BaseModel):
|
|
|
96
103
|
MODEL_CONFIG_KEY,
|
|
97
104
|
]
|
|
98
105
|
|
|
99
|
-
if PROVIDER_CONFIG_KEY in
|
|
106
|
+
if PROVIDER_CONFIG_KEY in config:
|
|
100
107
|
required_config = {
|
|
101
|
-
expected_field:
|
|
108
|
+
expected_field: config.pop(expected_field, None)
|
|
102
109
|
for expected_field in expected_fields
|
|
103
110
|
}
|
|
104
111
|
|
|
105
|
-
elif MODELS_CONFIG_KEY in
|
|
106
|
-
|
|
112
|
+
elif MODELS_CONFIG_KEY in config:
|
|
113
|
+
config = config.pop(MODELS_CONFIG_KEY)[0]
|
|
107
114
|
|
|
108
115
|
required_config = {
|
|
109
|
-
expected_field:
|
|
116
|
+
expected_field: config.pop(expected_field, None)
|
|
110
117
|
for expected_field in expected_fields
|
|
111
118
|
}
|
|
112
119
|
|
|
113
120
|
clean_config = clean_up_config(required_config)
|
|
114
|
-
return clean_config,
|
|
121
|
+
return clean_config, config
|
|
115
122
|
|
|
116
123
|
@property
|
|
117
124
|
def llm_config_as_dict(self) -> Dict[str, Any]:
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import sys
|
|
2
1
|
from pathlib import Path
|
|
3
2
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
|
4
3
|
|
|
5
4
|
import structlog
|
|
6
5
|
|
|
7
|
-
import rasa.shared.utils.io
|
|
8
6
|
from rasa.e2e_test.constants import SCHEMA_FILE_PATH
|
|
9
7
|
from rasa.e2e_test.e2e_test_case import Fixture, Metadata
|
|
10
|
-
from rasa.exceptions import ModelNotFound
|
|
8
|
+
from rasa.exceptions import ModelNotFound, ValidationError
|
|
11
9
|
from rasa.shared.utils.yaml import read_schema_file
|
|
12
10
|
|
|
13
11
|
if TYPE_CHECKING:
|
|
@@ -20,13 +18,11 @@ structlogger = structlog.get_logger()
|
|
|
20
18
|
def validate_path_to_test_cases(path: str) -> None:
|
|
21
19
|
"""Validate that path to test cases exists."""
|
|
22
20
|
if not Path(path).exists():
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
f"
|
|
26
|
-
f"
|
|
27
|
-
UserWarning,
|
|
21
|
+
raise ValidationError(
|
|
22
|
+
code="e2e_test.utils.validation.invalid_test_case_path",
|
|
23
|
+
event_info=f"Path to test cases does not exist: {path}. "
|
|
24
|
+
f"Please provide a valid path to test cases. ",
|
|
28
25
|
)
|
|
29
|
-
sys.exit(1)
|
|
30
26
|
|
|
31
27
|
|
|
32
28
|
def validate_test_case(
|
|
@@ -48,13 +44,11 @@ def validate_test_case(
|
|
|
48
44
|
SystemExit: If the test case, fixtures, or metadata are not defined.
|
|
49
45
|
"""
|
|
50
46
|
if test_case_name and not input_test_cases:
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
f"
|
|
54
|
-
f"
|
|
55
|
-
UserWarning,
|
|
47
|
+
raise ValidationError(
|
|
48
|
+
code="e2e_test.utils.validation.invalid_test_case",
|
|
49
|
+
event_info=f"Test case does not exist: {test_case_name}. "
|
|
50
|
+
f"Please check for typos and provide a valid test case name. ",
|
|
56
51
|
)
|
|
57
|
-
sys.exit(1)
|
|
58
52
|
|
|
59
53
|
all_good = True
|
|
60
54
|
for test_case in input_test_cases:
|
|
@@ -63,7 +57,11 @@ def validate_test_case(
|
|
|
63
57
|
all_good = all_good and all_good_fixtures and all_good_metadata
|
|
64
58
|
|
|
65
59
|
if not all_good:
|
|
66
|
-
|
|
60
|
+
raise ValidationError(
|
|
61
|
+
code="e2e_test.utils.validation.missing_fixtures_or_metadata",
|
|
62
|
+
event_info="Some fixtures and/or metadata are missing - "
|
|
63
|
+
"see previous logs for additional details.",
|
|
64
|
+
)
|
|
67
65
|
|
|
68
66
|
|
|
69
67
|
def validate_test_case_fixtures(
|
|
@@ -88,7 +86,7 @@ def validate_test_case_fixtures(
|
|
|
88
86
|
for fixture_name in test_case.fixture_names:
|
|
89
87
|
if fixture_name not in fixtures:
|
|
90
88
|
structlogger.error(
|
|
91
|
-
"validation.validate_test_case_fixtures",
|
|
89
|
+
"e2e_test.utils.validation.validate_test_case_fixtures",
|
|
92
90
|
event_info=(
|
|
93
91
|
f"Fixture '{fixture_name}' referenced in the "
|
|
94
92
|
f"test case '{test_case.name}' is not defined."
|
|
@@ -117,7 +115,7 @@ def validate_test_case_metadata(
|
|
|
117
115
|
all_good = True
|
|
118
116
|
if test_case.metadata_name and test_case.metadata_name not in metadata:
|
|
119
117
|
structlogger.error(
|
|
120
|
-
"validation.validate_test_case_metadata.test_case_metadata",
|
|
118
|
+
"e2e_test.utils.validation.validate_test_case_metadata.test_case_metadata",
|
|
121
119
|
event_info=(
|
|
122
120
|
f"Metadata '{test_case.metadata_name}' referenced in "
|
|
123
121
|
f"the test case '{test_case.name}' is not defined."
|
|
@@ -128,7 +126,7 @@ def validate_test_case_metadata(
|
|
|
128
126
|
for step in test_case.steps:
|
|
129
127
|
if step.metadata_name and step.metadata_name not in metadata:
|
|
130
128
|
structlogger.error(
|
|
131
|
-
"validation.validate_test_case_metadata.step_metadata",
|
|
129
|
+
"e2e_test.utils.validation.validate_test_case_metadata.step_metadata",
|
|
132
130
|
event_info=(
|
|
133
131
|
f"Metadata '{step.metadata_name}' referenced in the "
|
|
134
132
|
f"step of the test case '{test_case.name}' is not defined."
|