rasa-pro 3.9.17__py3-none-any.whl → 3.10.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rasa-pro might be problematic. Click here for more details.

Files changed (187) hide show
  1. README.md +5 -37
  2. rasa/__init__.py +1 -2
  3. rasa/__main__.py +5 -0
  4. rasa/anonymization/anonymization_rule_executor.py +2 -2
  5. rasa/api.py +26 -22
  6. rasa/cli/arguments/data.py +27 -2
  7. rasa/cli/arguments/default_arguments.py +25 -3
  8. rasa/cli/arguments/run.py +9 -9
  9. rasa/cli/arguments/train.py +2 -0
  10. rasa/cli/data.py +70 -8
  11. rasa/cli/e2e_test.py +108 -433
  12. rasa/cli/interactive.py +1 -0
  13. rasa/cli/llm_fine_tuning.py +395 -0
  14. rasa/cli/project_templates/calm/endpoints.yml +1 -1
  15. rasa/cli/project_templates/tutorial/endpoints.yml +1 -1
  16. rasa/cli/run.py +14 -13
  17. rasa/cli/scaffold.py +10 -8
  18. rasa/cli/train.py +8 -7
  19. rasa/cli/utils.py +15 -0
  20. rasa/constants.py +7 -1
  21. rasa/core/actions/action.py +98 -49
  22. rasa/core/actions/action_run_slot_rejections.py +4 -1
  23. rasa/core/actions/custom_action_executor.py +9 -6
  24. rasa/core/actions/direct_custom_actions_executor.py +80 -0
  25. rasa/core/actions/e2e_stub_custom_action_executor.py +68 -0
  26. rasa/core/actions/grpc_custom_action_executor.py +2 -2
  27. rasa/core/actions/http_custom_action_executor.py +6 -5
  28. rasa/core/agent.py +21 -17
  29. rasa/core/channels/__init__.py +2 -0
  30. rasa/core/channels/audiocodes.py +1 -16
  31. rasa/core/channels/voice_aware/__init__.py +0 -0
  32. rasa/core/channels/voice_aware/jambonz.py +103 -0
  33. rasa/core/channels/voice_aware/jambonz_protocol.py +344 -0
  34. rasa/core/channels/voice_aware/utils.py +20 -0
  35. rasa/core/channels/voice_native/__init__.py +0 -0
  36. rasa/core/constants.py +6 -1
  37. rasa/core/featurizers/single_state_featurizer.py +1 -22
  38. rasa/core/featurizers/tracker_featurizers.py +18 -115
  39. rasa/core/information_retrieval/faiss.py +7 -4
  40. rasa/core/information_retrieval/information_retrieval.py +8 -0
  41. rasa/core/information_retrieval/milvus.py +9 -2
  42. rasa/core/information_retrieval/qdrant.py +1 -1
  43. rasa/core/nlg/contextual_response_rephraser.py +32 -10
  44. rasa/core/nlg/summarize.py +4 -3
  45. rasa/core/policies/enterprise_search_policy.py +100 -44
  46. rasa/core/policies/flows/flow_executor.py +155 -98
  47. rasa/core/policies/intentless_policy.py +52 -28
  48. rasa/core/policies/ted_policy.py +33 -58
  49. rasa/core/policies/unexpected_intent_policy.py +7 -15
  50. rasa/core/processor.py +15 -46
  51. rasa/core/run.py +5 -4
  52. rasa/core/tracker_store.py +8 -4
  53. rasa/core/utils.py +45 -56
  54. rasa/dialogue_understanding/coexistence/llm_based_router.py +45 -12
  55. rasa/dialogue_understanding/commands/__init__.py +4 -0
  56. rasa/dialogue_understanding/commands/change_flow_command.py +0 -6
  57. rasa/dialogue_understanding/commands/session_start_command.py +59 -0
  58. rasa/dialogue_understanding/commands/set_slot_command.py +1 -5
  59. rasa/dialogue_understanding/commands/utils.py +38 -0
  60. rasa/dialogue_understanding/generator/constants.py +10 -3
  61. rasa/dialogue_understanding/generator/flow_retrieval.py +14 -5
  62. rasa/dialogue_understanding/generator/llm_based_command_generator.py +12 -2
  63. rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py +106 -87
  64. rasa/dialogue_understanding/generator/nlu_command_adapter.py +28 -6
  65. rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py +90 -37
  66. rasa/dialogue_understanding/patterns/default_flows_for_patterns.yml +15 -15
  67. rasa/dialogue_understanding/patterns/session_start.py +37 -0
  68. rasa/dialogue_understanding/processor/command_processor.py +13 -14
  69. rasa/e2e_test/aggregate_test_stats_calculator.py +124 -0
  70. rasa/e2e_test/assertions.py +1181 -0
  71. rasa/e2e_test/assertions_schema.yml +106 -0
  72. rasa/e2e_test/constants.py +20 -0
  73. rasa/e2e_test/e2e_config.py +220 -0
  74. rasa/e2e_test/e2e_config_schema.yml +26 -0
  75. rasa/e2e_test/e2e_test_case.py +131 -8
  76. rasa/e2e_test/e2e_test_converter.py +363 -0
  77. rasa/e2e_test/e2e_test_converter_prompt.jinja2 +70 -0
  78. rasa/e2e_test/e2e_test_coverage_report.py +364 -0
  79. rasa/e2e_test/e2e_test_result.py +26 -6
  80. rasa/e2e_test/e2e_test_runner.py +498 -73
  81. rasa/e2e_test/e2e_test_schema.yml +96 -0
  82. rasa/e2e_test/pykwalify_extensions.py +39 -0
  83. rasa/e2e_test/stub_custom_action.py +70 -0
  84. rasa/e2e_test/utils/__init__.py +0 -0
  85. rasa/e2e_test/utils/e2e_yaml_utils.py +55 -0
  86. rasa/e2e_test/utils/io.py +596 -0
  87. rasa/e2e_test/utils/validation.py +80 -0
  88. rasa/engine/recipes/default_components.py +0 -2
  89. rasa/engine/storage/local_model_storage.py +0 -1
  90. rasa/env.py +9 -0
  91. rasa/llm_fine_tuning/__init__.py +0 -0
  92. rasa/llm_fine_tuning/annotation_module.py +241 -0
  93. rasa/llm_fine_tuning/conversations.py +144 -0
  94. rasa/llm_fine_tuning/llm_data_preparation_module.py +178 -0
  95. rasa/llm_fine_tuning/notebooks/unsloth_finetuning.ipynb +407 -0
  96. rasa/llm_fine_tuning/paraphrasing/__init__.py +0 -0
  97. rasa/llm_fine_tuning/paraphrasing/conversation_rephraser.py +281 -0
  98. rasa/llm_fine_tuning/paraphrasing/default_rephrase_prompt_template.jina2 +44 -0
  99. rasa/llm_fine_tuning/paraphrasing/rephrase_validator.py +121 -0
  100. rasa/llm_fine_tuning/paraphrasing/rephrased_user_message.py +10 -0
  101. rasa/llm_fine_tuning/paraphrasing_module.py +128 -0
  102. rasa/llm_fine_tuning/storage.py +174 -0
  103. rasa/llm_fine_tuning/train_test_split_module.py +441 -0
  104. rasa/model_training.py +48 -16
  105. rasa/nlu/classifiers/diet_classifier.py +25 -38
  106. rasa/nlu/classifiers/logistic_regression_classifier.py +9 -44
  107. rasa/nlu/classifiers/sklearn_intent_classifier.py +16 -37
  108. rasa/nlu/extractors/crf_entity_extractor.py +50 -93
  109. rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py +45 -78
  110. rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py +17 -52
  111. rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py +3 -5
  112. rasa/nlu/persistor.py +129 -32
  113. rasa/server.py +45 -10
  114. rasa/shared/constants.py +63 -15
  115. rasa/shared/core/domain.py +15 -12
  116. rasa/shared/core/events.py +28 -2
  117. rasa/shared/core/flows/flow.py +208 -13
  118. rasa/shared/core/flows/flow_path.py +84 -0
  119. rasa/shared/core/flows/flows_list.py +28 -10
  120. rasa/shared/core/flows/flows_yaml_schema.json +269 -193
  121. rasa/shared/core/flows/validation.py +112 -25
  122. rasa/shared/core/flows/yaml_flows_io.py +149 -10
  123. rasa/shared/core/trackers.py +6 -0
  124. rasa/shared/core/training_data/visualization.html +2 -2
  125. rasa/shared/exceptions.py +4 -0
  126. rasa/shared/importers/importer.py +60 -11
  127. rasa/shared/importers/remote_importer.py +196 -0
  128. rasa/shared/nlu/constants.py +2 -0
  129. rasa/shared/nlu/training_data/features.py +2 -120
  130. rasa/shared/providers/_configs/__init__.py +0 -0
  131. rasa/shared/providers/_configs/azure_openai_client_config.py +181 -0
  132. rasa/shared/providers/_configs/client_config.py +57 -0
  133. rasa/shared/providers/_configs/default_litellm_client_config.py +130 -0
  134. rasa/shared/providers/_configs/huggingface_local_embedding_client_config.py +234 -0
  135. rasa/shared/providers/_configs/openai_client_config.py +175 -0
  136. rasa/shared/providers/_configs/self_hosted_llm_client_config.py +171 -0
  137. rasa/shared/providers/_configs/utils.py +101 -0
  138. rasa/shared/providers/_ssl_verification_utils.py +124 -0
  139. rasa/shared/providers/embedding/__init__.py +0 -0
  140. rasa/shared/providers/embedding/_base_litellm_embedding_client.py +254 -0
  141. rasa/shared/providers/embedding/_langchain_embedding_client_adapter.py +74 -0
  142. rasa/shared/providers/embedding/azure_openai_embedding_client.py +277 -0
  143. rasa/shared/providers/embedding/default_litellm_embedding_client.py +102 -0
  144. rasa/shared/providers/embedding/embedding_client.py +90 -0
  145. rasa/shared/providers/embedding/embedding_response.py +41 -0
  146. rasa/shared/providers/embedding/huggingface_local_embedding_client.py +191 -0
  147. rasa/shared/providers/embedding/openai_embedding_client.py +172 -0
  148. rasa/shared/providers/llm/__init__.py +0 -0
  149. rasa/shared/providers/llm/_base_litellm_client.py +227 -0
  150. rasa/shared/providers/llm/azure_openai_llm_client.py +338 -0
  151. rasa/shared/providers/llm/default_litellm_llm_client.py +84 -0
  152. rasa/shared/providers/llm/llm_client.py +76 -0
  153. rasa/shared/providers/llm/llm_response.py +50 -0
  154. rasa/shared/providers/llm/openai_llm_client.py +155 -0
  155. rasa/shared/providers/llm/self_hosted_llm_client.py +169 -0
  156. rasa/shared/providers/mappings.py +75 -0
  157. rasa/shared/utils/cli.py +30 -0
  158. rasa/shared/utils/io.py +65 -3
  159. rasa/shared/utils/llm.py +223 -200
  160. rasa/shared/utils/yaml.py +122 -7
  161. rasa/studio/download.py +19 -13
  162. rasa/studio/train.py +2 -3
  163. rasa/studio/upload.py +2 -3
  164. rasa/telemetry.py +113 -58
  165. rasa/tracing/config.py +2 -3
  166. rasa/tracing/instrumentation/attribute_extractors.py +29 -17
  167. rasa/tracing/instrumentation/instrumentation.py +4 -47
  168. rasa/utils/common.py +18 -19
  169. rasa/utils/endpoints.py +7 -4
  170. rasa/utils/io.py +66 -0
  171. rasa/utils/json_utils.py +60 -0
  172. rasa/utils/licensing.py +9 -1
  173. rasa/utils/ml_utils.py +4 -2
  174. rasa/utils/tensorflow/model_data.py +193 -2
  175. rasa/validator.py +195 -1
  176. rasa/version.py +1 -1
  177. {rasa_pro-3.9.17.dist-info → rasa_pro-3.10.3.dist-info}/METADATA +25 -51
  178. {rasa_pro-3.9.17.dist-info → rasa_pro-3.10.3.dist-info}/RECORD +183 -119
  179. rasa/nlu/classifiers/llm_intent_classifier.py +0 -519
  180. rasa/shared/providers/openai/clients.py +0 -43
  181. rasa/shared/providers/openai/session_handler.py +0 -110
  182. rasa/utils/tensorflow/feature_array.py +0 -366
  183. /rasa/{shared/providers/openai → cli/project_templates/tutorial/actions}/__init__.py +0 -0
  184. /rasa/cli/project_templates/tutorial/{actions.py → actions/actions.py} +0 -0
  185. {rasa_pro-3.9.17.dist-info → rasa_pro-3.10.3.dist-info}/NOTICE +0 -0
  186. {rasa_pro-3.9.17.dist-info → rasa_pro-3.10.3.dist-info}/WHEEL +0 -0
  187. {rasa_pro-3.9.17.dist-info → rasa_pro-3.10.3.dist-info}/entry_points.txt +0 -0
@@ -8,13 +8,14 @@ import tiktoken
8
8
  from jinja2 import Template
9
9
  from langchain.docstore.document import Document
10
10
  from langchain.schema.embeddings import Embeddings
11
- from langchain.vectorstores import FAISS
11
+ from langchain_community.vectorstores.faiss import FAISS
12
12
 
13
13
  import rasa.shared.utils.io
14
14
  from rasa import telemetry
15
15
  from rasa.core.constants import (
16
16
  CHAT_POLICY_PRIORITY,
17
17
  POLICY_PRIORITY,
18
+ UTTER_SOURCE_METADATA_KEY,
18
19
  )
19
20
  from rasa.core.policies.policy import Policy, PolicyPrediction, SupportedData
20
21
  from rasa.dialogue_understanding.stack.frames import (
@@ -27,7 +28,17 @@ from rasa.engine.storage.resource import Resource
27
28
  from rasa.engine.storage.storage import ModelStorage
28
29
  from rasa.graph_components.providers.forms_provider import Forms
29
30
  from rasa.graph_components.providers.responses_provider import Responses
30
- from rasa.shared.constants import REQUIRED_SLOTS_KEY
31
+ from rasa.shared.constants import (
32
+ REQUIRED_SLOTS_KEY,
33
+ EMBEDDINGS_CONFIG_KEY,
34
+ LLM_CONFIG_KEY,
35
+ MODEL_CONFIG_KEY,
36
+ MODEL_NAME_CONFIG_KEY,
37
+ PROMPT_CONFIG_KEY,
38
+ PROVIDER_CONFIG_KEY,
39
+ OPENAI_PROVIDER,
40
+ TIMEOUT_CONFIG_KEY,
41
+ )
31
42
  from rasa.shared.core.constants import ACTION_LISTEN_NAME
32
43
  from rasa.shared.core.domain import KEY_RESPONSES_TEXT, Domain
33
44
  from rasa.shared.core.events import (
@@ -42,6 +53,10 @@ from rasa.shared.core.trackers import DialogueStateTracker
42
53
  from rasa.shared.exceptions import FileIOException, RasaCoreException
43
54
  from rasa.shared.nlu.constants import PREDICTED_CONFIDENCE_KEY
44
55
  from rasa.shared.nlu.training_data.training_data import TrainingData
56
+ from rasa.shared.providers.embedding._langchain_embedding_client_adapter import (
57
+ _LangchainEmbeddingClientAdapter,
58
+ )
59
+ from rasa.shared.providers.llm.llm_client import LLMClient
45
60
  from rasa.shared.utils.io import deep_container_fingerprint
46
61
  from rasa.shared.utils.llm import (
47
62
  AI,
@@ -55,6 +70,7 @@ from rasa.shared.utils.llm import (
55
70
  llm_factory,
56
71
  sanitize_message_for_prompt,
57
72
  tracker_as_readable_transcript,
73
+ try_instantiate_llm_client,
58
74
  )
59
75
  from rasa.utils.ml_utils import (
60
76
  extract_ai_response_examples,
@@ -68,7 +84,6 @@ from rasa.utils.log_utils import log_llm
68
84
 
69
85
  if TYPE_CHECKING:
70
86
  from rasa.core.featurizers.tracker_featurizers import TrackerFeaturizer
71
- from langchain.llms.base import BaseLLM
72
87
 
73
88
  structlogger = structlog.get_logger()
74
89
 
@@ -87,18 +102,16 @@ MAX_NUMBER_OF_TOKENS_FOR_SAMPLES = 900
87
102
  # the config property name for the confidence of the nlu prediction
88
103
  NLU_ABSTENTION_THRESHOLD = "nlu_abstention_threshold"
89
104
 
90
- PROMPT = "prompt"
91
-
92
105
  DEFAULT_LLM_CONFIG = {
93
- "_type": "openai",
94
- "request_timeout": 5,
106
+ PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
107
+ MODEL_CONFIG_KEY: DEFAULT_OPENAI_CHAT_MODEL_NAME,
95
108
  "temperature": 0.0,
96
- "model_name": DEFAULT_OPENAI_CHAT_MODEL_NAME,
97
109
  "max_tokens": DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
110
+ TIMEOUT_CONFIG_KEY: 5,
98
111
  }
99
112
 
100
113
  DEFAULT_EMBEDDINGS_CONFIG = {
101
- "_type": "openai",
114
+ PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
102
115
  "model": DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
103
116
  }
104
117
 
@@ -106,8 +119,6 @@ DEFAULT_INTENTLESS_PROMPT_TEMPLATE = importlib.resources.open_text(
106
119
  "rasa.core.policies", "intentless_prompt_template.jinja2"
107
120
  ).name
108
121
 
109
- EMBEDDINGS_CONFIG_KEY = "embeddings"
110
- LLM_CONFIG_KEY = "llm"
111
122
  INTENTLESS_PROMPT_TEMPLATE_FILE_NAME = "intentless_policy_prompt.jinja2"
112
123
 
113
124
 
@@ -366,7 +377,7 @@ class IntentlessPolicy(Policy):
366
377
  NLU_ABSTENTION_THRESHOLD: 0.9,
367
378
  LLM_CONFIG_KEY: DEFAULT_LLM_CONFIG,
368
379
  EMBEDDINGS_CONFIG_KEY: DEFAULT_EMBEDDINGS_CONFIG,
369
- PROMPT: DEFAULT_INTENTLESS_PROMPT_TEMPLATE,
380
+ PROMPT_CONFIG_KEY: DEFAULT_INTENTLESS_PROMPT_TEMPLATE,
370
381
  }
371
382
 
372
383
  @staticmethod
@@ -405,7 +416,7 @@ class IntentlessPolicy(Policy):
405
416
  self.conversation_samples_index = samples_docsearch
406
417
  self.embedder = self._create_plain_embedder(config)
407
418
  self.prompt_template = prompt_template or rasa.shared.utils.io.read_file(
408
- self.config[PROMPT]
419
+ self.config[PROMPT_CONFIG_KEY]
409
420
  )
410
421
  self.trace_prompt_tokens = self.config.get("trace_prompt_tokens", False)
411
422
 
@@ -416,9 +427,10 @@ class IntentlessPolicy(Policy):
416
427
  Returns:
417
428
  The embedder.
418
429
  """
419
- return embedder_factory(
430
+ client = embedder_factory(
420
431
  config.get(EMBEDDINGS_CONFIG_KEY), DEFAULT_EMBEDDINGS_CONFIG
421
432
  )
433
+ return _LangchainEmbeddingClientAdapter(client)
422
434
 
423
435
  def embeddings_property(self, prop: str) -> Optional[str]:
424
436
  """Returns the property of the embeddings config."""
@@ -458,6 +470,13 @@ class IntentlessPolicy(Policy):
458
470
  A policy must return its resource locator so that potential children nodes
459
471
  can load the policy from the resource.
460
472
  """
473
+ try_instantiate_llm_client(
474
+ self.config.get(LLM_CONFIG_KEY),
475
+ DEFAULT_LLM_CONFIG,
476
+ "intentless_policy.train",
477
+ "IntentlessPolicy",
478
+ )
479
+
461
480
  responses = filter_responses(responses, forms, flows or FlowsList([]))
462
481
  telemetry.track_intentless_policy_train()
463
482
  response_texts = [r for r in extract_ai_response_examples(responses.data)]
@@ -500,11 +519,12 @@ class IntentlessPolicy(Policy):
500
519
 
501
520
  structlogger.info("intentless_policy.training.completed")
502
521
  telemetry.track_intentless_policy_train_completed(
503
- embeddings_type=self.embeddings_property("_type"),
504
- embeddings_model=self.embeddings_property("model")
505
- or self.embeddings_property("model_name"),
506
- llm_type=self.llm_property("_type"),
507
- llm_model=self.llm_property("model") or self.llm_property("model_name"),
522
+ embeddings_type=self.embeddings_property(PROVIDER_CONFIG_KEY),
523
+ embeddings_model=self.embeddings_property(MODEL_CONFIG_KEY)
524
+ or self.embeddings_property(MODEL_NAME_CONFIG_KEY),
525
+ llm_type=self.llm_property(PROVIDER_CONFIG_KEY),
526
+ llm_model=self.llm_property(MODEL_CONFIG_KEY)
527
+ or self.llm_property(MODEL_NAME_CONFIG_KEY),
508
528
  )
509
529
 
510
530
  self.persist()
@@ -578,11 +598,12 @@ class IntentlessPolicy(Policy):
578
598
  )
579
599
 
580
600
  telemetry.track_intentless_policy_predict(
581
- embeddings_type=self.embeddings_property("_type"),
582
- embeddings_model=self.embeddings_property("model")
583
- or self.embeddings_property("model_name"),
584
- llm_type=self.llm_property("_type"),
585
- llm_model=self.llm_property("model") or self.llm_property("model_name"),
601
+ embeddings_type=self.embeddings_property(PROVIDER_CONFIG_KEY),
602
+ embeddings_model=self.embeddings_property(MODEL_CONFIG_KEY)
603
+ or self.embeddings_property(MODEL_NAME_CONFIG_KEY),
604
+ llm_type=self.llm_property(PROVIDER_CONFIG_KEY),
605
+ llm_model=self.llm_property(MODEL_CONFIG_KEY)
606
+ or self.llm_property(MODEL_NAME_CONFIG_KEY),
586
607
  score=score,
587
608
  )
588
609
 
@@ -595,7 +616,9 @@ class IntentlessPolicy(Policy):
595
616
  else:
596
617
  events = []
597
618
 
598
- return self._prediction(result, events=events)
619
+ action_metadata = {UTTER_SOURCE_METADATA_KEY: self.__class__.__name__}
620
+
621
+ return self._prediction(result, events=events, action_metadata=action_metadata)
599
622
 
600
623
  async def generate_answer(
601
624
  self,
@@ -619,9 +642,10 @@ class IntentlessPolicy(Policy):
619
642
  )
620
643
  return await self._generate_llm_answer(llm, prompt)
621
644
 
622
- async def _generate_llm_answer(self, llm: "BaseLLM", prompt: str) -> Optional[str]:
645
+ async def _generate_llm_answer(self, llm: LLMClient, prompt: str) -> Optional[str]:
623
646
  try:
624
- return await llm.apredict(prompt)
647
+ llm_response = await llm.acompletion(prompt)
648
+ return llm_response.choices[0]
625
649
  except Exception as e:
626
650
  # unfortunately, langchain does not wrap LLM exceptions which means
627
651
  # we have to catch all exceptions here
@@ -916,7 +940,7 @@ class IntentlessPolicy(Policy):
916
940
  def fingerprint_addon(cls, config: Dict[str, Any]) -> Optional[str]:
917
941
  """Add a fingerprint of the knowledge base for the graph."""
918
942
  prompt_template = get_prompt_template(
919
- config.get("prompt"),
943
+ config.get(PROMPT_CONFIG_KEY),
920
944
  DEFAULT_INTENTLESS_PROMPT_TEMPLATE,
921
945
  )
922
946
  return deep_container_fingerprint(prompt_template)
@@ -1,15 +1,15 @@
1
1
  from __future__ import annotations
2
-
3
2
  import logging
3
+
4
+ from rasa.engine.recipes.default_recipe import DefaultV1Recipe
4
5
  from pathlib import Path
5
6
  from collections import defaultdict
6
7
  import contextlib
7
- from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
8
8
 
9
9
  import numpy as np
10
10
  import tensorflow as tf
11
+ from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
11
12
 
12
- from rasa.engine.recipes.default_recipe import DefaultV1Recipe
13
13
  from rasa.engine.graph import ExecutionContext
14
14
  from rasa.engine.storage.resource import Resource
15
15
  from rasa.engine.storage.storage import ModelStorage
@@ -49,22 +49,18 @@ from rasa.shared.core.generator import TrackerWithCachedStates
49
49
  from rasa.shared.core.events import EntitiesAdded, Event
50
50
  from rasa.shared.core.domain import Domain
51
51
  from rasa.shared.nlu.training_data.message import Message
52
- from rasa.shared.nlu.training_data.features import (
53
- Features,
54
- save_features,
55
- load_features,
56
- )
52
+ from rasa.shared.nlu.training_data.features import Features
57
53
  import rasa.shared.utils.io
58
54
  import rasa.utils.io
59
55
  from rasa.utils import train_utils
60
- from rasa.utils.tensorflow.feature_array import (
61
- FeatureArray,
62
- serialize_nested_feature_arrays,
63
- deserialize_nested_feature_arrays,
64
- )
65
56
  from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
66
57
  from rasa.utils.tensorflow import rasa_layers
67
- from rasa.utils.tensorflow.model_data import RasaModelData, FeatureSignature, Data
58
+ from rasa.utils.tensorflow.model_data import (
59
+ RasaModelData,
60
+ FeatureSignature,
61
+ FeatureArray,
62
+ Data,
63
+ )
68
64
  from rasa.utils.tensorflow.model_data_utils import convert_to_data_format
69
65
  from rasa.utils.tensorflow.constants import (
70
66
  LABEL,
@@ -965,32 +961,22 @@ class TEDPolicy(Policy):
965
961
  model_path: Path where model is to be persisted
966
962
  """
967
963
  model_filename = self._metadata_filename()
968
- rasa.shared.utils.io.dump_obj_as_json_to_file(
969
- model_path / f"{model_filename}.priority.json", self.priority
970
- )
971
- rasa.shared.utils.io.dump_obj_as_json_to_file(
972
- model_path / f"{model_filename}.meta.json", self.config
964
+ rasa.utils.io.json_pickle(
965
+ model_path / f"{model_filename}.priority.pkl", self.priority
973
966
  )
974
- # save data example
975
- serialize_nested_feature_arrays(
976
- self.data_example,
977
- str(model_path / f"{model_filename}.data_example.st"),
978
- str(model_path / f"{model_filename}.data_example_metadata.json"),
967
+ rasa.utils.io.pickle_dump(
968
+ model_path / f"{model_filename}.meta.pkl", self.config
979
969
  )
980
- # save label data
981
- serialize_nested_feature_arrays(
982
- dict(self._label_data.data) if self._label_data is not None else {},
983
- str(model_path / f"{model_filename}.label_data.st"),
984
- str(model_path / f"{model_filename}.label_data_metadata.json"),
970
+ rasa.utils.io.pickle_dump(
971
+ model_path / f"{model_filename}.data_example.pkl", self.data_example
985
972
  )
986
- # save fake features
987
- metadata = save_features(
988
- self.fake_features, str(model_path / f"{model_filename}.fake_features.st")
973
+ rasa.utils.io.pickle_dump(
974
+ model_path / f"{model_filename}.fake_features.pkl", self.fake_features
989
975
  )
990
- rasa.shared.utils.io.dump_obj_as_json_to_file(
991
- model_path / f"{model_filename}.fake_features_metadata.json", metadata
976
+ rasa.utils.io.pickle_dump(
977
+ model_path / f"{model_filename}.label_data.pkl",
978
+ dict(self._label_data.data) if self._label_data is not None else {},
992
979
  )
993
-
994
980
  entity_tag_specs = (
995
981
  [tag_spec._asdict() for tag_spec in self._entity_tag_specs]
996
982
  if self._entity_tag_specs
@@ -1008,29 +994,18 @@ class TEDPolicy(Policy):
1008
994
  model_path: Path where model is to be persisted.
1009
995
  """
1010
996
  tf_model_file = model_path / f"{cls._metadata_filename()}.tf_model"
1011
-
1012
- # load data example
1013
- loaded_data = deserialize_nested_feature_arrays(
1014
- str(model_path / f"{cls._metadata_filename()}.data_example.st"),
1015
- str(model_path / f"{cls._metadata_filename()}.data_example_metadata.json"),
997
+ loaded_data = rasa.utils.io.pickle_load(
998
+ model_path / f"{cls._metadata_filename()}.data_example.pkl"
1016
999
  )
1017
- # load label data
1018
- loaded_label_data = deserialize_nested_feature_arrays(
1019
- str(model_path / f"{cls._metadata_filename()}.label_data.st"),
1020
- str(model_path / f"{cls._metadata_filename()}.label_data_metadata.json"),
1000
+ label_data = rasa.utils.io.pickle_load(
1001
+ model_path / f"{cls._metadata_filename()}.label_data.pkl"
1021
1002
  )
1022
- label_data = RasaModelData(data=loaded_label_data)
1023
-
1024
- # load fake features
1025
- metadata = rasa.shared.utils.io.read_json_file(
1026
- model_path / f"{cls._metadata_filename()}.fake_features_metadata.json"
1003
+ fake_features = rasa.utils.io.pickle_load(
1004
+ model_path / f"{cls._metadata_filename()}.fake_features.pkl"
1027
1005
  )
1028
- fake_features = load_features(
1029
- str(model_path / f"{cls._metadata_filename()}.fake_features.st"), metadata
1030
- )
1031
-
1032
- priority = rasa.shared.utils.io.read_json_file(
1033
- model_path / f"{cls._metadata_filename()}.priority.json"
1006
+ label_data = RasaModelData(data=label_data)
1007
+ priority = rasa.utils.io.json_unpickle(
1008
+ model_path / f"{cls._metadata_filename()}.priority.pkl"
1034
1009
  )
1035
1010
  entity_tag_specs = rasa.shared.utils.io.read_json_file(
1036
1011
  model_path / f"{cls._metadata_filename()}.entity_tag_specs.json"
@@ -1048,8 +1023,8 @@ class TEDPolicy(Policy):
1048
1023
  )
1049
1024
  for tag_spec in entity_tag_specs
1050
1025
  ]
1051
- model_config = rasa.shared.utils.io.read_json_file(
1052
- model_path / f"{cls._metadata_filename()}.meta.json"
1026
+ model_config = rasa.utils.io.pickle_load(
1027
+ model_path / f"{cls._metadata_filename()}.meta.pkl"
1053
1028
  )
1054
1029
 
1055
1030
  return {
@@ -1095,7 +1070,7 @@ class TEDPolicy(Policy):
1095
1070
  ) -> TEDPolicy:
1096
1071
  featurizer = TrackerFeaturizer.load(model_path)
1097
1072
 
1098
- if not (model_path / f"{cls._metadata_filename()}.data_example.st").is_file():
1073
+ if not (model_path / f"{cls._metadata_filename()}.data_example.pkl").is_file():
1099
1074
  return cls(
1100
1075
  config,
1101
1076
  model_storage,
@@ -5,7 +5,6 @@ from typing import Any, List, Optional, Text, Dict, Type, Union
5
5
 
6
6
  import numpy as np
7
7
  import tensorflow as tf
8
-
9
8
  import rasa.utils.common
10
9
  from rasa.engine.graph import ExecutionContext
11
10
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
@@ -17,7 +16,6 @@ from rasa.shared.core.domain import Domain
17
16
  from rasa.shared.core.trackers import DialogueStateTracker
18
17
  from rasa.shared.core.constants import SLOTS, ACTIVE_LOOP, ACTION_UNLIKELY_INTENT_NAME
19
18
  from rasa.shared.core.events import UserUttered, ActionExecuted
20
- import rasa.shared.utils.io
21
19
  from rasa.shared.nlu.constants import (
22
20
  INTENT,
23
21
  TEXT,
@@ -105,6 +103,8 @@ from rasa.utils.tensorflow.constants import (
105
103
  )
106
104
  from rasa.utils.tensorflow import layers
107
105
  from rasa.utils.tensorflow.model_data import RasaModelData, FeatureArray, Data
106
+
107
+ import rasa.utils.io as io_utils
108
108
  from rasa.core.exceptions import RasaCoreException
109
109
  from rasa.shared.utils import common
110
110
 
@@ -881,12 +881,9 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
881
881
  model_path: Path where model is to be persisted
882
882
  """
883
883
  super().persist_model_utilities(model_path)
884
-
885
- from safetensors.numpy import save_file
886
-
887
- save_file(
888
- {str(k): np.array(v) for k, v in self.label_quantiles.items()},
889
- model_path / f"{self._metadata_filename()}.label_quantiles.st",
884
+ io_utils.pickle_dump(
885
+ model_path / f"{self._metadata_filename()}.label_quantiles.pkl",
886
+ self.label_quantiles,
890
887
  )
891
888
 
892
889
  @classmethod
@@ -897,14 +894,9 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
897
894
  model_path: Path where model is to be persisted.
898
895
  """
899
896
  model_utilties = super()._load_model_utilities(model_path)
900
-
901
- from safetensors.numpy import load_file
902
-
903
- loaded_label_quantiles = load_file(
904
- model_path / f"{cls._metadata_filename()}.label_quantiles.st"
897
+ label_quantiles = io_utils.pickle_load(
898
+ model_path / f"{cls._metadata_filename()}.label_quantiles.pkl"
905
899
  )
906
- label_quantiles = {int(k): list(v) for k, v in loaded_label_quantiles.items()}
907
-
908
900
  model_utilties.update({"label_quantiles": label_quantiles})
909
901
  return model_utilties
910
902
 
rasa/core/processor.py CHANGED
@@ -3,12 +3,12 @@ import copy
3
3
  import logging
4
4
  import structlog
5
5
  import os
6
+ import re
6
7
  from pathlib import Path
7
8
  import tarfile
8
9
  import time
9
10
  from types import LambdaType
10
11
  from typing import Any, Dict, List, Optional, TYPE_CHECKING, Text, Tuple, Union
11
-
12
12
  from rasa.core.actions.action_exceptions import ActionExecutionRejection
13
13
  from rasa.core.actions.forms import FormAction
14
14
  from rasa.core.http_interpreter import RasaNLUHttpInterpreter
@@ -751,20 +751,17 @@ class MessageProcessor:
751
751
  message=processed_message, domain=self.domain
752
752
  )
753
753
 
754
- # Invalid use of slash syntax
754
+ # Invalid use of slash syntax, sanitize the message before passing
755
+ # it to the graph
755
756
  if (
756
757
  processed_message.starts_with_slash_syntax()
757
758
  and not processed_message.has_intent()
758
759
  and not processed_message.has_commands()
759
760
  ):
760
- parse_data = self._parse_invalid_use_of_slash_syntax(
761
- processed_message, tracker, only_output_properties
762
- )
761
+ message = self._sanitize_message(message)
763
762
 
764
763
  # Intent or commands are not explicitly present. Pass message to graph.
765
- elif not (
766
- processed_message.has_intent() or processed_message.has_commands()
767
- ):
764
+ if not (processed_message.has_intent() or processed_message.has_commands()):
768
765
  parse_data = await self._parse_message_with_graph(
769
766
  message, tracker, only_output_properties
770
767
  )
@@ -788,44 +785,16 @@ class MessageProcessor:
788
785
 
789
786
  return parse_data
790
787
 
791
- def _parse_invalid_use_of_slash_syntax(
792
- self,
793
- message: Message,
794
- tracker: Optional[DialogueStateTracker] = None,
795
- only_output_properties: bool = True,
796
- ) -> Dict[Text, Any]:
797
- structlogger.warning(
798
- "processor.message.parse.invalid_use_of_slash_syntax",
799
- event_info=(
800
- "Message starts with '/', but no intents or commands are"
801
- "passed. Returning CannotHandleCommand() as a fallback."
802
- ),
803
- message=message.get(TEXT),
804
- )
805
- parse_data: Dict[Text, Any] = {
806
- TEXT: "",
807
- INTENT: {INTENT_NAME_KEY: None, PREDICTED_CONFIDENCE_KEY: 0.0},
808
- ENTITIES: [],
809
- }
810
- parse_data.update(
811
- message.as_dict(only_output_properties=only_output_properties)
812
- )
813
- commands = parse_data.get(COMMANDS, [])
814
- commands += [
815
- CannotHandleCommand(RASA_PATTERN_CANNOT_HANDLE_INVALID_INTENT).as_dict()
816
- ]
817
-
818
- if (
819
- tracker is not None
820
- and tracker.has_coexistence_routing_slot
821
- and tracker.get_slot(ROUTE_TO_CALM_SLOT) is None
822
- ):
823
- # if we are currently not routing to either CALM or dm1
824
- # we make a sticky routing to CALM
825
- commands += [SetSlotCommand(ROUTE_TO_CALM_SLOT, True).as_dict()]
826
-
827
- parse_data[COMMANDS] = commands
828
- return parse_data
788
+ def _sanitize_message(self, message: UserMessage) -> UserMessage:
789
+ """Sanitize user message by removing prepended slashes before the
790
+ actual content.
791
+ """
792
+ # Regex pattern to match leading slashes and any whitespace before
793
+ # actual content
794
+ pattern = r"^[/\s]+"
795
+ # Remove the matched pattern from the beginning of the message
796
+ message.text = re.sub(pattern, "", message.text).strip()
797
+ return message
829
798
 
830
799
  async def _parse_message_with_commands_and_intents(
831
800
  self,
rasa/core/run.py CHANGED
@@ -9,12 +9,12 @@ from functools import partial
9
9
  from typing import (
10
10
  Any,
11
11
  Callable,
12
+ Dict,
12
13
  List,
13
14
  Optional,
14
15
  Text,
15
16
  Tuple,
16
17
  Union,
17
- Dict,
18
18
  )
19
19
 
20
20
  from sanic import Sanic
@@ -24,7 +24,6 @@ import rasa.core.utils
24
24
  import rasa.shared.utils.common
25
25
  import rasa.shared.utils.io
26
26
  import rasa.utils
27
- from rasa.utils import licensing
28
27
  import rasa.utils.common
29
28
  import rasa.utils.io
30
29
  from rasa import server, telemetry
@@ -34,9 +33,11 @@ from rasa.core.agent import Agent
34
33
  from rasa.core.channels import console
35
34
  from rasa.core.channels.channel import InputChannel
36
35
  from rasa.core.utils import AvailableEndpoints
36
+ from rasa.nlu.persistor import StorageType
37
37
  from rasa.plugin import plugin_manager
38
38
  from rasa.shared.exceptions import RasaException
39
39
  from rasa.shared.utils.yaml import read_config_file
40
+ from rasa.utils import licensing
40
41
 
41
42
  logger = logging.getLogger() # get the root logger
42
43
 
@@ -210,7 +211,7 @@ def serve_application(
210
211
  jwt_private_key: Optional[Text] = None,
211
212
  jwt_method: Optional[Text] = None,
212
213
  endpoints: Optional[AvailableEndpoints] = None,
213
- remote_storage: Optional[Text] = None,
214
+ remote_storage: Optional[StorageType] = None,
214
215
  log_file: Optional[Text] = None,
215
216
  ssl_certificate: Optional[Text] = None,
216
217
  ssl_keyfile: Optional[Text] = None,
@@ -295,7 +296,7 @@ def serve_application(
295
296
  async def load_agent_on_start(
296
297
  model_path: Text,
297
298
  endpoints: AvailableEndpoints,
298
- remote_storage: Optional[Text],
299
+ remote_storage: Optional[StorageType],
299
300
  app: Sanic,
300
301
  loop: AbstractEventLoop,
301
302
  ) -> Agent:
@@ -26,10 +26,10 @@ from typing import (
26
26
  from boto3.dynamodb.conditions import Key
27
27
  from pymongo.collection import Collection
28
28
 
29
- import rasa.core.utils as core_utils
30
29
  import rasa.shared.utils.cli
31
30
  import rasa.shared.utils.common
32
31
  import rasa.shared.utils.io
32
+ import rasa.utils.json_utils
33
33
  from rasa.plugin import plugin_manager
34
34
  from rasa.shared.core.constants import ACTION_LISTEN_NAME
35
35
  from rasa.core.brokers.broker import EventBroker
@@ -705,7 +705,7 @@ class DynamoTrackerStore(TrackerStore, SerializedTrackerAsDict):
705
705
 
706
706
  DynamoDB cannot store `float`s, so we'll convert them to `Decimal`s.
707
707
  """
708
- return core_utils.replace_floats_with_decimals(
708
+ return rasa.utils.json_utils.replace_floats_with_decimals(
709
709
  SerializedTrackerAsDict.serialise_tracker(tracker)
710
710
  )
711
711
 
@@ -747,12 +747,16 @@ class DynamoTrackerStore(TrackerStore, SerializedTrackerAsDict):
747
747
  events_with_floats = []
748
748
  for dialogue in dialogues:
749
749
  if dialogue.get("events"):
750
- events = core_utils.replace_decimals_with_floats(dialogue["events"])
750
+ events = rasa.utils.json_utils.replace_decimals_with_floats(
751
+ dialogue["events"]
752
+ )
751
753
  events_with_floats += events
752
754
  else:
753
755
  events = dialogues[0].get("events", [])
754
756
  # `float`s are stored as `Decimal` objects - we need to convert them back
755
- events_with_floats = core_utils.replace_decimals_with_floats(events)
757
+ events_with_floats = rasa.utils.json_utils.replace_decimals_with_floats(
758
+ events
759
+ )
756
760
 
757
761
  if self.domain is None:
758
762
  slots = []