rasa-pro 3.10.10__py3-none-any.whl → 3.10.12__py3-none-any.whl

rasa/core/featurizers/tracker_featurizers.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
-from pathlib import Path
-from collections import defaultdict
-from abc import abstractmethod
-import jsonpickle
-import logging
 
-from tqdm import tqdm
+import logging
+from abc import abstractmethod
+from collections import defaultdict
+from pathlib import Path
 from typing import (
     Tuple,
     List,
@@ -18,25 +16,30 @@ from typing import (
     Set,
     DefaultDict,
     cast,
+    Type,
+    Callable,
+    ClassVar,
 )
+
 import numpy as np
+from tqdm import tqdm
 
-from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer
-from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization
-from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError
 import rasa.shared.core.trackers
 import rasa.shared.utils.io
-from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME
-from rasa.shared.nlu.training_data.features import Features
-from rasa.shared.core.trackers import DialogueStateTracker
-from rasa.shared.core.domain import State, Domain
-from rasa.shared.core.events import Event, ActionExecuted, UserUttered
+from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError
+from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization
+from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer
 from rasa.shared.core.constants import (
     USER,
     ACTION_UNLIKELY_INTENT_NAME,
     PREVIOUS_ACTION,
 )
+from rasa.shared.core.domain import State, Domain
+from rasa.shared.core.events import Event, ActionExecuted, UserUttered
+from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import RasaException
+from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME
+from rasa.shared.nlu.training_data.features import Features
 from rasa.utils.tensorflow.constants import LABEL_PAD_ID
 from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
 
@@ -64,6 +67,10 @@ class InvalidStory(RasaException):
 class TrackerFeaturizer:
     """Base class for actual tracker featurizers."""
 
+    # Class registry to store all subclasses
+    _registry: ClassVar[Dict[str, Type["TrackerFeaturizer"]]] = {}
+    _featurizer_type: str = "TrackerFeaturizer"
+
     def __init__(
         self, state_featurizer: Optional[SingleStateFeaturizer] = None
     ) -> None:
@@ -74,6 +81,36 @@ class TrackerFeaturizer:
         """
         self.state_featurizer = state_featurizer
 
+    @classmethod
+    def register(cls, featurizer_type: str) -> Callable:
+        """Decorator to register featurizer subclasses."""
+
+        def wrapper(subclass: Type["TrackerFeaturizer"]) -> Type["TrackerFeaturizer"]:
+            cls._registry[featurizer_type] = subclass
+            # Store the type identifier in the class for serialization
+            subclass._featurizer_type = featurizer_type
+            return subclass
+
+        return wrapper
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer":
+        """Create featurizer instance from dictionary."""
+        featurizer_type = data.pop("type")
+
+        if featurizer_type not in cls._registry:
+            raise ValueError(f"Unknown featurizer type: {featurizer_type}")
+
+        # Get the correct subclass and instantiate it
+        subclass = cls._registry[featurizer_type]
+        return subclass.create_from_dict(data)
+
+    @classmethod
+    @abstractmethod
+    def create_from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer":
+        """Each subclass must implement its own creation from dict method."""
+        pass
+
     @staticmethod
     def _create_states(
         tracker: DialogueStateTracker,
@@ -465,9 +502,7 @@ class TrackerFeaturizer:
             self.state_featurizer.entity_tag_specs = []
 
         # noinspection PyTypeChecker
-        rasa.shared.utils.io.write_text_file(
-            str(jsonpickle.encode(self)), featurizer_file
-        )
+        rasa.shared.utils.io.dump_obj_as_json_to_file(featurizer_file, self.to_dict())
 
     @staticmethod
     def load(path: Union[Text, Path]) -> Optional[TrackerFeaturizer]:
@@ -481,7 +516,17 @@ class TrackerFeaturizer:
         """
         featurizer_file = Path(path) / FEATURIZER_FILE
         if featurizer_file.is_file():
-            return jsonpickle.decode(rasa.shared.utils.io.read_file(featurizer_file))
+            data = rasa.shared.utils.io.read_json_file(featurizer_file)
+
+            if "type" not in data:
+                logger.error(
+                    f"Couldn't load featurizer for policy. "
+                    f"File '{featurizer_file}' does not contain all "
+                    f"necessary information. 'type' is missing."
+                )
+                return None
+
+            return TrackerFeaturizer.from_dict(data)
 
         logger.error(
             f"Couldn't load featurizer for policy. "
@@ -508,7 +553,16 @@ class TrackerFeaturizer:
             )
         ]
 
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "type": self.__class__._featurizer_type,
+            "state_featurizer": (
+                self.state_featurizer.to_dict() if self.state_featurizer else None
+            ),
+        }
+
 
+@TrackerFeaturizer.register("FullDialogueTrackerFeaturizer")
 class FullDialogueTrackerFeaturizer(TrackerFeaturizer):
     """Creates full dialogue training data for time distributed architectures.
 
@@ -646,7 +700,20 @@ class FullDialogueTrackerFeaturizer(TrackerFeaturizer):
 
         return trackers_as_states
 
+    def to_dict(self) -> Dict[str, Any]:
+        return super().to_dict()
 
+    @classmethod
+    def create_from_dict(cls, data: Dict[str, Any]) -> "FullDialogueTrackerFeaturizer":
+        state_featurizer = SingleStateFeaturizer.create_from_dict(
+            data["state_featurizer"]
+        )
+        return cls(
+            state_featurizer,
+        )
+
+
+@TrackerFeaturizer.register("MaxHistoryTrackerFeaturizer")
 class MaxHistoryTrackerFeaturizer(TrackerFeaturizer):
     """Truncates the tracker history into `max_history` long sequences.
 
@@ -884,7 +951,25 @@ class MaxHistoryTrackerFeaturizer(TrackerFeaturizer):
 
         return trackers_as_states
 
+    def to_dict(self) -> Dict[str, Any]:
+        data = super().to_dict()
+        data.update(
+            {
+                "remove_duplicates": self.remove_duplicates,
+                "max_history": self.max_history,
+            }
+        )
+        return data
+
+    @classmethod
+    def create_from_dict(cls, data: Dict[str, Any]) -> "MaxHistoryTrackerFeaturizer":
+        state_featurizer = SingleStateFeaturizer.create_from_dict(
+            data["state_featurizer"]
+        )
+        return cls(state_featurizer, data["max_history"], data["remove_duplicates"])
 
+
+@TrackerFeaturizer.register("IntentMaxHistoryTrackerFeaturizer")
 class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer):
     """Truncates the tracker history into `max_history` long sequences.
 
@@ -1159,6 +1244,18 @@ class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer):
 
         return trackers_as_states
 
+    def to_dict(self) -> Dict[str, Any]:
+        return super().to_dict()
+
+    @classmethod
+    def create_from_dict(
+        cls, data: Dict[str, Any]
+    ) -> "IntentMaxHistoryTrackerFeaturizer":
+        state_featurizer = SingleStateFeaturizer.create_from_dict(
+            data["state_featurizer"]
+        )
+        return cls(state_featurizer, data["max_history"], data["remove_duplicates"])
+
 
 def _is_prev_action_unlikely_intent_in_state(state: State) -> bool:
     prev_action_name = state.get(PREVIOUS_ACTION, {}).get(ACTION_NAME)

rasa/core/policies/ted_policy.py

@@ -1,15 +1,15 @@
 from __future__ import annotations
-import logging
 
-from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+import logging
 from pathlib import Path
 from collections import defaultdict
 import contextlib
+from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
 
 import numpy as np
 import tensorflow as tf
-from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type
 
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.graph import ExecutionContext
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
@@ -49,18 +49,22 @@ from rasa.shared.core.generator import TrackerWithCachedStates
 from rasa.shared.core.events import EntitiesAdded, Event
 from rasa.shared.core.domain import Domain
 from rasa.shared.nlu.training_data.message import Message
-from rasa.shared.nlu.training_data.features import Features
+from rasa.shared.nlu.training_data.features import (
+    Features,
+    save_features,
+    load_features,
+)
 import rasa.shared.utils.io
 import rasa.utils.io
 from rasa.utils import train_utils
-from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
-from rasa.utils.tensorflow import rasa_layers
-from rasa.utils.tensorflow.model_data import (
-    RasaModelData,
-    FeatureSignature,
+from rasa.utils.tensorflow.feature_array import (
     FeatureArray,
-    Data,
+    serialize_nested_feature_arrays,
+    deserialize_nested_feature_arrays,
 )
+from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
+from rasa.utils.tensorflow import rasa_layers
+from rasa.utils.tensorflow.model_data import RasaModelData, FeatureSignature, Data
 from rasa.utils.tensorflow.model_data_utils import convert_to_data_format
 from rasa.utils.tensorflow.constants import (
     LABEL,
@@ -961,22 +965,32 @@ class TEDPolicy(Policy):
             model_path: Path where model is to be persisted
         """
         model_filename = self._metadata_filename()
-        rasa.utils.io.json_pickle(
-            model_path / f"{model_filename}.priority.pkl", self.priority
-        )
-        rasa.utils.io.pickle_dump(
-            model_path / f"{model_filename}.meta.pkl", self.config
+        rasa.shared.utils.io.dump_obj_as_json_to_file(
+            model_path / f"{model_filename}.priority.json", self.priority
         )
-        rasa.utils.io.pickle_dump(
-            model_path / f"{model_filename}.data_example.pkl", self.data_example
+        rasa.shared.utils.io.dump_obj_as_json_to_file(
+            model_path / f"{model_filename}.meta.json", self.config
         )
-        rasa.utils.io.pickle_dump(
-            model_path / f"{model_filename}.fake_features.pkl", self.fake_features
+        # save data example
+        serialize_nested_feature_arrays(
+            self.data_example,
+            str(model_path / f"{model_filename}.data_example.st"),
+            str(model_path / f"{model_filename}.data_example_metadata.json"),
         )
-        rasa.utils.io.pickle_dump(
-            model_path / f"{model_filename}.label_data.pkl",
+        # save label data
+        serialize_nested_feature_arrays(
             dict(self._label_data.data) if self._label_data is not None else {},
+            str(model_path / f"{model_filename}.label_data.st"),
+            str(model_path / f"{model_filename}.label_data_metadata.json"),
+        )
+        # save fake features
+        metadata = save_features(
+            self.fake_features, str(model_path / f"{model_filename}.fake_features.st")
+        )
+        rasa.shared.utils.io.dump_obj_as_json_to_file(
+            model_path / f"{model_filename}.fake_features_metadata.json", metadata
         )
+
         entity_tag_specs = (
             [tag_spec._asdict() for tag_spec in self._entity_tag_specs]
             if self._entity_tag_specs
@@ -994,18 +1008,29 @@ class TEDPolicy(Policy):
             model_path: Path where model is to be persisted.
         """
         tf_model_file = model_path / f"{cls._metadata_filename()}.tf_model"
-        loaded_data = rasa.utils.io.pickle_load(
-            model_path / f"{cls._metadata_filename()}.data_example.pkl"
+
+        # load data example
+        loaded_data = deserialize_nested_feature_arrays(
+            str(model_path / f"{cls._metadata_filename()}.data_example.st"),
+            str(model_path / f"{cls._metadata_filename()}.data_example_metadata.json"),
         )
-        label_data = rasa.utils.io.pickle_load(
-            model_path / f"{cls._metadata_filename()}.label_data.pkl"
+        # load label data
+        loaded_label_data = deserialize_nested_feature_arrays(
+            str(model_path / f"{cls._metadata_filename()}.label_data.st"),
+            str(model_path / f"{cls._metadata_filename()}.label_data_metadata.json"),
         )
-        fake_features = rasa.utils.io.pickle_load(
-            model_path / f"{cls._metadata_filename()}.fake_features.pkl"
+        label_data = RasaModelData(data=loaded_label_data)
+
+        # load fake features
+        metadata = rasa.shared.utils.io.read_json_file(
+            model_path / f"{cls._metadata_filename()}.fake_features_metadata.json"
         )
-        label_data = RasaModelData(data=label_data)
-        priority = rasa.utils.io.json_unpickle(
-            model_path / f"{cls._metadata_filename()}.priority.pkl"
+        fake_features = load_features(
+            str(model_path / f"{cls._metadata_filename()}.fake_features.st"), metadata
+        )
+
+        priority = rasa.shared.utils.io.read_json_file(
+            model_path / f"{cls._metadata_filename()}.priority.json"
         )
         entity_tag_specs = rasa.shared.utils.io.read_json_file(
             model_path / f"{cls._metadata_filename()}.entity_tag_specs.json"
@@ -1023,8 +1048,8 @@ class TEDPolicy(Policy):
             )
             for tag_spec in entity_tag_specs
         ]
-        model_config = rasa.utils.io.pickle_load(
-            model_path / f"{cls._metadata_filename()}.meta.pkl"
+        model_config = rasa.shared.utils.io.read_json_file(
+            model_path / f"{cls._metadata_filename()}.meta.json"
         )
 
         return {
@@ -1070,7 +1095,7 @@ class TEDPolicy(Policy):
     ) -> TEDPolicy:
         featurizer = TrackerFeaturizer.load(model_path)
 
-        if not (model_path / f"{cls._metadata_filename()}.data_example.pkl").is_file():
+        if not (model_path / f"{cls._metadata_filename()}.data_example.st").is_file():
             return cls(
                 config,
                 model_storage,

rasa/core/policies/unexpected_intent_policy.py

@@ -5,6 +5,7 @@ from typing import Any, List, Optional, Text, Dict, Type, Union
 
 import numpy as np
 import tensorflow as tf
+
 import rasa.utils.common
 from rasa.engine.graph import ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
@@ -16,6 +17,7 @@ from rasa.shared.core.domain import Domain
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.core.constants import SLOTS, ACTIVE_LOOP, ACTION_UNLIKELY_INTENT_NAME
 from rasa.shared.core.events import UserUttered, ActionExecuted
+import rasa.shared.utils.io
 from rasa.shared.nlu.constants import (
     INTENT,
     TEXT,
@@ -103,8 +105,6 @@ from rasa.utils.tensorflow.constants import (
 )
 from rasa.utils.tensorflow import layers
 from rasa.utils.tensorflow.model_data import RasaModelData, FeatureArray, Data
-
-import rasa.utils.io as io_utils
 from rasa.core.exceptions import RasaCoreException
 from rasa.shared.utils import common
 
@@ -881,9 +881,12 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
             model_path: Path where model is to be persisted
         """
         super().persist_model_utilities(model_path)
-        io_utils.pickle_dump(
-            model_path / f"{self._metadata_filename()}.label_quantiles.pkl",
-            self.label_quantiles,
+
+        from safetensors.numpy import save_file
+
+        save_file(
+            {str(k): np.array(v) for k, v in self.label_quantiles.items()},
+            model_path / f"{self._metadata_filename()}.label_quantiles.st",
         )
 
     @classmethod
@@ -894,9 +897,14 @@ class UnexpecTEDIntentPolicy(TEDPolicy):
             model_path: Path where model is to be persisted.
         """
         model_utilties = super()._load_model_utilities(model_path)
-        label_quantiles = io_utils.pickle_load(
-            model_path / f"{cls._metadata_filename()}.label_quantiles.pkl"
+
+        from safetensors.numpy import load_file
+
+        loaded_label_quantiles = load_file(
+            model_path / f"{cls._metadata_filename()}.label_quantiles.st"
         )
+        label_quantiles = {int(k): list(v) for k, v in loaded_label_quantiles.items()}
+
         model_utilties.update({"label_quantiles": label_quantiles})
         return model_utilties
 

rasa/dialogue_understanding/commands/change_flow_command.py

@@ -36,3 +36,9 @@ class ChangeFlowCommand(Command):
         # the change flow command is not actually pushing anything to the tracker,
         # but it is predicted by the MultiStepLLMCommandGenerator and used internally
         return []
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, ChangeFlowCommand)
+
+    def __hash__(self) -> int:
+        return hash(self.command())

rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py

@@ -229,9 +229,9 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
         commands: List[Command] = []
 
         slot_set_re = re.compile(
-            r"""SetSlot\((\"?[a-zA-Z_][a-zA-Z0-9_-]*?\"?), ?(.*)\)"""
+            r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
         )
-        start_flow_re = re.compile(r"StartFlow\(([a-zA-Z0-9_-]+?)\)")
+        start_flow_re = re.compile(r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)")
         change_flow_re = re.compile(r"ChangeFlow\(\)")
         cancel_flow_re = re.compile(r"CancelFlow\(\)")
         chitchat_re = re.compile(r"ChitChat\(\)")
@@ -280,9 +280,19 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
                 commands.append(HumanHandoffCommand())
             elif match := clarify_re.search(action):
                 options = sorted([opt.strip() for opt in match.group(1).split(",")])
+                # Remove surrounding quotes if present
+                cleaned_options = []
+                for flow in options:
+                    if (flow.startswith('"') and flow.endswith('"')) or (
+                        flow.startswith("'") and flow.endswith("'")
+                    ):
+                        cleaned_options.append(flow[1:-1])
+                    else:
+                        cleaned_options.append(flow)
+                # check if flow is valid
                 valid_options = [
                     flow
-                    for flow in options
+                    for flow in cleaned_options
                     if flow in flows.user_flow_ids
                     and flow not in user_flows_on_the_stack(tracker.stack)
                 ]
@@ -293,6 +303,13 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             elif change_flow_re.search(action):
                 commands.append(ChangeFlowCommand())
 
+        if not commands:
+            structlogger.debug(
+                "multi_step_llm_command_generator.parse_commands",
+                message="No commands were parsed from the LLM actions.",
+                actions=actions,
+            )
+
         return commands
 
     ### Helper methods

rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -185,6 +185,12 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if not commands:
             # no commands are parsed or there's an invalid command
+            structlogger.warning(
+                "single_step_llm_command_generator.predict_commands",
+                message="No commands were predicted as the LLM response could "
+                "not be parsed or the LLM responded with an invalid command."
+                "Returning a CannotHandleCommand instead.",
+            )
             commands = [CannotHandleCommand()]
 
         if tracker.has_coexistence_routing_slot:
@@ -285,14 +291,16 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         commands: List[Command] = []
 
-        slot_set_re = re.compile(r"""SetSlot\(([a-zA-Z_][a-zA-Z0-9_-]*?), ?(.*)\)""")
-        start_flow_re = re.compile(r"StartFlow\(([a-zA-Z0-9_-]+?)\)")
+        slot_set_re = re.compile(
+            r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
+        )
+        start_flow_re = re.compile(r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)")
         cancel_flow_re = re.compile(r"CancelFlow\(\)")
         chitchat_re = re.compile(r"ChitChat\(\)")
         skip_question_re = re.compile(r"SkipQuestion\(\)")
         knowledge_re = re.compile(r"SearchAndReply\(\)")
         humand_handoff_re = re.compile(r"HumanHandoff\(\)")
-        clarify_re = re.compile(r"Clarify\(([a-zA-Z0-9_, ]+)\)")
+        clarify_re = re.compile(r"Clarify\(([\"\'a-zA-Z0-9_, ]+)\)")
 
         for action in actions.strip().splitlines():
             if match := slot_set_re.search(action):
@@ -321,14 +329,31 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
                 commands.append(HumanHandoffCommand())
             elif match := clarify_re.search(action):
                 options = sorted([opt.strip() for opt in match.group(1).split(",")])
+                # Remove surrounding quotes if present
+                cleaned_options = []
+                for flow in options:
+                    if (flow.startswith('"') and flow.endswith('"')) or (
+                        flow.startswith("'") and flow.endswith("'")
+                    ):
+                        cleaned_options.append(flow[1:-1])
+                    else:
+                        cleaned_options.append(flow)
+                # check if flow is valid
                 valid_options = [
-                    flow for flow in options if flow in flows.user_flow_ids
+                    flow for flow in cleaned_options if flow in flows.user_flow_ids
                 ]
                 if len(set(valid_options)) == 1:
                     commands.extend(cls.start_flow_by_name(valid_options[0], flows))
                 elif len(valid_options) > 1:
                     commands.append(ClarifyCommand(valid_options))
 
+        if not commands:
+            structlogger.debug(
+                "single_step_llm_command_generator.parse_commands",
+                message="No commands were parsed from the LLM actions.",
+                actions=actions,
+            )
+
         return commands
 
     @classmethod

rasa/e2e_test/e2e_test_runner.py

@@ -190,11 +190,11 @@ class E2ETestRunner:
                     error=f"Message handling timed out for user message '{step.text}'.",
                     exc_info=True,
                 )
-            except Exception:
+            except Exception as exc:
                 structlogger.error(
                     "e2e_test_runner.run_prediction_loop",
                     error=f"An exception occurred while handling "
-                    f"user message '{step.text}'.",
+                    f"user message '{step.text}'. Error: {exc}",
                 )
             tracker = await self.agent.tracker_store.retrieve(sender_id)  # type: ignore[assignment]
             turns[position], event_cursor = self.get_actual_step_output(