rasa-pro 3.11.0__py3-none-any.whl → 3.11.0a2__py3-none-any.whl

rasa/utils/common.py

@@ -90,11 +90,6 @@ EXPECTED_WARNINGS: List[Tuple[Type[Warning], str]] = [
     # Ignore Keras DeprecationWarning since it requires that we
     # upgrade tensorflow-macos to 2.13.0 version.
     (DeprecationWarning, "invalid escape sequence*"),
-    # Ignore importlib open_text and read_text warnings for now
-    (
-        DeprecationWarning,
-        "https://importlib-resources.readthedocs.io/en/latest/using.html#migrating-from-legacy",
-    ),
 ]
 
 PYTHON_LOGGING_SCHEMA_DOCS = (

rasa/utils/endpoints.py

@@ -1,7 +1,7 @@
 import os
 import ssl
 from types import ModuleType
-from typing import Any, List, Optional, Text, Dict, Union
+from typing import Any, Optional, Text, Dict, Union
 
 import aiohttp
 import structlog
@@ -41,32 +41,6 @@ def read_endpoint_config(
         return None
 
 
-def read_property_config_from_endpoints_file(
-    filename: str, property_name: str
-) -> Optional[Union[Dict[str, Any], List]]:
-    """Read a property from an endpoint configuration file."""
-    if not filename:
-        return None
-
-    try:
-        content = read_config_file(filename)
-
-        if content.get(property_name) is None:
-            return None
-
-        return content[property_name]
-    except FileNotFoundError:
-        structlogger.error(
-            "endpoint.read.failed_no_such_file",
-            filename=os.path.abspath(filename),
-            event_info=(
-                "Failed to read endpoint configuration file - "
-                "the file was not found."
-            ),
-        )
-        return None
-
-
 def concat_url(base: Text, subpath: Optional[Text]) -> Text:
     """Append a subpath to a base url.
 

rasa/utils/io.py

@@ -2,6 +2,7 @@ import asyncio
 import filecmp
 import logging
 import os
+import pickle
 import tempfile
 import warnings
 import re
@@ -18,6 +19,7 @@ from typing import (
     Type,
     Callable,
     TYPE_CHECKING,
+    Pattern,
 )
 
 from ruamel import yaml
@@ -96,6 +98,29 @@ def enable_async_loop_debugging(
     return event_loop
 
 
+def pickle_dump(filename: Union[Text, Path], obj: Any) -> None:
+    """Saves object to file.
+
+    Args:
+        filename: the filename to save the object to
+        obj: the object to store
+    """
+    with open(filename, "wb") as f:
+        pickle.dump(obj, f)
+
+
+def pickle_load(filename: Union[Text, Path]) -> Any:
+    """Loads an object from a file.
+
+    Args:
+        filename: the filename to load the object from
+
+    Returns: the loaded object
+    """
+    with open(filename, "rb") as f:
+        return pickle.load(f)
+
+
 def create_temporary_file(data: Any, suffix: Text = "", mode: Text = "w+") -> Text:
     """Creates a tempfile.NamedTemporaryFile object for data."""
     encoding = None if "b" in mode else rasa.shared.utils.io.DEFAULT_ENCODING
@@ -166,14 +191,63 @@ def create_validator(
     return FunctionValidator
 
 
-def remove_emojis(s: str) -> str:
-    import demoji
+def json_unpickle(
+    file_name: Union[Text, Path], encode_non_string_keys: bool = False
+) -> Any:
+    """Unpickle an object from file using json.
+
+    Args:
+        file_name: the file to load the object from
+        encode_non_string_keys: If set to `True` then jsonpickle will encode non-string
+          dictionary keys instead of coercing them into strings via `repr()`.
+
+    Returns: the object
+    """
+    import jsonpickle.ext.numpy as jsonpickle_numpy
+    import jsonpickle
+
+    jsonpickle_numpy.register_handlers()
+
+    file_content = rasa.shared.utils.io.read_file(file_name)
+    return jsonpickle.loads(file_content, keys=encode_non_string_keys)
+
+
+def json_pickle(
+    file_name: Union[Text, Path], obj: Any, encode_non_string_keys: bool = False
+) -> None:
+    """Pickle an object to a file using json.
+
+    Args:
+        file_name: the file to store the object to
+        obj: the object to store
+        encode_non_string_keys: If set to `True` then jsonpickle will encode non-string
+          dictionary keys instead of coercing them into strings via `repr()`.
+    """
+    import jsonpickle.ext.numpy as jsonpickle_numpy
+    import jsonpickle
+
+    jsonpickle_numpy.register_handlers()
 
-    replaced = demoji.replace(s)
-    if replaced == s:
-        return s
-    # remove duplicate or trailing whitespaces if emojis were removed
-    return re.sub(r" +", " ", replaced).strip()
+    rasa.shared.utils.io.write_text_file(
+        jsonpickle.dumps(obj, keys=encode_non_string_keys), file_name
+    )
+
+
+def get_emoji_regex() -> Pattern:
+    """Returns regex to identify emojis."""
+    return re.compile(
+        "["
+        "\U0001f600-\U0001f64f"  # emoticons
+        "\U0001f300-\U0001f5ff"  # symbols & pictographs
+        "\U0001f680-\U0001f6ff"  # transport & map symbols
+        "\U0001f1e0-\U0001f1ff"  # flags (iOS)
+        "\U00002702-\U000027b0"
+        "\U000024c2-\U0001f251"
+        "\u200d"  # zero width joiner
+        "\u200c"  # zero width non-joiner
+        "]+",
+        flags=re.UNICODE,
+    )
 
 
 def are_directories_equal(dir1: Path, dir2: Path) -> bool:

rasa/utils/log_utils.py

@@ -78,7 +78,6 @@ def _anonymizer(
 
 def configure_structlog(
     log_level: Optional[int] = None,
-    include_time: bool = False,
 ) -> None:
     """Configure logging of the server."""
     if log_level is None:  # Log level NOTSET is 0 so we use `is None` here
@@ -115,9 +114,6 @@ def configure_structlog(
         SentryProcessor(event_level=logging.FATAL),
     ]
 
-    if include_time:
-        shared_processors.append(structlog.processors.TimeStamper(fmt="iso"))
-
     if not FORCE_JSON_LOGGING and sys.stderr.isatty():
         # Pretty printing when we run in a terminal session.
         # Automatically prints pretty tracebacks when "rich" is installed
@@ -147,14 +143,11 @@ def configure_structlog(
         # logger.
         cache_logger_on_first_use=True,
     )
-    # doing logger creation inline, to prevent usage of unconfigured logger
-    structlog.get_logger().debug("structlog.configured")
 
 
 def log_llm(logger: Any, log_module: str, log_event: str, **kwargs: Any) -> None:
-    """Logs LLM-specific events depending on a flag passed through an env var.
-
-    If the module's flag is set to INFO (e.g.
+    """Logs LLM-specific events depending on a flag passed through an environment
+    variable. If the module's flag is set to INFO (e.g.
     LOG_PROMPT_LLM_COMMAND_GENERATOR=INFO), its prompt is logged at INFO level,
     overriding the general log level setting.
 

rasa/utils/tensorflow/model_data.py

@@ -20,8 +20,6 @@ import numpy as np
 import scipy.sparse
 from sklearn.model_selection import train_test_split
 
-from rasa.utils.tensorflow.feature_array import FeatureArray
-
 logger = logging.getLogger(__name__)
 
 
@@ -39,6 +37,199 @@ def ragged_array_to_ndarray(ragged_array: Iterable[np.ndarray]) -> np.ndarray:
         return np.array(ragged_array, dtype=object)
 
 
+class FeatureArray(np.ndarray):
+    """Stores any kind of features ready to be used by a RasaModel.
+
+    Next to the input numpy array of features, it also received the number of
+    dimensions of the features.
+    As our features can have 1 to 4 dimensions we might have different number of numpy
+    arrays stacked. The number of dimensions helps us to figure out how to handle this
+    particular feature array. Also, it is automatically determined whether the feature
+    array is sparse or not and the number of units is determined as well.
+
+    Subclassing np.array: https://numpy.org/doc/stable/user/basics.subclassing.html
+    """
+
+    def __new__(
+        cls, input_array: np.ndarray, number_of_dimensions: int
+    ) -> "FeatureArray":
+        """Create and return a new object.  See help(type) for accurate signature."""
+        FeatureArray._validate_number_of_dimensions(number_of_dimensions, input_array)
+
+        feature_array = np.asarray(input_array).view(cls)
+
+        if number_of_dimensions <= 2:
+            feature_array.units = input_array.shape[-1]
+            feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix)
+        elif number_of_dimensions == 3:
+            feature_array.units = input_array[0].shape[-1]
+            feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix)
+        elif number_of_dimensions == 4:
+            feature_array.units = input_array[0][0].shape[-1]
+            feature_array.is_sparse = isinstance(
+                input_array[0][0], scipy.sparse.spmatrix
+            )
+        else:
+            raise ValueError(
+                f"Number of dimensions '{number_of_dimensions}' currently not "
+                f"supported."
+            )
+
+        feature_array.number_of_dimensions = number_of_dimensions
+
+        return feature_array
+
+    def __init__(
+        self, input_array: Any, number_of_dimensions: int, **kwargs: Any
+    ) -> None:
+        """Initialize. FeatureArray.
+
+        Needed in order to avoid 'Invalid keyword argument number_of_dimensions
+        to function FeatureArray.__init__ '
+        Args:
+            input_array: the array that contains features
+            number_of_dimensions: number of dimensions in input_array
+        """
+        super().__init__(**kwargs)
+        self.number_of_dimensions = number_of_dimensions
+
+    def __array_finalize__(self, obj: Optional[np.ndarray]) -> None:
+        """This method is called when the system allocates a new array from obj.
+
+        Args:
+            obj: A subclass (subtype) of ndarray.
+        """
+        if obj is None:
+            return
+
+        self.units = getattr(obj, "units", None)
+        self.number_of_dimensions = getattr(obj, "number_of_dimensions", None)  # type: ignore[assignment]
+        self.is_sparse = getattr(obj, "is_sparse", None)
+
+        default_attributes = {
+            "units": self.units,
+            "number_of_dimensions": self.number_of_dimensions,
+            "is_spare": self.is_sparse,
+        }
+        self.__dict__.update(default_attributes)
+
+    # pytype: disable=attribute-error
+    def __array_ufunc__(
+        self, ufunc: Any, method: Text, *inputs: Any, **kwargs: Any
+    ) -> Any:
+        """Overwrite this method as we are subclassing numpy array.
+
+        Args:
+            ufunc: The ufunc object that was called.
+            method: A string indicating which Ufunc method was called
+                    (one of "__call__", "reduce", "reduceat", "accumulate", "outer",
+                    "inner").
+            *inputs: A tuple of the input arguments to the ufunc.
+            **kwargs: Any additional arguments
+
+        Returns:
+            The result of the operation.
+        """
+        f = {
+            "reduce": ufunc.reduce,
+            "accumulate": ufunc.accumulate,
+            "reduceat": ufunc.reduceat,
+            "outer": ufunc.outer,
+            "at": ufunc.at,
+            "__call__": ufunc,
+        }
+        # convert the inputs to np.ndarray to prevent recursion, call the function,
+        # then cast it back as FeatureArray
+        output = FeatureArray(
+            f[method](*(i.view(np.ndarray) for i in inputs), **kwargs),
+            number_of_dimensions=kwargs["number_of_dimensions"],
+        )
+        output.__dict__ = self.__dict__  # carry forward attributes
+        return output
+
+    def __reduce__(self) -> Tuple[Any, Any, Any]:
+        """Needed in order to pickle this object.
+
+        Returns:
+            A tuple.
+        """
+        pickled_state = super(FeatureArray, self).__reduce__()
+        if isinstance(pickled_state, str):
+            raise TypeError("np array __reduce__ returned string instead of tuple.")
+        new_state = pickled_state[2] + (
+            self.number_of_dimensions,
+            self.is_sparse,
+            self.units,
+        )
+        return pickled_state[0], pickled_state[1], new_state
+
+    def __setstate__(self, state: Any, **kwargs: Any) -> None:
+        """Sets the state.
+
+        Args:
+            state: The state argument must be a sequence that contains the following
+                   elements version, shape, dtype, isFortan, rawdata.
+            **kwargs: Any additional parameter
+        """
+        # Needed in order to load the object
+        self.number_of_dimensions = state[-3]
+        self.is_sparse = state[-2]
+        self.units = state[-1]
+        super(FeatureArray, self).__setstate__(state[0:-3], **kwargs)
+
+    # pytype: enable=attribute-error
+
+    @staticmethod
+    def _validate_number_of_dimensions(
+        number_of_dimensions: int, input_array: np.ndarray
+    ) -> None:
+        """Validates if the the input array has given number of dimensions.
+
+        Args:
+            number_of_dimensions: number of dimensions
+            input_array: input array
+
+        Raises: ValueError in case the dimensions do not match
+        """
+        _sub_array = input_array
+        dim = 0
+        # Go number_of_dimensions into the given input_array
+        for i in range(1, number_of_dimensions + 1):
+            _sub_array = _sub_array[0]
+            if isinstance(_sub_array, scipy.sparse.spmatrix):
+                dim = i
+                break
+            if isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0:
+                # sequence dimension is 0, we are dealing with "fake" features
+                dim = i
+                break
+
+        # If the resulting sub_array is sparse, the remaining number of dimensions
+        # should be at least 2
+        if isinstance(_sub_array, scipy.sparse.spmatrix):
+            if dim > 2:
+                raise ValueError(
+                    f"Given number of dimensions '{number_of_dimensions}' does not "
+                    f"match dimensions of given input array: {input_array}."
+                )
+        elif isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0:
+            # sequence dimension is 0, we are dealing with "fake" features,
+            # but they should be of dim 2
+            if dim > 2:
+                raise ValueError(
+                    f"Given number of dimensions '{number_of_dimensions}' does not "
+                    f"match dimensions of given input array: {input_array}."
+                )
+        # If the resulting sub_array is dense, the sub_array should be a single number
+        elif not np.issubdtype(type(_sub_array), np.integer) and not isinstance(
+            _sub_array, (np.float32, np.float64)
+        ):
+            raise ValueError(
+                f"Given number of dimensions '{number_of_dimensions}' does not match "
+                f"dimensions of given input array: {input_array}."
+            )
+
+
 class FeatureSignature(NamedTuple):
     """Signature of feature arrays.
 

rasa/validator.py

@@ -1,7 +1,6 @@
 import logging
 import re
 import string
-import sys
 from collections import defaultdict
 from typing import Set, Text, Optional, Dict, Any, List, Tuple
 
@@ -23,11 +22,6 @@ from rasa.shared.core.flows.steps.collect import CollectInformationFlowStep
 from rasa.shared.core.flows.steps.action import ActionFlowStep
 from rasa.shared.core.flows.steps.link import LinkFlowStep
 from rasa.shared.core.flows import FlowsList
-from rasa.shared.core.flows.utils import (
-    warn_deprecated_collect_step_config,
-    get_duplicate_slot_persistence_config_error_message,
-    get_invalid_slot_persistence_config_error_message,
-)
 import rasa.shared.nlu.constants
 from rasa.shared.constants import (
     ASSISTANT_ID_DEFAULT_VALUE,
@@ -61,7 +55,6 @@ from rasa.shared.importers.importer import TrainingDataImporter
 from rasa.shared.nlu.constants import COMMANDS
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
-
 import rasa.shared.utils.cli
 import rasa.shared.utils.io
 
@@ -305,7 +298,7 @@ class Validator:
             return any(cls.check_for_placeholder(i) for i in value)
         return False
 
-    def check_for_no_empty_parenthesis_in_responses(self) -> bool:
+    def check_for_no_empty_paranthesis_in_responses(self) -> bool:
         """Checks if there are no empty parenthesis in utterances."""
         everything_is_alright = True
 
@@ -316,12 +309,12 @@ class Validator:
                     for key in RESPONSE_KEYS_TO_INTERPOLATE
                 ):
                     structlogger.error(
-                        "validator.empty_parenthesis_in_utterances",
+                        "validator.empty_paranthesis_in_utterances",
                         response=response_text,
                         event_info=(
                             f"The response '{response_text}' in the domain file "
-                            f"contains empty parenthesis, which is not permitted. "
-                            f"Please remove the empty parenthesis."
+                            f"contains empty parenthesis, which is not permitted."
+                            f" Please remove the empty parenthesis."
                         ),
                     )
                     everything_is_alright = False
@@ -1244,7 +1237,6 @@ class Validator:
             self.verify_flows_steps_against_domain(),
             self.verify_unique_flows(),
             self.verify_predicates(),
-            self.verify_slot_persistence_configuration(),
         ]
 
         all_good = all(flow_validation_conditions)
@@ -1541,101 +1533,3 @@ class Validator:
             ),
         )
         return False
-
-    def verify_slot_persistence_configuration(self) -> bool:
-        """Verifies the validity of slot persistence after flow ends configuration.
-
-        Only slots used in either a collect step or a set_slot step can be persisted and
-        the configuration can either set at the flow level or the collect step level,
-        but not both.
-
-        Returns:
-            bool: True if all slot persistence configuration is valid, False otherwise.
-
-        Raises:
-            DeprecationWarning: If reset_after_flow_ends is used in collect steps.
-        """
-        all_good = True
-
-        for flow in self.flows.underlying_flows:
-            flow_id = flow.id
-            persist_slots = flow.persisted_slots
-            has_flow_level_persistence = True if persist_slots else False
-            flow_slots = set()
-
-            for step in flow.steps_with_calls_resolved:
-                if isinstance(step, SetSlotsFlowStep):
-                    flow_slots.update([slot["key"] for slot in step.slots])
-
-                elif isinstance(step, CollectInformationFlowStep):
-                    collect_step = step.collect
-                    flow_slots.add(collect_step)
-                    if not step.reset_after_flow_ends:
-                        warn_deprecated_collect_step_config(flow_id, collect_step)
-
-                        if has_flow_level_persistence:
-                            structlogger.error(
-                                "validator.verify_slot_persistence_configuration.duplicate_config",
-                                flow=flow_id,
-                                collect_step=collect_step,
-                                event_info=get_duplicate_slot_persistence_config_error_message(
-                                    flow_id, collect_step
-                                ),
-                            )
-                            all_good = False
-
-            if has_flow_level_persistence:
-                if not self._is_persist_slots_valid(persist_slots, flow_slots, flow_id):
-                    all_good = False
-        return all_good
-
-    def _is_persist_slots_valid(
-        self, persist_slots: List[str], flow_slots: Set[str], flow_id: str
-    ) -> bool:
-        invalid_slots = set(persist_slots) - flow_slots
-        is_valid = False if invalid_slots else True
-
-        if invalid_slots:
-            structlogger.error(
-                "validator.verify_slot_persistence_configuration.invalid_persist_slot",
-                flow=flow_id,
-                event_info=get_invalid_slot_persistence_config_error_message(
-                    flow_id, invalid_slots
-                ),
-            )
-        return is_valid
-
-    def verify_studio_supported_validations(self) -> bool:
-        """Validates the assistant project for Rasa Studio supported features.
-
-        Ensure to add new validations here if they are required for
-        Rasa Studio Upload CLI.
-        """
-        if self.domain.is_empty():
-            structlogger.error(
-                "rasa.validator.verify_studio_supported_validations.empty_domain",
-                event_info="Encountered empty domain during validation.",
-            )
-            sys.exit(1)
-
-        self.warn_if_config_mandatory_keys_are_not_set()
-
-        valid_responses = (
-            self.check_for_no_empty_parenthesis_in_responses()
-            and self.validate_button_payloads()
-        )
-        valid_nlu = self.verify_nlu()
-        valid_flows = all(
-            [
-                self.verify_flows_steps_against_domain(),
-                self.verify_unique_flows(),
-                self.verify_predicates(),
-            ]
-        )
-        valid_calm_slot_mappings = self.validate_CALM_slot_mappings()
-
-        all_good = (
-            valid_responses and valid_nlu and valid_flows and valid_calm_slot_mappings
-        )
-
-        return all_good

rasa/version.py

@@ -1,3 +1,3 @@
 # this file will automatically be changed,
 # do not add anything but the version number here!
-__version__ = "3.11.0"
+__version__ = "3.11.0a2"