rasa-pro 3.8.16__py3-none-any.whl

rasa/shared/utils/common.py

@@ -0,0 +1,308 @@
+import asyncio
+import functools
+import importlib
+import inspect
+import logging
+import pkgutil
+import sys
+from types import ModuleType
+from typing import Text, Dict, Optional, Any, List, Callable, Collection, Type
+
+import rasa.shared.utils.io
+from rasa.shared.constants import DOCS_URL_MIGRATION_GUIDE
+from rasa.shared.exceptions import RasaException
+
+logger = logging.getLogger(__name__)
+
+
+def class_from_module_path(
+    module_path: Text, lookup_path: Optional[Text] = None
+) -> Type:
+    """Given the module name and path of a class, tries to retrieve the class.
+
+    The loaded class can be used to instantiate new objects.
+
+    Args:
+        module_path: either an absolute path to a Python class,
+                     or the name of the class in the local / global scope.
+        lookup_path: a path where to load the class from, if it cannot
+                     be found in the local / global scope.
+
+    Returns:
+        a Python class
+
+    Raises:
+        ImportError, in case the Python class cannot be found.
+        RasaException, in case the imported result is something other than a class
+    """
+    warn_and_exit_if_module_path_contains_rasa_plus(module_path, lookup_path)
+
+    klass = None
+    if "." in module_path:
+        module_name, _, class_name = module_path.rpartition(".")
+        m = importlib.import_module(module_name)
+        klass = getattr(m, class_name, None)
+    elif lookup_path:
+        # try to import the class from the lookup path
+        m = importlib.import_module(lookup_path)
+        klass = getattr(m, module_path, None)
+
+    if klass is None:
+        raise ImportError(f"Cannot retrieve class from path {module_path}.")
+
+    if not inspect.isclass(klass):
+        raise RasaException(
+            f"`class_from_module_path()` is expected to return a class, "
+            f"but for {module_path} we got a {type(klass)}."
+        )
+    return klass
+
+
+def import_package_modules(package_name: str) -> List[ModuleType]:
+    """Import all modules in a package."""
+    package = importlib.import_module(package_name)
+    return [
+        importlib.import_module(f"{package_name}.{module_name}")
+        for _, module_name, _ in pkgutil.iter_modules(package.__path__)
+    ]
+
+
+def all_subclasses(cls: Any) -> List[Any]:
+    """Returns all known (imported) subclasses of a class."""
+    classes = cls.__subclasses__() + [
+        g for s in cls.__subclasses__() for g in all_subclasses(s)
+    ]
+
+    return [subclass for subclass in classes if not inspect.isabstract(subclass)]
+
+
+def module_path_from_instance(inst: Any) -> Text:
+    """Return the module path of an instance's class."""
+    return inst.__module__ + "." + inst.__class__.__name__
+
+
+def sort_list_of_dicts_by_first_key(dicts: List[Dict]) -> List[Dict]:
+    """Sorts a list of dictionaries by their first key."""
+    return sorted(dicts, key=lambda d: next(iter(d.keys())))
+
+
+def lazy_property(function: Callable) -> Any:
+    """Allows to avoid recomputing a property over and over.
+
+    The result gets stored in a local var. Computation of the property
+    will happen once, on the first call of the property. All
+    succeeding calls will use the value stored in the private property.
+    """
+    attr_name = "_lazy_" + function.__name__
+
+    def _lazyprop(self: Any) -> Any:
+        if not hasattr(self, attr_name):
+            setattr(self, attr_name, function(self))
+        return getattr(self, attr_name)
+
+    return property(_lazyprop)
+
+
+def cached_method(f: Callable[..., Any]) -> Callable[..., Any]:
+    """Caches method calls based on the call's `args` and `kwargs`.
+
+    Works for `async` and `sync` methods. Don't apply this to functions.
+
+    Args:
+        f: The decorated method whose return value should be cached.
+
+    Returns:
+        The return value which the method gives for the first call with the given
+        arguments.
+    """
+    assert "self" in arguments_of(f), "This decorator can only be used with methods."
+
+    class Cache:
+        """Helper class to abstract the caching details."""
+
+        def __init__(self, caching_object: object, args: Any, kwargs: Any) -> None:
+            self.caching_object = caching_object
+            self.cache = getattr(caching_object, self._cache_name(), {})
+            # noinspection PyUnresolvedReferences
+            self.cache_key = functools._make_key(args, kwargs, typed=False)
+
+        def _cache_name(self) -> Text:
+            return f"_cached_{self.caching_object.__class__.__name__}_{f.__name__}"
+
+        def is_cached(self) -> bool:
+            return self.cache_key in self.cache
+
+        def cache_result(self, result: Any) -> None:
+            self.cache[self.cache_key] = result
+            setattr(self.caching_object, self._cache_name(), self.cache)
+
+        def cached_result(self) -> Any:
+            return self.cache[self.cache_key]
+
+    if asyncio.iscoroutinefunction(f):
+
+        @functools.wraps(f)
+        async def decorated(self: object, *args: Any, **kwargs: Any) -> Any:
+            cache = Cache(self, args, kwargs)
+            if not cache.is_cached():
+                # Store the task immediately so that other concurrent calls of the
+                # method can re-use the same task and don't schedule a second execution.
+                to_cache = asyncio.ensure_future(f(self, *args, **kwargs))
+                cache.cache_result(to_cache)
+            return await cache.cached_result()
+
+        return decorated
+    else:
+
+        @functools.wraps(f)
+        def decorated(self: object, *args: Any, **kwargs: Any) -> Any:
+            cache = Cache(self, args, kwargs)
+            if not cache.is_cached():
+                to_cache = f(self, *args, **kwargs)
+                cache.cache_result(to_cache)
+            return cache.cached_result()
+
+        return decorated
+
+
+def transform_collection_to_sentence(collection: Collection[Text]) -> Text:
+    """Transforms e.g. a list like ['A', 'B', 'C'] into a sentence 'A, B and C'."""
+    x = list(collection)
+    if len(x) >= 2:
+        return ", ".join(map(str, x[:-1])) + " and " + x[-1]
+    return "".join(collection)
+
+
+def minimal_kwargs(
+    kwargs: Dict[Text, Any], func: Callable, excluded_keys: Optional[List] = None
+) -> Dict[Text, Any]:
+    """Returns only the kwargs which are required by a function. Keys, contained in
+    the exception list, are not included.
+
+    Args:
+        kwargs: All available kwargs.
+        func: The function which should be called.
+        excluded_keys: Keys to exclude from the result.
+
+    Returns:
+        Subset of kwargs which are accepted by `func`.
+
+    """
+    excluded_keys = excluded_keys or []
+
+    possible_arguments = arguments_of(func)
+
+    return {
+        k: v
+        for k, v in kwargs.items()
+        if k in possible_arguments and k not in excluded_keys
+    }
+
+
+def mark_as_experimental_feature(feature_name: Text) -> None:
+    """Warns users that they are using an experimental feature."""
+    logger.warning(
+        f"The {feature_name} is currently experimental and might change or be "
+        "removed in the future 🔬 Please share your feedback on it in the "
+        "forum (https://forum.rasa.com) to help us make this feature "
+        "ready for production."
+    )
+
+
+def arguments_of(func: Callable) -> List[Text]:
+    """Return the parameters of the function `func` as a list of names."""
+    import inspect
+
+    return list(inspect.signature(func).parameters.keys())
+
+
+def extract_duplicates(list1: List[Any], list2: List[Any]) -> List[Any]:
+    """Extracts duplicates from two lists."""
+    if list1:
+        dict1 = {
+            (sorted(list(i.keys()))[0] if isinstance(i, dict) else i): i for i in list1
+        }
+    else:
+        dict1 = {}
+
+    if list2:
+        dict2 = {
+            (sorted(list(i.keys()))[0] if isinstance(i, dict) else i): i for i in list2
+        }
+    else:
+        dict2 = {}
+
+    set1 = set(dict1.keys())
+    set2 = set(dict2.keys())
+    dupes = set1.intersection(set2)
+    return sorted(list(dupes))
+
+
+def clean_duplicates(dupes: Dict[Text, Any]) -> Dict[Text, Any]:
+    """Removes keys for empty values."""
+    duplicates = dupes.copy()
+    for k in dupes:
+        if not dupes[k]:
+            duplicates.pop(k)
+
+    return duplicates
+
+
+def merge_dicts(
+    tempDict1: Dict[Text, Any],
+    tempDict2: Dict[Text, Any],
+    override_existing_values: bool = False,
+) -> Dict[Text, Any]:
+    """Merges two dicts."""
+    if override_existing_values:
+        merged_dicts, b = tempDict1.copy(), tempDict2.copy()
+
+    else:
+        merged_dicts, b = tempDict2.copy(), tempDict1.copy()
+    merged_dicts.update(b)
+    return merged_dicts
+
+
+def merge_lists(
+    list1: List[Any], list2: List[Any], override: bool = False
+) -> List[Any]:
+    """Merges two lists."""
+    return sorted(list(set(list1 + list2)))
+
+
+def merge_lists_of_dicts(
+    dict_list1: List[Dict],
+    dict_list2: List[Dict],
+    override_existing_values: bool = False,
+) -> List[Dict]:
+    """Merges two dict lists."""
+    dict1 = {
+        (sorted(list(i.keys()))[0] if isinstance(i, dict) else i): i for i in dict_list1
+    }
+    dict2 = {
+        (sorted(list(i.keys()))[0] if isinstance(i, dict) else i): i for i in dict_list2
+    }
+    merged_dicts = merge_dicts(dict1, dict2, override_existing_values)
+    return list(merged_dicts.values())
+
+
+def warn_and_exit_if_module_path_contains_rasa_plus(
+    module_path: Text, lookup_path: Optional[str] = None
+) -> None:
+    """Warns and exits if the module path contains `rasa_plus`.
+
+    Args:
+        module_path: The module path to check.
+        lookup_path: The lookup path to check.
+    """
+    if "rasa_plus" in module_path.lower() or (
+        lookup_path and "rasa_plus" in lookup_path.lower()
+    ):
+        rasa.shared.utils.io.raise_warning(
+            f"Your module path '{module_path}' contains 'rasa_plus'. "
+            f"The path to this Rasa Pro component has changed, please "
+            f"follow the migration guide in the official documentation "
+            f"to update your code: ",
+            docs=DOCS_URL_MIGRATION_GUIDE,
+        )
+        sys.exit(1)

rasa/shared/utils/constants.py

@@ -0,0 +1 @@
+DEFAULT_ENCODING = "utf-8"

rasa/shared/utils/io.py

@@ -0,0 +1,403 @@
+from collections import OrderedDict
+import errno
+import glob
+from hashlib import md5
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Text, Type, Union
+import warnings
+import random
+import string
+import portalocker
+
+from rasa.shared.constants import (
+    DEFAULT_LOG_LEVEL,
+    ENV_LOG_LEVEL,
+    NEXT_MAJOR_VERSION_FOR_DEPRECATIONS,
+)
+from rasa.shared.exceptions import (
+    FileIOException,
+    FileNotFoundException,
+)
+from rasa.shared.utils.constants import DEFAULT_ENCODING
+
+
+class bcolors:
+    HEADER = "\033[95m"
+    OKBLUE = "\033[94m"
+    OKGREEN = "\033[92m"
+    WARNING = "\033[93m"
+    FAIL = "\033[91m"
+    ENDC = "\033[0m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+
+
+def wrap_with_color(*args: Any, color: Text) -> Text:
+    return color + " ".join(str(s) for s in args) + bcolors.ENDC
+
+
+def raise_warning(
+    message: Text,
+    category: Optional[Type[Warning]] = None,
+    docs: Optional[Text] = None,
+    **kwargs: Any,
+) -> None:
+    """Emit a `warnings.warn` with sensible defaults and a colored warning msg."""
+    original_formatter = warnings.formatwarning
+
+    def should_show_source_line() -> bool:
+        if "stacklevel" not in kwargs:
+            if category == UserWarning or category is None:
+                return False
+            if category == FutureWarning:
+                return False
+        return True
+
+    def formatwarning(
+        message: Union[Warning, Text],
+        category: Type[Warning],
+        filename: Text,
+        lineno: int,
+        line: Optional[Text] = None,
+    ) -> Text:
+        """Function to format a warning the standard way."""
+        if not should_show_source_line():
+            if docs:
+                line = f"More info at {docs}"
+            else:
+                line = ""
+
+        formatted_message = original_formatter(
+            message, category, filename, lineno, line
+        )
+        return wrap_with_color(formatted_message, color=bcolors.WARNING)
+
+    if "stacklevel" not in kwargs:
+        # try to set useful defaults for the most common warning categories
+        if category == DeprecationWarning:
+            kwargs["stacklevel"] = 3
+        elif category in (UserWarning, FutureWarning):
+            kwargs["stacklevel"] = 2
+
+    warnings.formatwarning = formatwarning
+    warnings.warn(message, category=category, **kwargs)
+    warnings.formatwarning = original_formatter
+
+
+def write_text_file(
+    content: Text,
+    file_path: Union[Text, Path],
+    encoding: Text = DEFAULT_ENCODING,
+    append: bool = False,
+) -> None:
+    """Writes text to a file.
+
+    Args:
+        content: The content to write.
+        file_path: The path to which the content should be written.
+        encoding: The encoding which should be used.
+        append: Whether to append to the file or to truncate the file.
+
+    """
+    mode = "a" if append else "w"
+    with open(file_path, mode, encoding=encoding) as file:
+        file.write(content)
+
+
+def read_file(filename: Union[Text, Path], encoding: Text = DEFAULT_ENCODING) -> Any:
+    """Read text from a file."""
+    try:
+        with open(filename, encoding=encoding) as f:
+            return f.read()
+    except FileNotFoundError:
+        raise FileNotFoundException(
+            f"Failed to read file, " f"'{os.path.abspath(filename)}' does not exist."
+        )
+    except UnicodeDecodeError:
+        raise FileIOException(
+            f"Failed to read file '{os.path.abspath(filename)}', "
+            f"could not read the file using {encoding} to decode "
+            f"it. Please make sure the file is stored with this "
+            f"encoding."
+        )
+
+
+def read_json_file(filename: Union[Text, Path]) -> Any:
+    """Read json from a file."""
+    content = read_file(filename)
+    try:
+        return json.loads(content)
+    except ValueError as e:
+        raise FileIOException(
+            f"Failed to read json from '{os.path.abspath(filename)}'. Error: {e}"
+        )
+
+
+def list_directory(path: Text) -> List[Text]:
+    """Returns all files and folders excluding hidden files.
+
+    If the path points to a file, returns the file. This is a recursive
+    implementation returning files in any depth of the path.
+    """
+    if not isinstance(path, str):
+        raise ValueError(
+            f"`resource_name` must be a string type. " f"Got `{type(path)}` instead"
+        )
+
+    if os.path.isfile(path):
+        return [path]
+    elif os.path.isdir(path):
+        results: List[Text] = []
+        for base, dirs, files in os.walk(path, followlinks=True):
+            # sort files for same order across runs
+            files = sorted(files, key=_filename_without_prefix)
+            # add not hidden files
+            good_files = filter(lambda x: not x.startswith("."), files)
+            results.extend(os.path.join(base, f) for f in good_files)
+            # add not hidden directories
+            good_directories = filter(lambda x: not x.startswith("."), dirs)
+            results.extend(os.path.join(base, f) for f in good_directories)
+        return results
+    else:
+        raise ValueError(f"Could not locate the resource '{os.path.abspath(path)}'.")
+
+
+def list_files(path: Text) -> List[Text]:
+    """Returns all files excluding hidden files.
+
+    If the path points to a file, returns the file.
+    """
+    return [fn for fn in list_directory(path) if os.path.isfile(fn)]
+
+
+def _filename_without_prefix(file: Text) -> Text:
+    """Splits of a filenames prefix until after the first ``_``."""
+    return "_".join(file.split("_")[1:])
+
+
+def list_subdirectories(path: Text) -> List[Text]:
+    """Returns all folders excluding hidden files.
+
+    If the path points to a file, returns an empty list.
+    """
+    return [fn for fn in glob.glob(os.path.join(path, "*")) if os.path.isdir(fn)]
+
+
+def deep_container_fingerprint(
+    obj: Union[List[Any], Dict[Any, Any], Any], encoding: Text = DEFAULT_ENCODING
+) -> Text:
+    """Calculate a hash which is stable.
+
+    Works for lists and dictionaries. For keys and values, we recursively call
+    `hash(...)` on them. In case of a dict, the hash is independent of the containers
+    key order. Keep in mind that a list with items in a different order
+    will not create the same hash!
+
+    Args:
+        obj: dictionary or list to be hashed.
+        encoding: encoding used for dumping objects as strings
+
+    Returns:
+        hash of the container.
+    """
+    if isinstance(obj, dict):
+        return get_dictionary_fingerprint(obj, encoding)
+    elif isinstance(obj, list):
+        return get_list_fingerprint(obj, encoding)
+    elif hasattr(obj, "fingerprint") and callable(obj.fingerprint):
+        return obj.fingerprint()
+    else:
+        return get_text_hash(str(obj), encoding)
+
+
+def get_dictionary_fingerprint(
+    dictionary: Dict[Any, Any], encoding: Text = DEFAULT_ENCODING
+) -> Text:
+    """Calculate the fingerprint for a dictionary.
+
+    The dictionary can contain any keys and values which are either a dict,
+    a list or a elements which can be dumped as a string.
+
+    Args:
+        dictionary: dictionary to be hashed
+        encoding: encoding used for dumping objects as strings
+
+    Returns:
+        The hash of the dictionary
+    """
+    stringified = json.dumps(
+        {
+            deep_container_fingerprint(k, encoding): deep_container_fingerprint(
+                v, encoding
+            )
+            for k, v in dictionary.items()
+        },
+        sort_keys=True,
+    )
+    return get_text_hash(stringified, encoding)
+
+
+def get_list_fingerprint(
+    elements: List[Any], encoding: Text = DEFAULT_ENCODING
+) -> Text:
+    """Calculate a fingerprint for an unordered list.
+
+    Args:
+        elements: unordered list
+        encoding: encoding used for dumping objects as strings
+
+    Returns:
+        the fingerprint of the list
+    """
+    stringified = json.dumps(
+        [deep_container_fingerprint(element, encoding) for element in elements]
+    )
+    return get_text_hash(stringified, encoding)
+
+
+def get_text_hash(text: Text, encoding: Text = DEFAULT_ENCODING) -> Text:
+    """Calculate the md5 hash for a text."""
+    # deepcode ignore InsecureHash: Not used for a cryptographic purpose
+    return md5(text.encode(encoding)).hexdigest()  # nosec
+
+
+def json_to_string(obj: Any, **kwargs: Any) -> Text:
+    """Dumps a JSON-serializable object to string.
+
+    Args:
+        obj: JSON-serializable object.
+        kwargs: serialization options. Defaults to 2 space indentation
+                and disable escaping of non-ASCII characters.
+
+    Returns:
+        The objects serialized to JSON, as a string.
+    """
+    indent = kwargs.pop("indent", 2)
+    ensure_ascii = kwargs.pop("ensure_ascii", False)
+    return json.dumps(obj, indent=indent, ensure_ascii=ensure_ascii, **kwargs)
+
+
+def convert_to_ordered_dict(obj: Any) -> Any:
+    """Convert object to an `OrderedDict`.
+
+    Args:
+        obj: Object to convert.
+
+    Returns:
+        An `OrderedDict` with all nested dictionaries converted if `obj` is a
+        dictionary, otherwise the object itself.
+    """
+    if isinstance(obj, OrderedDict):
+        return obj
+    # use recursion on lists
+    if isinstance(obj, list):
+        return [convert_to_ordered_dict(element) for element in obj]
+
+    if isinstance(obj, dict):
+        out = OrderedDict()
+        # use recursion on dictionaries
+        for k, v in obj.items():
+            out[k] = convert_to_ordered_dict(v)
+
+        return out
+
+    # return all other objects
+    return obj
+
+
+def is_logging_disabled() -> bool:
+    """Returns `True` if log level is set to WARNING or ERROR, `False` otherwise."""
+    log_level = os.environ.get(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL)
+
+    return log_level in ("ERROR", "WARNING")
+
+
+def create_directory_for_file(file_path: Union[Text, Path]) -> None:
+    """Creates any missing parent directories of this file path."""
+    create_directory(os.path.dirname(file_path))
+
+
+def dump_obj_as_json_to_file(filename: Union[Text, Path], obj: Any) -> None:
+    """Dump an object as a json string to a file."""
+    write_text_file(json.dumps(obj, ensure_ascii=False, indent=2), filename)
+
+
+def create_directory(directory_path: Text) -> None:
+    """Creates a directory and its super paths.
+
+    Succeeds even if the path already exists.
+    """
+    try:
+        os.makedirs(directory_path)
+    except OSError as e:
+        # be happy if someone already created the path
+        if e.errno != errno.EEXIST:
+            raise
+
+
+def raise_deprecation_warning(
+    message: Text,
+    warn_until_version: Text = NEXT_MAJOR_VERSION_FOR_DEPRECATIONS,
+    docs: Optional[Text] = None,
+    **kwargs: Any,
+) -> None:
+    """Thin wrapper around `raise_warning()` to raise a deprecation warning.
+
+    It requires
+    a version until which we'll warn, and after which the support for the feature will
+    be removed.
+    """
+    if warn_until_version not in message:
+        message = f"{message} (will be removed in {warn_until_version})"
+
+    # need the correct stacklevel now
+    kwargs.setdefault("stacklevel", 3)
+    # we're raising a `FutureWarning` instead of a `DeprecationWarning` because
+    # we want these warnings to be visible in the terminal of our users
+    # https://docs.python.org/3/library/warnings.html#warning-categories
+    raise_warning(message, FutureWarning, docs, **kwargs)
+
+
+def is_subdirectory(path: Text, potential_parent_directory: Text) -> bool:
+    """Checks if `path` is a subdirectory of `potential_parent_directory`.
+
+    Args:
+        path: Path to a file or directory.
+        potential_parent_directory: Potential parent directory.
+
+    Returns:
+        `True` if `path` is a subdirectory of `potential_parent_directory`.
+    """
+    if path is None or potential_parent_directory is None:
+        return False
+
+    path = os.path.abspath(path)
+    potential_parent_directory = os.path.abspath(potential_parent_directory)
+
+    return potential_parent_directory in path
+
+
+def random_string(length: int) -> Text:
+    """Returns a random string of given length."""
+    return "".join(random.choices(string.ascii_uppercase + string.digits, k=length))
+
+
+def handle_print_blocking(output: Text) -> None:
+    """Handle print blocking (BlockingIOError) by getting the STDOUT lock.
+
+    Args:
+        output: Text to be printed to STDOUT.
+    """
+    # Locking again to obtain STDOUT with a lock.
+    with portalocker.Lock(sys.stdout) as lock:
+        if sys.platform == "win32":
+            # colorama is used to fix a regression where colors can not be printed on
+            # windows. https://github.com/RasaHQ/rasa/issues/7053
+            from colorama import AnsiToWin32
+
+            lock = AnsiToWin32(lock).stream
+
+        print(output, file=lock, flush=True)