rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/utils/endpoints.py

@@ -0,0 +1,331 @@
+import os
+import ssl
+from types import ModuleType
+from typing import Any, List, Optional, Text, Dict, Union
+
+import aiohttp
+import structlog
+from aiohttp.client_exceptions import ContentTypeError
+from sanic.request import Request
+
+from rasa.core.constants import DEFAULT_REQUEST_TIMEOUT
+from rasa.shared.exceptions import FileNotFoundException
+from rasa.shared.utils.yaml import read_config_file
+
+structlogger = structlog.get_logger()
+
+
+def read_endpoint_config(
+    filename: Text, endpoint_type: Text
+) -> Optional["EndpointConfig"]:
+    """Read an endpoint configuration file from disk and extract one config."""
+    if not filename:
+        return None
+
+    try:
+        content = read_config_file(filename)
+
+        if content.get(endpoint_type) is None:
+            return None
+
+        return EndpointConfig.from_dict(content[endpoint_type])
+    except FileNotFoundError:
+        structlogger.error(
+            "endpoint.read.failed_no_such_file",
+            filename=os.path.abspath(filename),
+            event_info=(
+                "Failed to read endpoint configuration file - "
+                "the file was not found."
+            ),
+        )
+        return None
+
+
+def read_property_config_from_endpoints_file(
+    filename: str, property_name: str
+) -> Optional[Union[Dict[str, Any], List]]:
+    """Read a property from an endpoint configuration file."""
+    if not filename:
+        return None
+
+    try:
+        content = read_config_file(filename)
+
+        if content.get(property_name) is None:
+            return None
+
+        return content[property_name]
+    except FileNotFoundError:
+        structlogger.error(
+            "endpoint.read.failed_no_such_file",
+            filename=os.path.abspath(filename),
+            event_info=(
+                "Failed to read endpoint configuration file - "
+                "the file was not found."
+            ),
+        )
+        return None
+
+
+def concat_url(base: Text, subpath: Optional[Text]) -> Text:
+    """Append a subpath to a base url.
+
+    Strips leading slashes from the subpath if necessary. This behaves
+    differently than `urlparse.urljoin` and will not treat the subpath
+    as a base url if it starts with `/` but will always append it to the
+    `base`.
+
+    Args:
+        base: Base URL.
+        subpath: Optional path to append to the base URL.
+
+    Returns:
+        Concatenated URL with base and subpath.
+    """
+    if not subpath:
+        if base.endswith("/"):
+            structlogger.debug(
+                "endpoint.concat_url.trailing_slash",
+                url=base,
+                event_info=(
+                    "The URL has a trailing slash. Please make sure the "
+                    "target server supports trailing slashes for this endpoint."
+                ),
+            )
+        return base
+
+    url = base
+    if not base.endswith("/"):
+        url += "/"
+    if subpath.startswith("/"):
+        subpath = subpath[1:]
+    return url + subpath
+
+
+class EndpointConfig:
+    """Configuration for an external HTTP endpoint."""
+
+    def __init__(
+        self,
+        url: Optional[Text] = None,
+        params: Optional[Dict[Text, Any]] = None,
+        headers: Optional[Dict[Text, Any]] = None,
+        basic_auth: Optional[Dict[Text, Text]] = None,
+        token: Optional[Text] = None,
+        token_name: Text = "token",
+        cafile: Optional[Text] = None,
+        actions_module: Optional[Union[Text, ModuleType]] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Creates an `EndpointConfig` instance."""
+        self.url = url
+        self.params = params or {}
+        self.headers = headers or {}
+        self.basic_auth = basic_auth or {}
+        self.token = token
+        self.token_name = token_name
+        self.type = kwargs.pop("store_type", kwargs.pop("type", None))
+        self.cafile = cafile
+        self.actions_module = actions_module
+        self.kwargs = kwargs
+
+    def session(self) -> aiohttp.ClientSession:
+        """Creates and returns a configured aiohttp client session."""
+        # create authentication parameters
+        if self.basic_auth:
+            auth = aiohttp.BasicAuth(
+                self.basic_auth["username"], self.basic_auth["password"]
+            )
+        else:
+            auth = None
+
+        return aiohttp.ClientSession(
+            headers=self.headers,
+            auth=auth,
+            timeout=aiohttp.ClientTimeout(total=DEFAULT_REQUEST_TIMEOUT),
+        )
+
+    def combine_parameters(
+        self, kwargs: Optional[Dict[Text, Any]] = None
+    ) -> Dict[Text, Any]:
+        # construct GET parameters
+        params = self.params.copy()
+
+        # set the authentication token if present
+        if self.token:
+            params[self.token_name] = self.token
+
+        if kwargs and "params" in kwargs:
+            params.update(kwargs["params"])
+            del kwargs["params"]
+        return params
+
+    async def request(
+        self,
+        method: Text = "post",
+        subpath: Optional[Text] = None,
+        content_type: Optional[Text] = "application/json",
+        compress: bool = False,
+        **kwargs: Any,
+    ) -> Optional[Any]:
+        """Send a HTTP request to the endpoint. Return json response, if available.
+
+        All additional arguments will get passed through
+        to aiohttp's `session.request`.
+        """
+        # create the appropriate headers
+        headers = {}
+        if content_type:
+            headers["Content-Type"] = content_type
+
+        if "headers" in kwargs:
+            headers.update(kwargs["headers"])
+            del kwargs["headers"]
+
+        if self.headers:
+            headers.update(self.headers)
+
+        url = concat_url(self.url, subpath)
+
+        sslcontext = None
+        if self.cafile:
+            try:
+                sslcontext = ssl.create_default_context(cafile=self.cafile)
+            except FileNotFoundError as e:
+                raise FileNotFoundException(
+                    f"Failed to find certificate file, "
+                    f"'{os.path.abspath(self.cafile)}' does not exist."
+                ) from e
+
+        async with self.session() as session:
+            async with session.request(
+                method,
+                url,
+                headers=headers,
+                params=self.combine_parameters(kwargs),
+                compress=compress,
+                ssl=sslcontext,
+                **kwargs,
+            ) as response:
+                if response.status >= 400:
+                    raise ClientResponseError(
+                        response.status,
+                        response.reason,
+                        await response.content.read(),
+                    )
+                try:
+                    return await response.json()
+                except ContentTypeError:
+                    return None
+
+    @classmethod
+    def from_dict(cls, data: Dict[Text, Any]) -> "EndpointConfig":
+        return EndpointConfig(**data)
+
+    def copy(self) -> "EndpointConfig":
+        return EndpointConfig(
+            self.url,
+            self.params,
+            self.headers,
+            self.basic_auth,
+            self.token,
+            self.token_name,
+            **self.kwargs,
+        )
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(self, type(other)):
+            return (
+                other.url == self.url
+                and other.params == self.params
+                and other.headers == self.headers
+                and other.basic_auth == self.basic_auth
+                and other.token == self.token
+                and other.token_name == self.token_name
+            )
+        else:
+            return False
+
+    def __ne__(self, other: Any) -> bool:
+        return not self.__eq__(other)
+
+
+class ClientResponseError(aiohttp.ClientError):
+    def __init__(self, status: int, message: Text, text: Text) -> None:
+        self.status = status
+        self.message = message
+        self.text = text
+        super().__init__(f"{status}, {message}, body='{text}'")
+
+
+def bool_arg(request: Request, name: Text, default: bool = True) -> bool:
+    """Returns a passed boolean argument of the request or a default.
+
+    Checks the `name` parameter of the request if it contains a valid
+    boolean value. If not, `default` is returned.
+
+    Args:
+        request: Sanic request.
+        name: Name of argument.
+        default: Default value for `name` argument.
+
+    Returns:
+        A bool value if `name` is a valid boolean, `default` otherwise.
+    """
+    return str(request.args.get(name, default)).lower() == "true"
+
+
+def float_arg(
+    request: Request, key: Text, default: Optional[float] = None
+) -> Optional[float]:
+    """Returns a passed argument cast as a float or None.
+
+    Checks the `key` parameter of the request if it contains a valid
+    float value. If not, `default` is returned.
+
+    Args:
+        request: Sanic request.
+        key: Name of argument.
+        default: Default value for `key` argument.
+
+    Returns:
+        A float value if `key` is a valid float, `default` otherwise.
+    """
+    arg = request.args.get(key, default)
+
+    if arg is default:
+        return arg
+
+    try:
+        return float(str(arg))
+    except (ValueError, TypeError):
+        structlogger.warning("endpoint.float_arg.convert_failed", arg=arg, key=key)
+        return default
+
+
+def int_arg(
+    request: Request, key: Text, default: Optional[int] = None
+) -> Optional[int]:
+    """Returns a passed argument cast as an int or None.
+
+    Checks the `key` parameter of the request if it contains a valid
+    int value. If not, `default` is returned.
+
+    Args:
+        request: Sanic request.
+        key: Name of argument.
+        default: Default value for `key` argument.
+
+    Returns:
+        An int value if `key` is a valid integer, `default` otherwise.
+    """
+    arg = request.args.get(key, default)
+
+    if arg is default:
+        return arg
+
+    try:
+        return int(str(arg))
+    except (ValueError, TypeError):
+        structlogger.warning("endpoint.int_arg.convert_failed", arg=arg, key=key)
+        return default

rasa/utils/io.py

@@ -0,0 +1,252 @@
+import asyncio
+import filecmp
+import logging
+import os
+import tempfile
+import warnings
+import re
+from asyncio import AbstractEventLoop
+from collections import OrderedDict
+from io import StringIO
+from pathlib import Path
+from typing import (
+    Optional,
+    Text,
+    Any,
+    Union,
+    List,
+    Type,
+    Callable,
+    TYPE_CHECKING,
+)
+
+from ruamel import yaml
+from ruamel.yaml import RoundTripRepresenter
+from typing_extensions import Protocol
+
+import rasa.shared.constants
+import rasa.shared.utils.io
+
+if TYPE_CHECKING:
+    from prompt_toolkit.validation import Validator
+
+YAML_LINE_MAX_WIDTH = 4096
+
+
+class WriteRow(Protocol):
+    """Describes a csv writer supporting a `writerow` method (workaround for typing)."""
+
+    def writerow(self, row: List[Text]) -> None:
+        """Write the given row.
+
+        Args:
+            row: the entries of a row as a list of strings
+        """
+        ...
+
+
+def configure_colored_logging(loglevel: Text) -> None:
+    """Configures coloredlogs library for specified loglevel.
+
+    Args:
+        loglevel: The loglevel to configure the library for
+    """
+    import coloredlogs
+
+    loglevel = loglevel or os.environ.get(
+        rasa.shared.constants.ENV_LOG_LEVEL, rasa.shared.constants.DEFAULT_LOG_LEVEL
+    )
+
+    field_styles = coloredlogs.DEFAULT_FIELD_STYLES.copy()
+    field_styles["asctime"] = {}
+    level_styles = coloredlogs.DEFAULT_LEVEL_STYLES.copy()
+    level_styles["debug"] = {}
+    coloredlogs.install(
+        level=loglevel,
+        use_chroot=False,
+        fmt="%(asctime)s %(levelname)-8s %(name)s  - %(message)s",
+        level_styles=level_styles,
+        field_styles=field_styles,
+    )
+
+
+def enable_async_loop_debugging(
+    event_loop: AbstractEventLoop, slow_callback_duration: float = 0.1
+) -> AbstractEventLoop:
+    """Enables debugging on an event loop.
+
+    Args:
+        event_loop: The event loop to enable debugging on
+        slow_callback_duration: The threshold at which a callback should be
+                                alerted as slow.
+    """
+    logging.info(
+        "Enabling coroutine debugging. Loop id {}.".format(id(asyncio.get_event_loop()))
+    )
+
+    # Enable debugging
+    event_loop.set_debug(True)
+
+    # Make the threshold for "slow" tasks very very small for
+    # illustration. The default is 0.1 (= 100 milliseconds).
+    event_loop.slow_callback_duration = slow_callback_duration
+
+    # Report all mistakes managing asynchronous resources.
+    warnings.simplefilter("always", ResourceWarning)
+    return event_loop
+
+
+def create_temporary_file(data: Any, suffix: Text = "", mode: Text = "w+") -> Text:
+    """Creates a tempfile.NamedTemporaryFile object for data."""
+    encoding = None if "b" in mode else rasa.shared.utils.io.DEFAULT_ENCODING
+    f = tempfile.NamedTemporaryFile(
+        mode=mode, suffix=suffix, delete=False, encoding=encoding
+    )
+    f.write(data)
+
+    f.close()
+    return f.name
+
+
+def create_temporary_directory() -> Text:
+    """Creates a tempfile.TemporaryDirectory."""
+    f = tempfile.TemporaryDirectory()
+    return f.name
+
+
+def create_path(file_path: Text) -> None:
+    """Makes sure all directories in the 'file_path' exists."""
+    parent_dir = os.path.dirname(os.path.abspath(file_path))
+    if not os.path.exists(parent_dir):
+        os.makedirs(parent_dir)
+
+
+def file_type_validator(
+    valid_file_types: List[Text], error_message: Text
+) -> Type["Validator"]:
+    """Creates a `Validator` class which can be used with `questionary` to validate
+    file paths.
+    """
+
+    def is_valid(path: Text) -> bool:
+        return path is not None and any(
+            [path.endswith(file_type) for file_type in valid_file_types]
+        )
+
+    return create_validator(is_valid, error_message)
+
+
+def not_empty_validator(error_message: Text) -> Type["Validator"]:
+    """Creates a `Validator` class which can be used with `questionary` to validate
+    that the user entered something other than whitespace.
+    """
+
+    def is_valid(input: Text) -> bool:
+        return input is not None and input.strip() != ""
+
+    return create_validator(is_valid, error_message)
+
+
+def create_validator(
+    function: Callable[[Text], bool], error_message: Text
+) -> Type["Validator"]:
+    """Helper method to create `Validator` classes from callable functions. Should be
+    removed when questionary supports `Validator` objects.
+    """
+    from prompt_toolkit.validation import Validator, ValidationError
+    from prompt_toolkit.document import Document
+
+    class FunctionValidator(Validator):
+        @staticmethod
+        def validate(document: Document) -> None:
+            is_valid = function(document.text)
+            if not is_valid:
+                raise ValidationError(message=error_message)
+
+    return FunctionValidator
+
+
+def remove_emojis(s: str) -> str:
+    import demoji
+
+    replaced = demoji.replace(s)
+    if replaced == s:
+        return s
+    # remove duplicate or trailing whitespaces if emojis were removed
+    return re.sub(r" +", " ", replaced).strip()
+
+
+def are_directories_equal(dir1: Path, dir2: Path) -> bool:
+    """Compares two directories recursively.
+
+    Files in each directory are
+    assumed to be equal if their names and contents are equal.
+
+    Args:
+        dir1: The first directory.
+        dir2: The second directory.
+
+    Returns:
+        `True` if they are equal, `False` otherwise.
+    """
+    dirs_cmp = filecmp.dircmp(dir1, dir2)
+    if dirs_cmp.left_only or dirs_cmp.right_only:
+        return False
+
+    (_, mismatches, errors) = filecmp.cmpfiles(
+        dir1, dir2, dirs_cmp.common_files, shallow=False
+    )
+
+    if mismatches or errors:
+        return False
+
+    for common_dir in dirs_cmp.common_dirs:
+        new_dir1 = Path(dir1, common_dir)
+        new_dir2 = Path(dir2, common_dir)
+
+        is_equal = are_directories_equal(new_dir1, new_dir2)
+        if not is_equal:
+            return False
+
+    return True
+
+
+def write_yaml(
+    data: Any,
+    target: Union[Text, Path, StringIO],
+    transform: Optional[Callable] = None,
+    should_preserve_key_order: bool = False,
+) -> None:
+    """Writes a yaml to the file or to the stream.
+
+    Args:
+        data: The data to write.
+        target: The path to the file which should be written or a stream object
+        transform: A function to transform the data before writing it.
+        should_preserve_key_order: Whether to force preserve key order in `data`.
+    """
+    yaml.add_representer(
+        OrderedDict,
+        RoundTripRepresenter.represent_dict,
+        representer=RoundTripRepresenter,
+    )
+
+    if should_preserve_key_order:
+        data = rasa.shared.utils.io.convert_to_ordered_dict(data)
+
+    dumper = yaml.YAML()
+    # no wrap lines
+    dumper.width = YAML_LINE_MAX_WIDTH  # type: ignore
+
+    # use `null` to represent `None`
+    dumper.representer.add_representer(
+        type(None),
+        lambda self, _: self.represent_scalar("tag:yaml.org,2002:null", "null"),
+    )
+
+    if isinstance(target, StringIO):
+        dumper.dump(data, target, transform=transform)
+        return
+
+    with Path(target).open("w", encoding="utf-8") as outfile:
+        dumper.dump(data, outfile, transform=transform)

rasa/utils/json_utils.py

@@ -0,0 +1,60 @@
+import json
+from decimal import Decimal
+from typing import Any, Text
+
+
+class DecimalEncoder(json.JSONEncoder):
+    """`json.JSONEncoder` that dumps `Decimal`s as `float`s."""
+
+    def default(self, obj: Any) -> Any:
+        """Get serializable object for `o`.
+
+        Args:
+            obj: Object to serialize.
+
+        Returns:
+            `obj` converted to `float` if `o` is a `Decimals`, else the base class
+            `default()` method.
+        """
+        if isinstance(obj, Decimal):
+            return float(obj)
+        return super().default(obj)
+
+
+class SetEncoder(json.JSONEncoder):
+    """`json.JSONEncoder` that dumps `set`s as `list`s."""
+
+    def default(self, obj: Any) -> Any:
+        if isinstance(obj, set):
+            return list(obj)
+        return super().default(obj)
+
+
+def replace_floats_with_decimals(obj: Any, round_digits: int = 9) -> Any:
+    """Convert all instances in `obj` of `float` to `Decimal`.
+
+    Args:
+        obj: Input object.
+        round_digits: Rounding precision of `Decimal` values.
+
+    Returns:
+        Input `obj` with all `float` types replaced by `Decimal`s rounded to
+        `round_digits` decimal places.
+    """
+
+    def _float_to_rounded_decimal(s: Text) -> Decimal:
+        return Decimal(s).quantize(Decimal(10) ** -round_digits)
+
+    return json.loads(json.dumps(obj), parse_float=_float_to_rounded_decimal)
+
+
+def replace_decimals_with_floats(obj: Any) -> Any:
+    """Convert all instances in `obj` of `Decimal` to `float`.
+
+    Args:
+        obj: A `List` or `Dict` object.
+
+    Returns:
+        Input `obj` with all `Decimal` types replaced by `float`s.
+    """
+    return json.loads(json.dumps(obj, cls=DecimalEncoder))