rasa-pro 3.11.3a1.dev2__py3-none-any.whl

rasa/shared/nlu/training_data/util.py

@@ -0,0 +1,223 @@
+import json
+import logging
+import os
+import re
+from typing import Any, Dict, Optional, Text, Match, List
+
+import scipy.sparse
+from rasa.shared.nlu.constants import (
+    ENTITIES,
+    EXTRACTOR,
+    PRETRAINED_EXTRACTORS,
+    ENTITY_ATTRIBUTE_START,
+    ENTITY_ATTRIBUTE_END,
+    ENTITY_ATTRIBUTE_VALUE,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_ROLE,
+    ENTITY_ATTRIBUTE_GROUP,
+)
+from rasa.shared.constants import UTTER_PREFIX
+import rasa.shared.utils.io
+import rasa.shared.data
+
+logger = logging.getLogger(__name__)
+
+ESCAPE_DCT = {"\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r", "\t": "\\t"}
+ESCAPE_CHARS = set(ESCAPE_DCT.keys())
+ESCAPE = re.compile(f'[{"".join(ESCAPE_DCT.values())}]')
+UNESCAPE_DCT = {espaced_char: char for char, espaced_char in ESCAPE_DCT.items()}
+UNESCAPE = re.compile(f'[{"".join(UNESCAPE_DCT.values())}]')
+GROUP_COMPLETE_MATCH = 0
+
+
+def transform_entity_synonyms(
+    synonyms: List[Dict[Text, Any]], known_synonyms: Optional[Dict[Text, Any]] = None
+) -> Dict[Text, Any]:
+    """Transforms the entity synonyms into a text->value dictionary."""
+    entity_synonyms = known_synonyms if known_synonyms else {}
+    for s in synonyms:
+        if "value" in s and "synonyms" in s:
+            for synonym in s["synonyms"]:
+                entity_synonyms[synonym] = s["value"]
+    return entity_synonyms
+
+
+def check_duplicate_synonym(
+    entity_synonyms: Dict[Text, Any], text: Text, syn: Text, context_str: Text = ""
+) -> None:
+    if text in entity_synonyms and entity_synonyms[text] != syn:
+        rasa.shared.utils.io.raise_warning(
+            f"Found inconsistent entity synonyms while {context_str}, "
+            f"overwriting {text}->{entity_synonyms[text]} "
+            f"with {text}->{syn} during merge."
+        )
+
+
+def get_file_format_extension(resource_name: Text) -> Text:
+    """Get the file extension based on training data format. It supports both a folder and
+    a file, and tries to guess the format as follows:
+
+    - if the resource is a file and has a known format, return this format's extension
+    - if the resource is a folder and all the resources have the
+      same known format, return it's extension
+    - otherwise, default to DEFAULT_FILE_FORMAT (yml).
+
+    Args:
+        resource_name: The name of the resource, can be a file or a folder.
+
+    Returns:
+        The resource file format.
+    """  # noqa: E501
+    from rasa.shared.nlu.training_data import loading
+
+    if resource_name is None or not os.path.exists(resource_name):
+        raise AttributeError(f"Resource '{resource_name}' does not exist.")
+
+    files = rasa.shared.utils.io.list_files(resource_name)
+
+    file_formats = list(map(lambda f: loading.guess_format(f), files))
+
+    if not file_formats:
+        return rasa.shared.data.yaml_file_extension()
+
+    known_file_formats = {loading.RASA_YAML: rasa.shared.data.yaml_file_extension()}
+    fformat = file_formats[0]
+    if all(f == fformat for f in file_formats):
+        return known_file_formats.get(fformat, rasa.shared.data.yaml_file_extension())
+
+    return rasa.shared.data.yaml_file_extension()
+
+
+def remove_untrainable_entities_from(example: Dict[Text, Any]) -> None:
+    """Remove untrainable entities from serialised training example `example`.
+
+    Entities with an untrainable extractor will be removed. Untrainable extractors
+    are defined in `rasa.nlu.constants.PRETRAINED_EXTRACTORS`.
+
+    Args:
+        example: Serialised training example to inspect.
+    """
+    example_entities = example.get(ENTITIES)
+
+    if not example_entities:
+        # example contains no entities, so there's nothing to do
+        return None
+
+    trainable_entities = []
+
+    for entity in example_entities:
+        if entity.get(EXTRACTOR) in PRETRAINED_EXTRACTORS:
+            logger.debug(
+                f"Excluding entity '{json.dumps(entity)}' from training data. "
+                f"Entity examples extracted by the following classes are not "
+                f"dumped to training data in markdown format: "
+                f"`{'`, `'.join(sorted(PRETRAINED_EXTRACTORS))}`."
+            )
+        else:
+            trainable_entities.append(entity)
+
+    example[ENTITIES] = trainable_entities
+
+
+def intent_response_key_to_template_key(intent_response_key: Text) -> Text:
+    """Resolve the response template key for a given intent response key.
+
+    Args:
+        intent_response_key: retrieval intent with the response key suffix attached.
+
+    Returns: The corresponding response template.
+
+    """
+    return f"{UTTER_PREFIX}{intent_response_key}"
+
+
+def template_key_to_intent_response_key(template_key: Text) -> Text:
+    """Resolve the intent response key for the given response template.
+
+    Args:
+        template_key: Name of the response template.
+
+    Returns: The corresponding intent response key.
+
+    """
+    return template_key.split(UTTER_PREFIX)[1]
+
+
+def has_string_escape_chars(s: Text) -> bool:
+    """Checks whether there are any of the escape characters in the string."""
+    intersection = ESCAPE_CHARS.intersection(set(s))
+    return len(intersection) > 0
+
+
+def encode_string(s: Text) -> Text:
+    """Return an encoded python string."""
+
+    def replace(match: Match) -> Text:
+        return ESCAPE_DCT[match.group(GROUP_COMPLETE_MATCH)]
+
+    return ESCAPE.sub(replace, s)
+
+
+def decode_string(s: Text) -> Text:
+    """Return a decoded python string."""
+
+    def replace(match: Match) -> Text:
+        return UNESCAPE_DCT[match.group(GROUP_COMPLETE_MATCH)]
+
+    return UNESCAPE.sub(replace, s)
+
+
+def build_entity(
+    start: int,
+    end: int,
+    value: Text,
+    entity_type: Text,
+    role: Optional[Text] = None,
+    group: Optional[Text] = None,
+    **kwargs: Any,
+) -> Dict[Text, Any]:
+    """Builds a standard entity dictionary.
+
+    Adds additional keyword parameters.
+
+    Args:
+        start: start position of entity
+        end: end position of entity
+        value: text value of the entity
+        entity_type: name of the entity type
+        role: role of the entity
+        group: group of the entity
+        **kwargs: additional parameters
+
+    Returns:
+        an entity dictionary
+    """
+    entity = {
+        ENTITY_ATTRIBUTE_START: start,
+        ENTITY_ATTRIBUTE_END: end,
+        ENTITY_ATTRIBUTE_VALUE: value,
+        ENTITY_ATTRIBUTE_TYPE: entity_type,
+    }
+
+    if role:
+        entity[ENTITY_ATTRIBUTE_ROLE] = role
+    if group:
+        entity[ENTITY_ATTRIBUTE_GROUP] = group
+
+    entity.update(kwargs)
+    return entity
+
+
+def sparse_matrix_to_string(m: scipy.sparse.spmatrix) -> Text:
+    """Turns a sparse matrix into a string.
+
+    Will return a line "(i,j)  v" for each value in the matrix.
+
+    taken from official scipy source to operate on full sparse matrix to not have
+    to change the `maxprint` property in-place.
+    https://github.com/scipy/scipy/blob/v1.7.0/scipy/sparse/base.py#L258
+    """
+    # make sure sparse matrix is in COOrdinate format
+    m_coo = m.tocoo()
+    triples = zip(list(zip(m_coo.row, m_coo.col)), m_coo.data)
+    return "\n".join([("  %s\t%s" % t) for t in triples])

rasa/shared/providers/_configs/azure_openai_client_config.py

@@ -0,0 +1,183 @@
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Optional
+
+import structlog
+
+from rasa.shared.constants import (
+    MODEL_CONFIG_KEY,
+    MODEL_NAME_CONFIG_KEY,
+    OPENAI_API_BASE_CONFIG_KEY,
+    API_BASE_CONFIG_KEY,
+    OPENAI_API_TYPE_CONFIG_KEY,
+    API_TYPE_CONFIG_KEY,
+    OPENAI_API_VERSION_CONFIG_KEY,
+    API_VERSION_CONFIG_KEY,
+    DEPLOYMENT_CONFIG_KEY,
+    DEPLOYMENT_NAME_CONFIG_KEY,
+    ENGINE_CONFIG_KEY,
+    RASA_TYPE_CONFIG_KEY,
+    LANGCHAIN_TYPE_CONFIG_KEY,
+    STREAM_CONFIG_KEY,
+    N_REPHRASES_CONFIG_KEY,
+    REQUEST_TIMEOUT_CONFIG_KEY,
+    TIMEOUT_CONFIG_KEY,
+    PROVIDER_CONFIG_KEY,
+    AZURE_OPENAI_PROVIDER,
+    AZURE_API_TYPE,
+)
+from rasa.shared.providers._configs.utils import (
+    resolve_aliases,
+    raise_deprecation_warnings,
+    validate_required_keys,
+    validate_forbidden_keys,
+)
+
+structlogger = structlog.get_logger()
+
+DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING = {
+    # Deployment name aliases
+    DEPLOYMENT_NAME_CONFIG_KEY: DEPLOYMENT_CONFIG_KEY,
+    ENGINE_CONFIG_KEY: DEPLOYMENT_CONFIG_KEY,
+    # Provider aliases
+    RASA_TYPE_CONFIG_KEY: PROVIDER_CONFIG_KEY,
+    LANGCHAIN_TYPE_CONFIG_KEY: PROVIDER_CONFIG_KEY,
+    # API type aliases
+    OPENAI_API_TYPE_CONFIG_KEY: API_TYPE_CONFIG_KEY,
+    # API base aliases
+    OPENAI_API_BASE_CONFIG_KEY: API_BASE_CONFIG_KEY,
+    # API version aliases
+    OPENAI_API_VERSION_CONFIG_KEY: API_VERSION_CONFIG_KEY,
+    # Model name aliases
+    MODEL_NAME_CONFIG_KEY: MODEL_CONFIG_KEY,
+    # Timeout aliases
+    REQUEST_TIMEOUT_CONFIG_KEY: TIMEOUT_CONFIG_KEY,
+}
+
+REQUIRED_KEYS = [DEPLOYMENT_CONFIG_KEY]
+
+FORBIDDEN_KEYS = [
+    STREAM_CONFIG_KEY,
+    N_REPHRASES_CONFIG_KEY,
+]
+
+
+@dataclass
+class AzureOpenAIClientConfig:
+    """Parses configuration for Azure OpenAI client, resolves aliases and
+    raises deprecation warnings.
+
+    Raises:
+        ValueError: Raised in cases of invalid configuration:
+            - If any of the required configuration keys are missing.
+            - If `api_type` has a value different from `azure`.
+    """
+
+    deployment: str
+
+    model: Optional[str]
+    api_base: Optional[str]
+    api_version: Optional[str]
+    # API Type is not used by LiteLLM backend, but we define
+    # it here for backward compatibility.
+    api_type: Optional[str] = AZURE_API_TYPE
+
+    # Provider is not used by LiteLLM backend, but we define it here since it's
+    # used as switch between different clients.
+    provider: str = AZURE_OPENAI_PROVIDER
+
+    extra_parameters: dict = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.provider != AZURE_OPENAI_PROVIDER:
+            message = f"Provider must be set to '{AZURE_OPENAI_PROVIDER}'."
+            structlogger.error(
+                "azure_openai_client_config.validation_error",
+                message=message,
+                provider=self.provider,
+            )
+            raise ValueError(message)
+        if self.deployment is None:
+            message = "Deployment cannot be set to None."
+            structlogger.error(
+                "azure_openai_client_config.validation_error",
+                message=message,
+                deployment=self.deployment,
+            )
+            raise ValueError(message)
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "AzureOpenAIClientConfig":
+        """Initializes a dataclass from the passed config.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Raises:
+            ValueError: Raised in cases of invalid configuration:
+                - If any of the required configuration keys are missing.
+                - If `api_type` has a value different from `azure`.
+
+        Returns:
+            AzureOpenAIClientConfig
+        """
+        # Check for deprecated keys
+        raise_deprecation_warnings(config, DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING)
+        # Resolve any potential aliases
+        config = cls.resolve_config_aliases(config)
+        # Validate that required keys are set
+        validate_required_keys(config, REQUIRED_KEYS)
+        # Validate that the forbidden keys are not present
+        validate_forbidden_keys(config, FORBIDDEN_KEYS)
+        # Init client config
+        this = AzureOpenAIClientConfig(
+            # Required parameters
+            deployment=config.pop(DEPLOYMENT_CONFIG_KEY),
+            # Pop the 'provider' key. Currently, it's *optional* because of
+            # backward compatibility with older versions.
+            provider=config.pop(PROVIDER_CONFIG_KEY, AZURE_OPENAI_PROVIDER),
+            # Optional
+            api_type=config.pop(API_TYPE_CONFIG_KEY, AZURE_API_TYPE),
+            model=config.pop(MODEL_CONFIG_KEY, None),
+            # Optional, can also be set through environment variables
+            # in clients.
+            api_base=config.pop(API_BASE_CONFIG_KEY, None),
+            api_version=config.pop(API_VERSION_CONFIG_KEY, None),
+            # The rest of parameters (e.g. model parameters) are considered
+            # as extra parameters (this also includes timeout).
+            extra_parameters=config,
+        )
+        return this
+
+    def to_dict(self) -> dict:
+        """Converts the config instance into a dictionary."""
+        d = asdict(self)
+        # Extra parameters should also be on the top level
+        d.pop("extra_parameters", None)
+        d.update(self.extra_parameters)
+        return d
+
+    @staticmethod
+    def resolve_config_aliases(config: Dict[str, Any]) -> Dict[str, Any]:
+        return resolve_aliases(config, DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING)
+
+
+def is_azure_openai_config(config: dict) -> bool:
+    """Check whether the configuration is meant to configure
+    an Azure OpenAI client.
+    """
+    # Resolve any aliases that are specific to Azure OpenAI configuration
+    config = AzureOpenAIClientConfig.resolve_config_aliases(config)
+
+    # Case: Configuration contains `provider: azure`.
+    if config.get(PROVIDER_CONFIG_KEY) == AZURE_OPENAI_PROVIDER:
+        return True
+
+    # Case: Configuration contains `deployment` key
+    # (specific to Azure OpenAI configuration)
+    if (
+        config.get(DEPLOYMENT_CONFIG_KEY) is not None
+        and config.get(PROVIDER_CONFIG_KEY) is None
+    ):
+        return True
+
+    return False

rasa/shared/providers/_configs/client_config.py

@@ -0,0 +1,57 @@
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class ClientConfig(Protocol):
+    """
+    Protocol for the client config that specifies the interface for interacting
+    with the API.
+    """
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "ClientConfig":
+        """
+        Initializes the client config with the given configuration.
+
+        This class method should be implemented to parse the given
+        configuration and create an instance of an client config.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Raises:
+            ValueError: Config is missing required keys.
+
+        Returns:
+            ClientConfig
+        """
+        ...
+
+    def to_dict(self) -> dict:
+        """
+        Returns the configuration for that the client config is initialized with.
+
+        This method should be implemented to return a dictionary containing
+        the configuration settings for the client config.
+
+        Returns:
+            dictionary containing the configuration settings for the client config.
+        """
+        ...
+
+    @staticmethod
+    def resolve_config_aliases(config: dict) -> dict:
+        """
+        Resolve any potential aliases in the configuration.
+
+        This method should be implemented to resolve any potential aliases in the
+        configuration.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Returns:
+            dictionary containing the resolved configuration settings for the
+            client config.
+        """
+        ...

rasa/shared/providers/_configs/default_litellm_client_config.py

@@ -0,0 +1,130 @@
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict
+
+import structlog
+
+from rasa.shared.constants import (
+    MODEL_CONFIG_KEY,
+    MODEL_NAME_CONFIG_KEY,
+    STREAM_CONFIG_KEY,
+    N_REPHRASES_CONFIG_KEY,
+    PROVIDER_CONFIG_KEY,
+    TIMEOUT_CONFIG_KEY,
+    REQUEST_TIMEOUT_CONFIG_KEY,
+)
+from rasa.shared.providers._configs.utils import (
+    validate_required_keys,
+    validate_forbidden_keys,
+    resolve_aliases,
+    raise_deprecation_warnings,
+)
+import rasa.shared.utils.cli
+
+structlogger = structlog.get_logger()
+
+
+DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING = {
+    # Timeout aliases
+    REQUEST_TIMEOUT_CONFIG_KEY: TIMEOUT_CONFIG_KEY,
+}
+
+REQUIRED_KEYS = [MODEL_CONFIG_KEY, PROVIDER_CONFIG_KEY]
+
+FORBIDDEN_KEYS = [
+    STREAM_CONFIG_KEY,
+    N_REPHRASES_CONFIG_KEY,
+]
+
+
+@dataclass
+class DefaultLiteLLMClientConfig:
+    """Parses configuration for default LiteLLM client, resolves aliases and
+    raises deprecation warnings.
+
+    Raises:
+        ValueError: Raised in cases of invalid configuration:
+            - If any of the required configuration keys are missing.
+    """
+
+    model: str
+    provider: str
+    extra_parameters: dict = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.model is None:
+            message = "Model cannot be set to None."
+            structlogger.error(
+                "default_litellm_client_config.validation_error",
+                message=message,
+                model=self.model,
+            )
+            raise ValueError(message)
+        if self.provider is None:
+            message = "Provider cannot be set to None."
+            structlogger.error(
+                "default_litellm_client_config.validation_error",
+                message=message,
+                provider=self.provider,
+            )
+            raise ValueError(message)
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "DefaultLiteLLMClientConfig":
+        """
+        Initializes a dataclass from the passed config.
+
+        Args:
+            config: (dict) The config from which to initialize.
+
+        Raises:
+            ValueError: Config is missing required keys.
+
+        Returns:
+            DefaultLiteLLMClientConfig
+        """
+        # Check for deprecated keys
+        raise_deprecation_warnings(config, DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING)
+        # Raise error for using `model_name` instead instead of `model`
+        cls.check_and_error_for_model_name_in_config(config)
+        # Resolve any potential aliases.
+        config = cls.resolve_config_aliases(config)
+        # Validate that the required keys are present
+        validate_required_keys(config, REQUIRED_KEYS)
+        # Validate that the forbidden keys are not present
+        validate_forbidden_keys(config, FORBIDDEN_KEYS)
+        this = DefaultLiteLLMClientConfig(
+            # Required parameters
+            model=config.pop(MODEL_CONFIG_KEY),
+            provider=config.pop(PROVIDER_CONFIG_KEY),
+            # The rest of parameters (e.g. model parameters) are considered
+            # as extra parameters
+            extra_parameters=config,
+        )
+        return this
+
+    def to_dict(self) -> dict:
+        """Converts the config instance into a dictionary."""
+        d = asdict(self)
+        # Extra parameters should also be on the top level
+        d.pop("extra_parameters", None)
+        d.update(self.extra_parameters)
+        return d
+
+    @staticmethod
+    def resolve_config_aliases(config: Dict[str, Any]) -> Dict[str, Any]:
+        return resolve_aliases(config, DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING)
+
+    @staticmethod
+    def check_and_error_for_model_name_in_config(config: Dict[str, Any]) -> None:
+        """Check for usage of deprecated model_name and raise an error if found."""
+        if config.get(MODEL_NAME_CONFIG_KEY) and not config.get(MODEL_CONFIG_KEY):
+            event_info = (
+                f"Unsupported parameter - {MODEL_NAME_CONFIG_KEY} is set. Please use "
+                f"{MODEL_CONFIG_KEY} instead."
+            )
+            structlogger.error(
+                "default_litellm_client_config.unsupported_parameter_in_config",
+                event_info=event_info,
+                config=config,
+            )
+            rasa.shared.utils.cli.print_error_and_exit(event_info)