rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/shared/nlu/training_data/util.py

@@ -0,0 +1,223 @@
+import json
+import logging
+import os
+import re
+from typing import Any, Dict, Optional, Text, Match, List
+
+import scipy.sparse
+from rasa.shared.nlu.constants import (
+    ENTITIES,
+    EXTRACTOR,
+    PRETRAINED_EXTRACTORS,
+    ENTITY_ATTRIBUTE_START,
+    ENTITY_ATTRIBUTE_END,
+    ENTITY_ATTRIBUTE_VALUE,
+    ENTITY_ATTRIBUTE_TYPE,
+    ENTITY_ATTRIBUTE_ROLE,
+    ENTITY_ATTRIBUTE_GROUP,
+)
+from rasa.shared.constants import UTTER_PREFIX
+import rasa.shared.utils.io
+import rasa.shared.data
+
+logger = logging.getLogger(__name__)
+
+ESCAPE_DCT = {"\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r", "\t": "\\t"}
+ESCAPE_CHARS = set(ESCAPE_DCT.keys())
+ESCAPE = re.compile(f'[{"".join(ESCAPE_DCT.values())}]')
+UNESCAPE_DCT = {espaced_char: char for char, espaced_char in ESCAPE_DCT.items()}
+UNESCAPE = re.compile(f'[{"".join(UNESCAPE_DCT.values())}]')
+GROUP_COMPLETE_MATCH = 0
+
+
+def transform_entity_synonyms(
+    synonyms: List[Dict[Text, Any]], known_synonyms: Optional[Dict[Text, Any]] = None
+) -> Dict[Text, Any]:
+    """Transforms the entity synonyms into a text->value dictionary."""
+    entity_synonyms = known_synonyms if known_synonyms else {}
+    for s in synonyms:
+        if "value" in s and "synonyms" in s:
+            for synonym in s["synonyms"]:
+                entity_synonyms[synonym] = s["value"]
+    return entity_synonyms
+
+
+def check_duplicate_synonym(
+    entity_synonyms: Dict[Text, Any], text: Text, syn: Text, context_str: Text = ""
+) -> None:
+    if text in entity_synonyms and entity_synonyms[text] != syn:
+        rasa.shared.utils.io.raise_warning(
+            f"Found inconsistent entity synonyms while {context_str}, "
+            f"overwriting {text}->{entity_synonyms[text]} "
+            f"with {text}->{syn} during merge."
+        )
+
+
+def get_file_format_extension(resource_name: Text) -> Text:
+    """Get the file extension based on training data format. It supports both a folder and
+    a file, and tries to guess the format as follows:
+
+    - if the resource is a file and has a known format, return this format's extension
+    - if the resource is a folder and all the resources have the
+      same known format, return it's extension
+    - otherwise, default to DEFAULT_FILE_FORMAT (yml).
+
+    Args:
+        resource_name: The name of the resource, can be a file or a folder.
+
+    Returns:
+        The resource file format.
+    """  # noqa: E501
+    from rasa.shared.nlu.training_data import loading
+
+    if resource_name is None or not os.path.exists(resource_name):
+        raise AttributeError(f"Resource '{resource_name}' does not exist.")
+
+    files = rasa.shared.utils.io.list_files(resource_name)
+
+    file_formats = list(map(lambda f: loading.guess_format(f), files))
+
+    if not file_formats:
+        return rasa.shared.data.yaml_file_extension()
+
+    known_file_formats = {loading.RASA_YAML: rasa.shared.data.yaml_file_extension()}
+    fformat = file_formats[0]
+    if all(f == fformat for f in file_formats):
+        return known_file_formats.get(fformat, rasa.shared.data.yaml_file_extension())
+
+    return rasa.shared.data.yaml_file_extension()
+
+
+def remove_untrainable_entities_from(example: Dict[Text, Any]) -> None:
+    """Remove untrainable entities from serialised training example `example`.
+
+    Entities with an untrainable extractor will be removed. Untrainable extractors
+    are defined in `rasa.nlu.constants.PRETRAINED_EXTRACTORS`.
+
+    Args:
+        example: Serialised training example to inspect.
+    """
+    example_entities = example.get(ENTITIES)
+
+    if not example_entities:
+        # example contains no entities, so there's nothing to do
+        return None
+
+    trainable_entities = []
+
+    for entity in example_entities:
+        if entity.get(EXTRACTOR) in PRETRAINED_EXTRACTORS:
+            logger.debug(
+                f"Excluding entity '{json.dumps(entity)}' from training data. "
+                f"Entity examples extracted by the following classes are not "
+                f"dumped to training data in markdown format: "
+                f"`{'`, `'.join(sorted(PRETRAINED_EXTRACTORS))}`."
+            )
+        else:
+            trainable_entities.append(entity)
+
+    example[ENTITIES] = trainable_entities
+
+
+def intent_response_key_to_template_key(intent_response_key: Text) -> Text:
+    """Resolve the response template key for a given intent response key.
+
+    Args:
+        intent_response_key: retrieval intent with the response key suffix attached.
+
+    Returns: The corresponding response template.
+
+    """
+    return f"{UTTER_PREFIX}{intent_response_key}"
+
+
+def template_key_to_intent_response_key(template_key: Text) -> Text:
+    """Resolve the intent response key for the given response template.
+
+    Args:
+        template_key: Name of the response template.
+
+    Returns: The corresponding intent response key.
+
+    """
+    return template_key.split(UTTER_PREFIX)[1]
+
+
+def has_string_escape_chars(s: Text) -> bool:
+    """Checks whether there are any of the escape characters in the string."""
+    intersection = ESCAPE_CHARS.intersection(set(s))
+    return len(intersection) > 0
+
+
+def encode_string(s: Text) -> Text:
+    """Return an encoded python string."""
+
+    def replace(match: Match) -> Text:
+        return ESCAPE_DCT[match.group(GROUP_COMPLETE_MATCH)]
+
+    return ESCAPE.sub(replace, s)
+
+
+def decode_string(s: Text) -> Text:
+    """Return a decoded python string."""
+
+    def replace(match: Match) -> Text:
+        return UNESCAPE_DCT[match.group(GROUP_COMPLETE_MATCH)]
+
+    return UNESCAPE.sub(replace, s)
+
+
+def build_entity(
+    start: int,
+    end: int,
+    value: Text,
+    entity_type: Text,
+    role: Optional[Text] = None,
+    group: Optional[Text] = None,
+    **kwargs: Any,
+) -> Dict[Text, Any]:
+    """Builds a standard entity dictionary.
+
+    Adds additional keyword parameters.
+
+    Args:
+        start: start position of entity
+        end: end position of entity
+        value: text value of the entity
+        entity_type: name of the entity type
+        role: role of the entity
+        group: group of the entity
+        **kwargs: additional parameters
+
+    Returns:
+        an entity dictionary
+    """
+    entity = {
+        ENTITY_ATTRIBUTE_START: start,
+        ENTITY_ATTRIBUTE_END: end,
+        ENTITY_ATTRIBUTE_VALUE: value,
+        ENTITY_ATTRIBUTE_TYPE: entity_type,
+    }
+
+    if role:
+        entity[ENTITY_ATTRIBUTE_ROLE] = role
+    if group:
+        entity[ENTITY_ATTRIBUTE_GROUP] = group
+
+    entity.update(kwargs)
+    return entity
+
+
+def sparse_matrix_to_string(m: scipy.sparse.spmatrix) -> Text:
+    """Turns a sparse matrix into a string.
+
+    Will return a line "(i,j)  v" for each value in the matrix.
+
+    taken from official scipy source to operate on full sparse matrix to not have
+    to change the `maxprint` property in-place.
+    https://github.com/scipy/scipy/blob/v1.7.0/scipy/sparse/base.py#L258
+    """
+    # make sure sparse matrix is in COOrdinate format
+    m_coo = m.tocoo()
+    triples = zip(list(zip(m_coo.row, m_coo.col)), m_coo.data)
+    return "\n".join([("  %s\t%s" % t) for t in triples])