rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/nlu/utils/hugging_face/transformers_pre_post_processors.py

@@ -0,0 +1,311 @@
+from typing import List, Tuple, Text
+import numpy as np
+
+
+def cleanup_tokens(
+    token_ids_string: List[Tuple[int, Text]], delimiter: Text
+) -> Tuple[List[int], List[Text]]:
+    """Utility method to apply delimiter based cleanup on list of tokens.
+
+    Args:
+        token_ids_string: List of tuples with each tuple containing
+            (token id, token string).
+        delimiter: character/string to be cleaned from token strings.
+
+    Returns:
+        Token ids and Token strings unpacked.
+    """
+    token_ids_string = [
+        (id, string.replace(delimiter, "")) for id, string in token_ids_string
+    ]
+
+    # remove empty strings
+    token_ids_string = [(id, string) for id, string in token_ids_string if string]
+
+    # return as individual token ids and token strings
+    token_ids, token_strings = zip(*token_ids_string)
+    # FIXME: zip official typing is not really properly set up
+    return token_ids, token_strings  # type: ignore[return-value]
+
+
+def bert_tokens_pre_processor(token_ids: List[int]) -> List[int]:
+    """Add BERT style special tokens(CLS and SEP).
+
+    Args:
+        token_ids: List of token ids without any special tokens.
+
+    Returns:
+        List of token ids augmented with special tokens.
+    """
+    BERT_CLS_ID = 101
+    BERT_SEP_ID = 102
+
+    processed_tokens = token_ids
+
+    processed_tokens.insert(0, BERT_CLS_ID)
+    processed_tokens.append(BERT_SEP_ID)
+
+    return processed_tokens
+
+
+def gpt_tokens_pre_processor(token_ids: List[int]) -> List[int]:
+    """Add GPT style special tokens(None).
+
+    Args:
+        token_ids: List of token ids without any special tokens.
+
+    Returns:
+        List of token ids augmented with special tokens.
+    """
+    return token_ids
+
+
+def xlnet_tokens_pre_processor(token_ids: List[int]) -> List[int]:
+    """Add XLNET style special tokens.
+
+    Args:
+        token_ids: List of token ids without any special tokens.
+
+    Returns:
+        List of token ids augmented with special tokens.
+    """
+    XLNET_CLS_ID = 3
+    XLNET_SEP_ID = 4
+
+    token_ids.append(XLNET_SEP_ID)
+    token_ids.append(XLNET_CLS_ID)
+
+    return token_ids
+
+
+def roberta_tokens_pre_processor(token_ids: List[int]) -> List[int]:
+    """Add RoBERTa style special tokens.
+
+    Args:
+        token_ids: List of token ids without any special tokens.
+
+    Returns:
+        List of token ids augmented with special tokens.
+    """
+    ROBERTA_BEG_ID = 0
+    ROBERTA_END_ID = 2
+
+    token_ids.insert(0, ROBERTA_BEG_ID)
+    token_ids.append(ROBERTA_END_ID)
+
+    return token_ids
+
+
+def xlm_tokens_pre_processor(token_ids: List[int]) -> List[int]:
+    """Add XLM style special tokens.
+
+    Args:
+        token_ids: List of token ids without any special tokens.
+
+    Returns:
+        List of token ids augmented with special tokens.
+    """
+    XLM_SEP_ID = 1
+
+    token_ids.insert(0, XLM_SEP_ID)
+    token_ids.append(XLM_SEP_ID)
+
+    return token_ids
+
+
+def camembert_tokens_pre_processor(token_ids: List[int]) -> List[int]:
+    """Add camembert style special tokens.
+
+    Args:
+        token_ids: List of token ids without any special tokens.
+
+    Returns:
+        List of token ids augmented with special tokens.
+    """
+    CAMEMBERT_BEG_ID = 5
+    CAMEMBERT_END_ID = 6
+
+    token_ids.insert(0, CAMEMBERT_BEG_ID)
+    token_ids.append(CAMEMBERT_END_ID)
+
+    return token_ids
+
+
+def bert_embeddings_post_processor(
+    sequence_embeddings: np.ndarray,
+) -> Tuple[np.ndarray, np.ndarray]:
+    """Post-process embeddings from BERT.
+
+    by removing CLS and SEP embeddings and returning CLS token embedding as
+    sentence representation.
+
+    Args:
+        sequence_embeddings: Sequence of token level embeddings received as output from
+            BERT.
+
+    Returns:
+        sentence level embedding and post-processed sequence level embedding.
+    """
+    sentence_embedding = sequence_embeddings[0]
+    post_processed_embedding = sequence_embeddings[1:-1]
+
+    return sentence_embedding, post_processed_embedding
+
+
+def gpt_embeddings_post_processor(
+    sequence_embeddings: np.ndarray,
+) -> Tuple[np.ndarray, np.ndarray]:
+    """Post-process embeddings from GPT models.
+
+    by taking a mean over sequence embeddings and returning that as sentence
+    representation.
+
+    Args:
+        sequence_embeddings: Sequence of token level embeddings received as output from
+            GPT.
+
+    Returns:
+        sentence level embedding and post-processed sequence level embedding.
+    """
+    sentence_embedding = np.mean(sequence_embeddings, axis=0)
+    post_processed_embedding = sequence_embeddings
+
+    return sentence_embedding, post_processed_embedding
+
+
+def xlnet_embeddings_post_processor(
+    sequence_embeddings: np.ndarray,
+) -> Tuple[np.ndarray, np.ndarray]:
+    """Post-process embeddings from XLNet models.
+
+    by taking a mean over sequence embeddings and returning that as sentence
+    representation. Remove last two time steps corresponding
+    to special tokens from the sequence embeddings.
+
+    Args:
+        sequence_embeddings: Sequence of token level embeddings received as output from
+            XLNet.
+
+    Returns:
+        sentence level embedding and post-processed sequence level embedding.
+    """
+    post_processed_embedding = sequence_embeddings[:-2]
+    sentence_embedding = np.mean(post_processed_embedding, axis=0)
+
+    return sentence_embedding, post_processed_embedding
+
+
+def roberta_embeddings_post_processor(
+    sequence_embeddings: np.ndarray,
+) -> Tuple[np.ndarray, np.ndarray]:
+    """Post process embeddings from Roberta models.
+
+    by taking a mean over sequence embeddings and returning that as sentence
+    representation. Remove first and last time steps
+    corresponding to special tokens from the sequence embeddings.
+
+    Args:
+        sequence_embeddings: Sequence of token level embeddings received as output from
+            Roberta
+
+    Returns:
+        sentence level embedding and post-processed sequence level embedding
+    """
+    post_processed_embedding = sequence_embeddings[1:-1]
+    sentence_embedding = np.mean(post_processed_embedding, axis=0)
+
+    return sentence_embedding, post_processed_embedding
+
+
+def xlm_embeddings_post_processor(
+    sequence_embeddings: np.ndarray,
+) -> Tuple[np.ndarray, np.ndarray]:
+    """Post process embeddings from XLM models.
+
+    by taking a mean over sequence embeddings and returning that as sentence
+    representation. Remove first and last time steps
+    corresponding to special tokens from the sequence embeddings.
+
+    Args:
+        sequence_embeddings: Sequence of token level embeddings received as output from
+            XLM
+
+    Returns:
+        sentence level embedding and post-processed sequence level embedding
+    """
+    post_processed_embedding = sequence_embeddings[1:-1]
+    sentence_embedding = np.mean(post_processed_embedding, axis=0)
+
+    return sentence_embedding, post_processed_embedding
+
+
+def bert_tokens_cleaner(
+    token_ids: List[int], token_strings: List[Text]
+) -> Tuple[List[int], List[Text]]:
+    """Token cleanup method for BERT.
+
+    Clean up tokens with the extra delimiters(##) BERT adds while breaking a token into
+    sub-tokens.
+
+    Args:
+        token_ids: List of token ids received as output from BERT Tokenizer.
+        token_strings: List of token strings received as output from BERT Tokenizer.
+
+    Returns:
+        Cleaned token ids and token strings.
+    """
+    return cleanup_tokens(list(zip(token_ids, token_strings)), "##")
+
+
+def openaigpt_tokens_cleaner(
+    token_ids: List[int], token_strings: List[Text]
+) -> Tuple[List[int], List[Text]]:
+    """Token cleanup method for GPT.
+
+    Clean up tokens with the extra delimiters(</w>) OpenAIGPT adds while breaking a
+    token into sub-tokens.
+
+    Args:
+        token_ids: List of token ids received as output from GPT Tokenizer.
+        token_strings: List of token strings received as output from GPT Tokenizer.
+
+    Returns:
+        Cleaned token ids and token strings.
+    """
+    return cleanup_tokens(list(zip(token_ids, token_strings)), "</w>")
+
+
+def gpt2_tokens_cleaner(
+    token_ids: List[int], token_strings: List[Text]
+) -> Tuple[List[int], List[Text]]:
+    """Token cleanup method for GPT2.
+
+    Clean up tokens with the extra delimiters(Ġ) GPT2 adds while breaking a token into
+    sub-tokens.
+
+    Args:
+        token_ids: List of token ids received as output from GPT Tokenizer.
+        token_strings: List of token strings received as output from GPT Tokenizer.
+
+    Returns:
+        Cleaned token ids and token strings.
+    """
+    return cleanup_tokens(list(zip(token_ids, token_strings)), "Ġ")
+
+
+def xlnet_tokens_cleaner(
+    token_ids: List[int], token_strings: List[Text]
+) -> Tuple[List[int], List[Text]]:
+    """Token cleanup method for XLNet.
+
+    Clean up tokens with the extra delimiters(▁) XLNet adds while breaking a token into
+    sub-tokens.
+
+    Args:
+        token_ids: List of token ids received as output from GPT Tokenizer.
+        token_strings: List of token strings received as output from GPT Tokenizer.
+
+    Returns:
+        Cleaned token ids and token strings.
+    """
+    return cleanup_tokens(list(zip(token_ids, token_strings)), "▁")

rasa/nlu/utils/mitie_utils.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+import typing
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Text
+
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.exceptions import InvalidConfigException
+
+if typing.TYPE_CHECKING:
+    import mitie
+
+
+class MitieModel:
+    """Wraps `MitieNLP` output to make it fingerprintable."""
+
+    def __init__(
+        self,
+        model_path: Path,
+        word_feature_extractor: Optional["mitie.total_word_feature_extractor"] = None,
+    ) -> None:
+        """Initializing MitieModel."""
+        import mitie
+
+        self.word_feature_extractor = (
+            word_feature_extractor or mitie.total_word_feature_extractor
+        )
+        self.model_path = model_path
+
+    def fingerprint(self) -> Text:
+        """Fingerprints the model path.
+
+        Use a static fingerprint as we assume this only changes if the file path
+        changes and want to avoid investigating the model in greater detail for now.
+
+        Returns:
+            Fingerprint for model.
+        """
+        return str(self.model_path)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MODEL_LOADER, is_trainable=False
+)
+class MitieNLP(GraphComponent):
+    """Component which provides the common configuration and loaded model to others.
+
+    This is used to avoid loading the Mitie model multiple times. Instead the Mitie
+    model is only loaded once and then shared by depending components.
+    """
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns default config (see parent class for full docstring)."""
+        return {
+            # name of the language model to load - this contains
+            # the MITIE feature extractor
+            "model": Path("data", "total_word_feature_extractor.dat")
+        }
+
+    def __init__(
+        self,
+        path_to_model_file: Path,
+        extractor: Optional["mitie.total_word_feature_extractor"] = None,
+    ) -> None:
+        """Constructs a new language model from the MITIE framework."""
+        self._path_to_model_file = path_to_model_file
+        self._extractor = extractor
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Lists required dependencies (see parent class for full docstring)."""
+        return ["mitie"]
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> MitieNLP:
+        """Creates component (see parent class for full docstring)."""
+        import mitie
+
+        model_file = config.get("model")
+        if not model_file:
+            raise InvalidConfigException(
+                "The MITIE component 'MitieNLP' needs "
+                "the configuration value for 'model'."
+                "Please take a look at the "
+                "documentation in the pipeline section "
+                "to get more info about this "
+                "parameter."
+            )
+        if not Path(model_file).is_file():
+            raise InvalidConfigException(
+                "The model file configured in the MITIE "
+                "component cannot be found. "
+                "Please ensure the directory path and/or "
+                "filename, '{}', are correct.".format(model_file)
+            )
+        extractor = mitie.total_word_feature_extractor(str(model_file))
+
+        return cls(Path(model_file), extractor)
+
+    def provide(self) -> MitieModel:
+        """Provides loaded `MitieModel` and path during training and inference."""
+        return MitieModel(
+            word_feature_extractor=self._extractor, model_path=self._path_to_model_file
+        )

rasa/nlu/utils/pattern_utils.py

@@ -0,0 +1,168 @@
+import re
+from typing import Dict, List, Text, Union
+
+import rasa.shared.utils.io
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.exceptions import InvalidConfigException
+
+
+def _convert_lookup_tables_to_regex(
+    training_data: TrainingData,
+    use_only_entities: bool = False,
+    use_word_boundaries: bool = True,
+) -> List[Dict[Text, Text]]:
+    r"""Convert the lookup tables from the training data to regex patterns.
+
+    Args:
+        training_data: The training data.
+        use_only_entities: If True only regex features with a name equal to a entity
+          are considered.
+        use_word_boundaries: If True add `\b` around the regex expression
+          for each lookup table expressions.
+
+    Returns:
+        A list of regex patterns.
+    """
+    patterns = []
+    for table in training_data.lookup_tables:
+        if use_only_entities and table["name"] not in training_data.entities:
+            continue
+        regex_pattern = _generate_lookup_regex(table, use_word_boundaries)
+        lookup_regex = {"name": table["name"], "pattern": regex_pattern}
+        patterns.append(lookup_regex)
+    return patterns
+
+
+def _generate_lookup_regex(
+    lookup_table: Dict[Text, Union[Text, List[Text]]], use_word_boundaries: bool = True
+) -> Text:
+    r"""Creates a regex pattern from the given lookup table.
+
+    The lookup table is either a file or a list of entries.
+
+    Args:
+        lookup_table: The lookup table.
+        use_word_boundaries: If True add `\b` around the regex expression
+          for each lookup table expressions.
+
+    Returns:
+        The regex pattern.
+    """
+    lookup_elements = lookup_table["elements"]
+
+    # if it's a list, it should be the elements directly
+    if isinstance(lookup_elements, list):
+        elements_to_regex = lookup_elements
+    # otherwise it's a file path.
+    else:
+        elements_to_regex = read_lookup_table_file(lookup_elements)
+
+    # sanitize the regex, escape special characters
+    elements_sanitized = [re.escape(e) for e in elements_to_regex]
+
+    if use_word_boundaries:
+        # regex matching elements with word boundaries on either side
+        return "(\\b" + "\\b|\\b".join(elements_sanitized) + "\\b)"
+    else:
+        return "(" + "|".join(elements_sanitized) + ")"
+
+
+def read_lookup_table_file(lookup_table_file: Text) -> List[Text]:
+    """Read the lookup table file.
+
+    Args:
+        lookup_table_file: the file path to the lookup table
+
+    Returns:
+        Elements listed in the lookup table file.
+    """
+    try:
+        f = open(lookup_table_file, "r", encoding=rasa.shared.utils.io.DEFAULT_ENCODING)
+    except OSError:
+        raise ValueError(
+            f"Could not load lookup table {lookup_table_file}. "
+            f"Please make sure you've provided the correct path."
+        )
+
+    elements_to_regex = []
+    with f:
+        for line in f:
+            new_element = line.strip()
+            if new_element:
+                elements_to_regex.append(new_element)
+    return elements_to_regex
+
+
+def _collect_regex_features(
+    training_data: TrainingData, use_only_entities: bool = False
+) -> List[Dict[Text, Text]]:
+    """Get regex features from training data.
+
+    Args:
+        training_data: The training data
+        use_only_entities: If True only regex features with a name equal to a entity
+          are considered.
+
+    Returns:
+        Regex features.
+    """
+    if not use_only_entities:
+        return training_data.regex_features
+
+    return [
+        regex
+        for regex in training_data.regex_features
+        if regex["name"] in training_data.entities
+    ]
+
+
+def extract_patterns(
+    training_data: TrainingData,
+    use_lookup_tables: bool = True,
+    use_regexes: bool = True,
+    use_only_entities: bool = False,
+    use_word_boundaries: bool = True,
+) -> List[Dict[Text, Text]]:
+    r"""Extract a list of patterns from the training data.
+
+    The patterns are constructed using the regex features and lookup tables defined
+    in the training data.
+
+    Args:
+        training_data: The training data.
+        use_only_entities: If True only lookup tables and regex features with a name
+          equal to a entity are considered.
+        use_regexes: Boolean indicating whether to use regex features or not.
+        use_lookup_tables: Boolean indicating whether to use lookup tables or not.
+        use_word_boundaries: Boolean indicating whether to use `\b` around the lookup
+            table regex expressions
+
+    Returns:
+        The list of regex patterns.
+    """
+    if not training_data.lookup_tables and not training_data.regex_features:
+        return []
+
+    patterns = []
+
+    if use_regexes:
+        patterns.extend(_collect_regex_features(training_data, use_only_entities))
+    if use_lookup_tables:
+        patterns.extend(
+            _convert_lookup_tables_to_regex(
+                training_data, use_only_entities, use_word_boundaries
+            )
+        )
+
+    # validate regexes, raise Error when invalid
+    for pattern in patterns:
+        try:
+            re.compile(pattern["pattern"])
+        except re.error:
+            raise InvalidConfigException(
+                f"Model training failed. '{pattern['pattern']}' "
+                "is not a valid regex. Please update your nlu "
+                f"training data configuration at {pattern}."
+            )
+
+    return patterns