rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/nlu/tokenizers/jieba_tokenizer.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+import glob
+import logging
+import os
+import shutil
+from typing import Any, Dict, List, Optional, Text
+
+from rasa.engine.graph import ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+
+from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
+from rasa.shared.nlu.training_data.message import Message
+
+from rasa.shared.nlu.training_data.training_data import TrainingData
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_TOKENIZER, is_trainable=True
+)
+class JiebaTokenizer(Tokenizer):
+    """This tokenizer is a wrapper for Jieba (https://github.com/fxsjy/jieba)."""
+
+    @staticmethod
+    def supported_languages() -> Optional[List[Text]]:
+        """Supported languages (see parent class for full docstring)."""
+        return ["zh"]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns default config (see parent class for full docstring)."""
+        return {
+            # default don't load custom dictionary
+            "dictionary_path": None,
+            # Flag to check whether to split intents
+            "intent_tokenization_flag": False,
+            # Symbol on which intent should be split
+            "intent_split_symbol": "_",
+            # Regular expression to detect tokens
+            "token_pattern": None,
+            # Symbol on which prefix should be split
+            "prefix_separator_symbol": None,
+        }
+
+    def __init__(
+        self, config: Dict[Text, Any], model_storage: ModelStorage, resource: Resource
+    ) -> None:
+        """Initialize the tokenizer."""
+        super().__init__(config)
+        self._model_storage = model_storage
+        self._resource = resource
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> JiebaTokenizer:
+        """Creates a new component (see parent class for full docstring)."""
+        # Path to the dictionaries on the local filesystem.
+        dictionary_path = config["dictionary_path"]
+
+        if dictionary_path is not None:
+            cls._load_custom_dictionary(dictionary_path)
+        return cls(config, model_storage, resource)
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["jieba"]
+
+    @staticmethod
+    def _load_custom_dictionary(path: Text) -> None:
+        """Load all the custom dictionaries stored in the path.
+
+        More information about the dictionaries file format can
+        be found in the documentation of jieba.
+        https://github.com/fxsjy/jieba#load-dictionary
+        """
+        import jieba
+
+        jieba_userdicts = glob.glob(f"{path}/*")
+        for jieba_userdict in jieba_userdicts:
+            logger.info(f"Loading Jieba User Dictionary at {jieba_userdict}")
+            jieba.load_userdict(jieba_userdict)
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Copies the dictionary to the model storage."""
+        self.persist()
+        return self._resource
+
+    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
+        """Tokenizes the text of the provided attribute of the incoming message."""
+        import jieba
+
+        text = message.get(attribute)
+
+        tokenized = jieba.tokenize(text)
+        tokens = [Token(word, start) for (word, start, end) in tokenized]
+
+        return self._apply_token_pattern(tokens)
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> JiebaTokenizer:
+        """Loads a custom dictionary from model storage."""
+        dictionary_path = config["dictionary_path"]
+
+        # If a custom dictionary path is in the config we know that it should have
+        # been saved to the model storage.
+        if dictionary_path is not None:
+            try:
+                with model_storage.read_from(resource) as resource_directory:
+                    cls._load_custom_dictionary(str(resource_directory))
+            except ValueError:
+                logger.debug(
+                    f"Failed to load {cls.__name__} from model storage. "
+                    f"Resource '{resource.name}' doesn't exist."
+                )
+        return cls(config, model_storage, resource)
+
+    @staticmethod
+    def _copy_files_dir_to_dir(input_dir: Text, output_dir: Text) -> None:
+        # make sure target path exists
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        target_file_list = glob.glob(f"{input_dir}/*")
+        for target_file in target_file_list:
+            shutil.copy2(target_file, output_dir)
+
+    def persist(self) -> None:
+        """Persist the custom dictionaries."""
+        dictionary_path = self._config["dictionary_path"]
+        if dictionary_path is not None:
+            with self._model_storage.write_to(self._resource) as resource_directory:
+                self._copy_files_dir_to_dir(dictionary_path, str(resource_directory))

rasa/nlu/tokenizers/mitie_tokenizer.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+from typing import List, Text, Dict, Any
+
+from rasa.engine.graph import ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
+from rasa.shared.nlu.training_data.message import Message
+
+from rasa.shared.utils.io import DEFAULT_ENCODING
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_TOKENIZER, is_trainable=False
+)
+class MitieTokenizer(Tokenizer):
+    """Tokenizes messages using the `mitie` library.."""
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns default config (see parent class for full docstring)."""
+        return {
+            # Flag to check whether to split intents
+            "intent_tokenization_flag": False,
+            # Symbol on which intent should be split
+            "intent_split_symbol": "_",
+            # Regular expression to detect tokens
+            "token_pattern": None,
+            # Symbol on which prefix should be split
+            "prefix_separator_symbol": None,
+        }
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["mitie"]
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> MitieTokenizer:
+        """Creates a new component (see parent class for full docstring)."""
+        return cls(config)
+
+    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
+        """Tokenizes the text of the provided attribute of the incoming message."""
+        import mitie
+
+        text = message.get(attribute)
+
+        encoded_sentence = text.encode(DEFAULT_ENCODING)
+        tokenized = mitie.tokenize_with_offsets(encoded_sentence)
+        tokens = [
+            self._token_from_offset(token, offset, encoded_sentence)
+            for token, offset in tokenized
+        ]
+
+        return self._apply_token_pattern(tokens)
+
+    def _token_from_offset(
+        self, text: bytes, offset: int, encoded_sentence: bytes
+    ) -> Token:
+        return Token(
+            text.decode(DEFAULT_ENCODING),
+            self._byte_to_char_offset(encoded_sentence, offset),
+        )
+
+    @staticmethod
+    def _byte_to_char_offset(text: bytes, byte_offset: int) -> int:
+        return len(text[:byte_offset].decode(DEFAULT_ENCODING))

rasa/nlu/tokenizers/spacy_tokenizer.py

@@ -0,0 +1,72 @@
+import typing
+from typing import Dict, Text, List, Any, Optional, Type
+
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.nlu.utils.spacy_utils import SpacyNLP
+from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
+from rasa.nlu.constants import SPACY_DOCS
+from rasa.shared.nlu.training_data.message import Message
+
+if typing.TYPE_CHECKING:
+    from spacy.tokens.doc import Doc
+
+POS_TAG_KEY = "pos"
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_TOKENIZER, is_trainable=False
+)
+class SpacyTokenizer(Tokenizer):
+    """Tokenizer that uses SpaCy."""
+
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [SpacyNLP]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """The component's default config (see parent class for full docstring)."""
+        return {
+            # Flag to check whether to split intents
+            "intent_tokenization_flag": False,
+            # Symbol on which intent should be split
+            "intent_split_symbol": "_",
+            # Regular expression to detect tokens
+            "token_pattern": None,
+            # Symbol on which prefix should be split
+            "prefix_separator_symbol": None,
+        }
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["spacy"]
+
+    def _get_doc(self, message: Message, attribute: Text) -> Optional["Doc"]:
+        return message.get(SPACY_DOCS[attribute])
+
+    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
+        """Tokenizes the text of the provided attribute of the incoming message."""
+        doc = self._get_doc(message, attribute)
+        if not doc:
+            return []
+
+        tokens = [
+            Token(
+                t.text, t.idx, lemma=t.lemma_, data={POS_TAG_KEY: self._tag_of_token(t)}
+            )
+            for t in doc
+            if t.text and t.text.strip()
+        ]
+
+        return self._apply_token_pattern(tokens)
+
+    @staticmethod
+    def _tag_of_token(token: Any) -> Text:
+        import spacy
+
+        if spacy.about.__version__ > "2" and token._.has("tag"):
+            return token._.get("tag")
+        else:
+            return token.tag_

rasa/nlu/tokenizers/tokenizer.py

@@ -0,0 +1,239 @@
+import abc
+import logging
+import re
+
+from typing import Text, List, Dict, Any, Optional
+
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.constants import TOKENS_NAMES, MESSAGE_ATTRIBUTES
+from rasa.shared.nlu.constants import (
+    INTENT,
+    INTENT_RESPONSE_KEY,
+    RESPONSE_IDENTIFIER_DELIMITER,
+    ACTION_NAME,
+)
+import rasa.shared.utils.io
+
+logger = logging.getLogger(__name__)
+
+
+class Token:
+    """Used by `Tokenizers` which split a single message into multiple `Token`s."""
+
+    def __init__(
+        self,
+        text: Text,
+        start: int,
+        end: Optional[int] = None,
+        data: Optional[Dict[Text, Any]] = None,
+        lemma: Optional[Text] = None,
+    ) -> None:
+        """Create a `Token`.
+
+        Args:
+            text: The token text.
+            start: The start index of the token within the entire message.
+            end: The end index of the token within the entire message.
+            data: Additional token data.
+            lemma: An optional lemmatized version of the token text.
+        """
+        self.text = text
+        self.start = start
+        self.end = end if end else start + len(text)
+
+        self.data = data if data else {}
+        self.lemma = lemma or text
+
+    def set(self, prop: Text, info: Any) -> None:
+        """Set property value."""
+        self.data[prop] = info
+
+    def get(self, prop: Text, default: Optional[Any] = None) -> Any:
+        """Returns token value."""
+        return self.data.get(prop, default)
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, Token):
+            return NotImplemented
+        return (self.start, self.end, self.text, self.lemma) == (
+            other.start,
+            other.end,
+            other.text,
+            other.lemma,
+        )
+
+    def __lt__(self, other: Any) -> bool:
+        if not isinstance(other, Token):
+            return NotImplemented
+        return (self.start, self.end, self.text, self.lemma) < (
+            other.start,
+            other.end,
+            other.text,
+            other.lemma,
+        )
+
+    def __repr__(self) -> Text:
+        return f"<Token object value='{self.text}' start={self.start} end={self.end} \
+        at {hex(id(self))}>"
+
+    def fingerprint(self) -> Text:
+        """Returns a stable hash for this Token."""
+        return rasa.shared.utils.io.deep_container_fingerprint(
+            [self.text, self.start, self.end, self.lemma, self.data]
+        )
+
+
+class Tokenizer(GraphComponent, abc.ABC):
+    """Base class for tokenizers."""
+
+    def __init__(self, config: Dict[Text, Any]) -> None:
+        """Construct a new tokenizer."""
+        self._config = config
+        # flag to check whether to split intents
+        self.intent_tokenization_flag = config["intent_tokenization_flag"]
+        # split symbol for intents
+        self.intent_split_symbol = config["intent_split_symbol"]
+        # token pattern to further split tokens
+        token_pattern = config.get("token_pattern")
+        self.token_pattern_regex = None
+        if token_pattern:
+            self.token_pattern_regex = re.compile(token_pattern)
+        # split intent to prefix and suffix greedily, None means don't split
+        self.prefix_separator_symbol = config.get("prefix_separator_symbol")
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Creates a new component (see parent class for full docstring)."""
+        return cls(config)
+
+    @abc.abstractmethod
+    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
+        """Tokenizes the text of the provided attribute of the incoming message."""
+        ...
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        """Tokenize all training data."""
+        for example in training_data.training_examples:
+            for attribute in MESSAGE_ATTRIBUTES:
+                if (
+                    example.get(attribute) is not None
+                    and not example.get(attribute) == ""
+                ):
+                    if attribute in [INTENT, ACTION_NAME, INTENT_RESPONSE_KEY]:
+                        tokens = self._split_name(example, attribute)
+                    else:
+                        tokens = self.tokenize(example, attribute)
+                    example.set(TOKENS_NAMES[attribute], tokens)
+        return training_data
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Tokenize the incoming messages."""
+        for message in messages:
+            for attribute in MESSAGE_ATTRIBUTES:
+                if isinstance(message.get(attribute), str):
+                    if attribute in [
+                        INTENT,
+                        ACTION_NAME,
+                        RESPONSE_IDENTIFIER_DELIMITER,
+                    ]:
+                        tokens = self._split_name(message, attribute)
+                    else:
+                        tokens = self.tokenize(message, attribute)
+
+                    message.set(TOKENS_NAMES[attribute], tokens)
+        return messages
+
+    def _tokenize_on_split_symbol(self, text: Text) -> List[Text]:
+        words = (
+            text.split(self.intent_split_symbol)
+            if self.intent_tokenization_flag
+            else [text]
+        )
+
+        return words
+
+    def _split_name(self, message: Message, attribute: Text = INTENT) -> List[Token]:
+        orig_text = message.get(attribute)
+
+        if (
+            self.prefix_separator_symbol is not None
+            and self.prefix_separator_symbol in orig_text
+        ):
+            prefix, text = orig_text.split(self.prefix_separator_symbol, maxsplit=1)
+        else:
+            prefix, text = None, orig_text
+
+        # for INTENT_RESPONSE_KEY attribute,
+        # first split by RESPONSE_IDENTIFIER_DELIMITER
+        if attribute == INTENT_RESPONSE_KEY:
+            intent, response_key = text.split(RESPONSE_IDENTIFIER_DELIMITER)
+            words = self._tokenize_on_split_symbol(
+                intent
+            ) + self._tokenize_on_split_symbol(response_key)
+
+        else:
+            words = self._tokenize_on_split_symbol(text)
+
+        if prefix is not None:
+            words = self._tokenize_on_split_symbol(prefix) + words
+
+        return self._convert_words_to_tokens(words, orig_text)
+
+    def _apply_token_pattern(self, tokens: List[Token]) -> List[Token]:
+        """Apply the token pattern to the given tokens.
+
+        Args:
+            tokens: list of tokens to split
+
+        Returns:
+            List of tokens.
+        """
+        if not self.token_pattern_regex:
+            return tokens
+
+        final_tokens = []
+        for token in tokens:
+            new_tokens = self.token_pattern_regex.findall(token.text)
+            new_tokens = [t for t in new_tokens if t]
+
+            if not new_tokens:
+                final_tokens.append(token)
+
+            running_offset = 0
+            for new_token in new_tokens:
+                word_offset = token.text.index(new_token, running_offset)
+                word_len = len(new_token)
+                running_offset = word_offset + word_len
+                final_tokens.append(
+                    Token(
+                        new_token,
+                        token.start + word_offset,
+                        data=token.data,
+                        lemma=token.lemma,
+                    )
+                )
+
+        return final_tokens
+
+    @staticmethod
+    def _convert_words_to_tokens(words: List[Text], text: Text) -> List[Token]:
+        running_offset = 0
+        tokens = []
+
+        for word in words:
+            word_offset = text.index(word, running_offset)
+            word_len = len(word)
+            running_offset = word_offset + word_len
+            tokens.append(Token(word, word_offset))
+
+        return tokens

rasa/nlu/tokenizers/whitespace_tokenizer.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+from typing import Any, Dict, List, Optional, Text
+
+import regex
+
+import rasa.shared.utils.io
+import rasa.utils.io
+
+from rasa.engine.graph import ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
+from rasa.shared.constants import DOCS_URL_COMPONENTS
+from rasa.shared.nlu.training_data.message import Message
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_TOKENIZER, is_trainable=False
+)
+class WhitespaceTokenizer(Tokenizer):
+    """Creates features for entity extraction."""
+
+    @staticmethod
+    def not_supported_languages() -> Optional[List[Text]]:
+        """The languages that are not supported."""
+        return ["zh", "ja", "th"]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns the component's default config."""
+        return {
+            # Flag to check whether to split intents
+            "intent_tokenization_flag": False,
+            # Symbol on which intent should be split
+            "intent_split_symbol": "_",
+            # Regular expression to detect tokens
+            "token_pattern": None,
+            # Symbol on which prefix should be split
+            "prefix_separator_symbol": None,
+        }
+
+    def __init__(self, config: Dict[Text, Any]) -> None:
+        """Initialize the tokenizer."""
+        super().__init__(config)
+        if "case_sensitive" in self._config:
+            rasa.shared.utils.io.raise_warning(
+                "The option 'case_sensitive' was moved from the tokenizers to the "
+                "featurizers.",
+                docs=DOCS_URL_COMPONENTS,
+            )
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> WhitespaceTokenizer:
+        """Creates a new component (see parent class for full docstring)."""
+        # Path to the dictionaries on the local filesystem.
+        return cls(config)
+
+    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
+        original_text = message.get(attribute)
+        text = rasa.utils.io.remove_emojis(original_text)
+        # we need to use regex instead of re, because of
+        # https://stackoverflow.com/questions/12746458/python-unicode-regular-expression-matching-failing-with-some-unicode-characters
+
+        # remove 'not a word character' if
+        words = regex.sub(
+            # there is a space or an end of a string after it
+            r"[^\w#@&]+(?=\s|$)|"
+            # there is a space or beginning of a string before it
+            # not followed by a number
+            r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
+            # not in between numbers and not . or @ or & or - or #
+            # e.g. 10'000.00 or blabla@gmail.com
+            # and not url characters
+            r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
+            " ",
+            text,
+        ).split()
+
+        words = [w for w in words if w]
+
+        # if we removed everything like smiles `:)`, use the whole text as 1 token
+        if not words:
+            text = original_text
+            words = [text]
+
+        tokens = self._convert_words_to_tokens(words, text)
+
+        return self._apply_token_pattern(tokens)

rasa/nlu/utils/__init__.py

@@ -0,0 +1,35 @@
+from typing import Any, Text
+
+import rasa.shared.utils.io
+
+
+def write_json_to_file(filename: Text, obj: Any, **kwargs: Any) -> None:
+    """Write an object as a json string to a file."""
+    write_to_file(filename, rasa.shared.utils.io.json_to_string(obj, **kwargs))
+
+
+def write_to_file(filename: Text, text: Any) -> None:
+    """Write a text to a file."""
+    rasa.shared.utils.io.write_text_file(str(text), filename)
+
+
+def is_url(resource_name: Text) -> bool:
+    """Check whether the url specified is a well formed one.
+
+    Args:
+        resource_name: Remote URL to validate
+
+    Returns:
+        `True` if valid, otherwise `False`.
+    """
+    from urllib import parse
+
+    try:
+        result = parse.urlparse(resource_name)
+    except Exception:
+        return False
+
+    if result.scheme == "file":
+        return bool(result.path)
+
+    return bool(result.scheme in ["http", "https", "ftp", "ftps"] and result.netloc)