rasa-pro 3.9.18__py3-none-any.whl

rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, List, Optional, Text, Tuple, Type
+
+import numpy as np
+import scipy.sparse
+
+from rasa.nlu.tokenizers.tokenizer import Tokenizer
+import rasa.shared.utils.io
+import rasa.utils.io
+import rasa.nlu.utils.pattern_utils as pattern_utils
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.constants import TOKENS_NAMES
+from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer
+from rasa.shared.nlu.constants import TEXT, RESPONSE, ACTION_TEXT
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.message import Message
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=True
+)
+class RegexFeaturizer(SparseFeaturizer, GraphComponent):
+    """Adds message features based on regex expressions."""
+
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [Tokenizer]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns the component's default config."""
+        return {
+            **SparseFeaturizer.get_default_config(),
+            # text will be processed with case sensitive as default
+            "case_sensitive": True,
+            # use lookup tables to generate features
+            "use_lookup_tables": True,
+            # use regexes to generate features
+            "use_regexes": True,
+            # use match word boundaries for lookup table
+            "use_word_boundaries": True,
+        }
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        known_patterns: Optional[List[Dict[Text, Text]]] = None,
+    ) -> None:
+        """Constructs new features for regexes and lookup table using regex expressions.
+
+        Args:
+            config: Configuration for the component.
+            model_storage: Storage which graph components can use to persist and load
+                themselves.
+            resource: Resource locator for this component which can be used to persist
+                and load itself from the `model_storage`.
+            execution_context: Information about the current graph run.
+            known_patterns: Regex Patterns the component should pre-load itself with.
+        """
+        super().__init__(execution_context.node_name, config)
+
+        self._model_storage = model_storage
+        self._resource = resource
+
+        self.known_patterns = known_patterns if known_patterns else []
+        self.case_sensitive = config["case_sensitive"]
+        self.finetune_mode = execution_context.is_finetuning
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> RegexFeaturizer:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource, execution_context)
+
+    def _merge_new_patterns(self, new_patterns: List[Dict[Text, Text]]) -> None:
+        """Updates already known patterns with new patterns extracted from data.
+
+        New patterns should always be added to the end of the existing
+        patterns and the order of the existing patterns should not be disturbed.
+
+        Args:
+            new_patterns: Patterns extracted from training data and to be merged with
+                known patterns.
+        """
+        pattern_name_index_map = {
+            pattern["name"]: index for index, pattern in enumerate(self.known_patterns)
+        }
+        for extra_pattern in new_patterns:
+            new_pattern_name = extra_pattern["name"]
+
+            # Some patterns may have just new examples added
+            # to them. These do not count as additional pattern.
+            if new_pattern_name in pattern_name_index_map:
+                self.known_patterns[pattern_name_index_map[new_pattern_name]][
+                    "pattern"
+                ] = extra_pattern["pattern"]
+            else:
+                self.known_patterns.append(extra_pattern)
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Trains the component with all patterns extracted from training data."""
+        patterns_from_data = pattern_utils.extract_patterns(
+            training_data,
+            use_lookup_tables=self._config["use_lookup_tables"],
+            use_regexes=self._config["use_regexes"],
+            use_word_boundaries=self._config["use_word_boundaries"],
+        )
+        if self.finetune_mode:
+            # Merge patterns extracted from data with known patterns
+            self._merge_new_patterns(patterns_from_data)
+        else:
+            self.known_patterns = patterns_from_data
+
+        self._persist()
+        return self._resource
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        """Processes the training examples (see parent class for full docstring)."""
+        for example in training_data.training_examples:
+            for attribute in [TEXT, RESPONSE, ACTION_TEXT]:
+                self._text_features_with_regex(example, attribute)
+
+        return training_data
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Featurizes all given messages in-place.
+
+        Returns:
+          the given list of messages which have been modified in-place
+        """
+        for message in messages:
+            self._text_features_with_regex(message, TEXT)
+
+        return messages
+
+    def _text_features_with_regex(self, message: Message, attribute: Text) -> None:
+        """Helper method to extract features and set them appropriately in the message.
+
+        Args:
+            message: Message to be featurized.
+            attribute: Attribute of message to be featurized.
+        """
+        if self.known_patterns:
+            sequence_features, sentence_features = self._features_for_patterns(
+                message, attribute
+            )
+
+            self.add_features_to_message(
+                sequence_features, sentence_features, attribute, message
+            )
+
+    def _features_for_patterns(
+        self, message: Message, attribute: Text
+    ) -> Tuple[Optional[scipy.sparse.coo_matrix], Optional[scipy.sparse.coo_matrix]]:
+        """Checks which known patterns match the message.
+
+        Given a sentence, returns a vector of {1,0} values indicating which
+        regexes did match. Furthermore, if the
+        message is tokenized, the function will mark all tokens with a dict
+        relating the name of the regex to whether it was matched.
+
+        Args:
+            message: Message to be featurized.
+            attribute: Attribute of message to be featurized.
+
+        Returns:
+           Token and sentence level features of message attribute.
+        """
+        # Attribute not set (e.g. response not present)
+        if not message.get(attribute):
+            return None, None
+
+        tokens = message.get(TOKENS_NAMES[attribute], [])
+
+        if not tokens:
+            # nothing to featurize
+            return None, None
+
+        flags = 0  # default flag
+        if not self.case_sensitive:
+            flags = re.IGNORECASE
+
+        sequence_length = len(tokens)
+
+        num_patterns = len(self.known_patterns)
+
+        sequence_features = np.zeros([sequence_length, num_patterns])
+        sentence_features = np.zeros([1, num_patterns])
+
+        for pattern_index, pattern in enumerate(self.known_patterns):
+            matches = list(
+                re.finditer(pattern["pattern"], message.get(attribute), flags=flags)
+            )
+
+            for token_index, t in enumerate(tokens):
+                patterns = t.get("pattern", default={})
+                patterns[pattern["name"]] = False
+
+                for match in matches:
+                    if t.start < match.end() and t.end > match.start():
+                        patterns[pattern["name"]] = True
+                        sequence_features[token_index][pattern_index] = 1.0
+                        if attribute in [RESPONSE, TEXT, ACTION_TEXT]:
+                            # sentence vector should contain all patterns
+                            sentence_features[0][pattern_index] = 1.0
+
+                t.set("pattern", patterns)
+
+        return (
+            scipy.sparse.coo_matrix(sequence_features),
+            scipy.sparse.coo_matrix(sentence_features),
+        )
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> RegexFeaturizer:
+        """Loads trained component (see parent class for full docstring)."""
+        known_patterns = None
+
+        try:
+            with model_storage.read_from(resource) as model_dir:
+                patterns_file_name = model_dir / "patterns.json"
+                known_patterns = rasa.shared.utils.io.read_json_file(patterns_file_name)
+        except (ValueError, FileNotFoundError):
+            logger.warning(
+                f"Failed to load `{cls.__class__.__name__}` from model storage. "
+                f"Resource '{resource.name}' doesn't exist."
+            )
+
+        return cls(
+            config,
+            model_storage,
+            resource,
+            execution_context,
+            known_patterns=known_patterns,
+        )
+
+    def _persist(self) -> None:
+        with self._model_storage.write_to(self._resource) as model_dir:
+            regex_file = model_dir / "patterns.json"
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                regex_file, self.known_patterns
+            )
+
+    @classmethod
+    def validate_config(cls, config: Dict[Text, Any]) -> None:
+        """Validates that the component is configured properly."""
+        pass

rasa/nlu/featurizers/sparse_featurizer/sparse_featurizer.py

@@ -0,0 +1,9 @@
+from abc import ABC
+import scipy.sparse
+from rasa.nlu.featurizers.featurizer import Featurizer
+
+
+class SparseFeaturizer(Featurizer[scipy.sparse.spmatrix], ABC):
+    """Base class for all sparse featurizers."""
+
+    pass

rasa/nlu/model.py

@@ -0,0 +1,24 @@
+import logging
+from typing import Text
+
+from rasa.shared.exceptions import RasaException
+
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: remove/move
+class InvalidModelError(RasaException):
+    """Raised when a model failed to load.
+
+    Attributes:
+        message -- explanation of why the model is invalid
+    """
+
+    def __init__(self, message: Text) -> None:
+        """Initialize message attribute."""
+        self.message = message
+        super(InvalidModelError, self).__init__(message)
+
+    def __str__(self) -> Text:
+        return self.message

rasa/nlu/persistor.py

@@ -0,0 +1,282 @@
+import abc
+import structlog
+import os
+import shutil
+from typing import Optional, Text, Tuple, TYPE_CHECKING
+
+from rasa.shared.exceptions import RasaException
+
+import rasa.shared.utils.common
+import rasa.utils.common
+
+if TYPE_CHECKING:
+    from azure.storage.blob import ContainerClient
+
+structlogger = structlog.get_logger()
+
+
+def get_persistor(name: Text) -> Optional["Persistor"]:
+    """Returns an instance of the requested persistor.
+
+    Currently, `aws`, `gcs`, `azure` and providing module paths are supported remote
+    storages.
+    """
+    if name == "aws":
+        return AWSPersistor(
+            os.environ.get("BUCKET_NAME"), os.environ.get("AWS_ENDPOINT_URL")
+        )
+    if name == "gcs":
+        return GCSPersistor(os.environ.get("BUCKET_NAME"))
+
+    if name == "azure":
+        return AzurePersistor(
+            os.environ.get("AZURE_CONTAINER"),
+            os.environ.get("AZURE_ACCOUNT_NAME"),
+            os.environ.get("AZURE_ACCOUNT_KEY"),
+        )
+    if name:
+        try:
+            persistor = rasa.shared.utils.common.class_from_module_path(name)
+            return persistor()
+        except ImportError:
+            raise ImportError(
+                f"Unknown model persistor {name}. Please make sure to "
+                "either use an included model persistor (`aws`, `gcs` "
+                "or `azure`) or specify the module path to an external "
+                "model persistor."
+            )
+    return None
+
+
+class Persistor(abc.ABC):
+    """Store models in cloud and fetch them when needed."""
+
+    def persist(self, model_directory: Text, model_name: Text) -> None:
+        """Uploads a model persisted in the `target_dir` to cloud storage."""
+        if not os.path.isdir(model_directory):
+            raise ValueError(f"Target directory '{model_directory}' not found.")
+
+        file_key, tar_path = self._compress(model_directory, model_name)
+        self._persist_tar(file_key, tar_path)
+
+    def retrieve(self, model_name: Text, target_path: Text) -> None:
+        """Downloads a model that has been persisted to cloud storage."""
+        tar_name = model_name
+
+        if not model_name.endswith("tar.gz"):
+            # ensure backward compatibility
+            tar_name = self._tar_name(model_name)
+
+        self._retrieve_tar(tar_name)
+        self._copy(os.path.basename(tar_name), target_path)
+
+    @abc.abstractmethod
+    def _retrieve_tar(self, filename: Text) -> None:
+        """Downloads a model previously persisted to cloud storage."""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def _persist_tar(self, filekey: Text, tarname: Text) -> None:
+        """Uploads a model persisted in the `target_dir` to cloud storage."""
+        raise NotImplementedError
+
+    def _compress(self, model_directory: Text, model_name: Text) -> Tuple[Text, Text]:
+        """Creates a compressed archive and returns key and tar."""
+        import tempfile
+
+        dirpath = tempfile.mkdtemp()
+        base_name = self._tar_name(model_name, include_extension=False)
+        tar_name = shutil.make_archive(
+            os.path.join(dirpath, base_name),
+            "gztar",
+            root_dir=model_directory,
+            base_dir=".",
+        )
+        file_key = os.path.basename(tar_name)
+        return file_key, tar_name
+
+    @staticmethod
+    def _tar_name(model_name: Text, include_extension: bool = True) -> Text:
+        ext = ".tar.gz" if include_extension else ""
+        return f"{model_name}{ext}"
+
+    @staticmethod
+    def _copy(compressed_path: Text, target_path: Text) -> None:
+        shutil.copy2(compressed_path, target_path)
+
+
+class AWSPersistor(Persistor):
+    """Store models on S3.
+
+    Fetches them when needed, instead of storing them on the local disk.
+    """
+
+    def __init__(
+        self,
+        bucket_name: Text,
+        endpoint_url: Optional[Text] = None,
+        region_name: Optional[Text] = None,
+    ) -> None:
+        import boto3
+
+        super().__init__()
+        self.s3 = boto3.resource(
+            "s3", endpoint_url=endpoint_url, region_name=region_name
+        )
+        self._ensure_bucket_exists(bucket_name, region_name)
+        self.bucket_name = bucket_name
+        self.bucket = self.s3.Bucket(bucket_name)
+
+    def _ensure_bucket_exists(
+        self, bucket_name: Text, region_name: Optional[Text] = None
+    ) -> None:
+        import botocore
+
+        # noinspection PyUnresolvedReferences
+        try:
+            self.s3.meta.client.head_bucket(Bucket=bucket_name)
+        except botocore.exceptions.ClientError as e:
+            error_code = int(e.response["Error"]["Code"])
+            if error_code == 403:
+                log = (
+                    f"Access to the specified bucket '{bucket_name}' is forbidden. "
+                    "Please make sure you have the necessary "
+                    "permission to access the bucket."
+                )
+                structlogger.error(
+                    "aws_persistor.ensure_bucket_exists.bucket_access_forbidden",
+                    bucket_name=bucket_name,
+                    event_info=log,
+                )
+                raise RasaException(log)
+            elif error_code == 404:
+                log = (
+                    f"The specified bucket '{bucket_name}' does not exist. "
+                    "Please make sure to create the bucket first."
+                )
+                structlogger.error(
+                    "aws_persistor.ensure_bucket_exists.bucket_not_found",
+                    bucket_name=bucket_name,
+                    event_info=log,
+                )
+                raise RasaException(log)
+
+    def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
+        """Uploads a model persisted in the `target_dir` to s3."""
+        with open(tar_path, "rb") as f:
+            self.s3.Object(self.bucket_name, file_key).put(Body=f)
+
+    def _retrieve_tar(self, model_path: Text) -> None:
+        """Downloads a model that has previously been persisted to s3."""
+        tar_name = os.path.basename(model_path)
+        with open(tar_name, "wb") as f:
+            self.bucket.download_fileobj(model_path, f)
+
+
+class GCSPersistor(Persistor):
+    """Store models on Google Cloud Storage.
+
+    Fetches them when needed, instead of storing them on the local disk.
+    """
+
+    def __init__(self, bucket_name: Text) -> None:
+        """Initialise class with client and bucket."""
+        # there are no type hints in this repo for now
+        # https://github.com/googleapis/python-storage/issues/393
+        from google.cloud import storage  # type: ignore[attr-defined]
+
+        super().__init__()
+
+        self.storage_client = storage.Client()
+        self._ensure_bucket_exists(bucket_name)
+
+        self.bucket_name = bucket_name
+        self.bucket = self.storage_client.bucket(bucket_name)
+
+    def _ensure_bucket_exists(self, bucket_name: Text) -> None:
+        from google.cloud import exceptions
+
+        try:
+            self.storage_client.get_bucket(bucket_name)
+        except exceptions.NotFound:
+            log = (
+                f"The specified bucket '{bucket_name}' does not exist. "
+                "Please make sure to create the bucket first."
+            )
+            structlogger.error(
+                "gcp_persistor.ensure_bucket_exists.bucket_not_found",
+                bucket_name=bucket_name,
+                event_info=log,
+            )
+            raise RasaException(log)
+        except exceptions.Forbidden:
+            log = (
+                f"Access to the specified bucket '{bucket_name}' is forbidden. "
+                "Please make sure you have the necessary "
+                "permission to access the bucket. "
+            )
+            structlogger.error(
+                "gcp_persistor.ensure_bucket_exists.bucket_access_forbidden",
+                bucket_name=bucket_name,
+                event_info=log,
+            )
+            raise RasaException(log)
+
+    def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
+        """Uploads a model persisted in the `target_dir` to GCS."""
+        blob = self.bucket.blob(file_key)
+        blob.upload_from_filename(tar_path)
+
+    def _retrieve_tar(self, target_filename: Text) -> None:
+        """Downloads a model that has previously been persisted to GCS."""
+        blob = self.bucket.blob(target_filename)
+        blob.download_to_filename(target_filename)
+
+
+class AzurePersistor(Persistor):
+    """Store models on Azure."""
+
+    def __init__(
+        self, azure_container: Text, azure_account_name: Text, azure_account_key: Text
+    ) -> None:
+        from azure.storage.blob import BlobServiceClient
+
+        super().__init__()
+
+        self.blob_service = BlobServiceClient(
+            account_url=f"https://{azure_account_name}.blob.core.windows.net/",
+            credential=azure_account_key,
+        )
+        self.container_name = azure_container
+        self._ensure_container_exists()
+
+    def _ensure_container_exists(self) -> None:
+        if self._container_client().exists():
+            pass
+        else:
+            log = (
+                f"The specified container '{self.container_name}' does not exist."
+                "Please make sure to create the container first."
+            )
+            structlogger.error(
+                "azure_persistor.ensure_container_exists.container_not_found",
+                container_name=self.container_name,
+                event_info=log,
+            )
+            raise RasaException(log)
+
+    def _container_client(self) -> "ContainerClient":
+        return self.blob_service.get_container_client(self.container_name)
+
+    def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
+        """Uploads a model persisted in the `target_dir` to Azure."""
+        with open(tar_path, "rb") as data:
+            self._container_client().upload_blob(name=file_key, data=data)
+
+    def _retrieve_tar(self, target_filename: Text) -> None:
+        """Downloads a model that has previously been persisted to Azure."""
+        blob_client = self._container_client().get_blob_client(target_filename)
+
+        with open(target_filename, "wb") as blob:
+            download_stream = blob_client.download_blob()
+            blob.write(download_stream.readall())

rasa/nlu/run.py

@@ -0,0 +1,27 @@
+import asyncio
+import logging
+from typing import Text
+
+from rasa.core.agent import Agent
+from rasa.shared.utils.cli import print_info, print_success
+from rasa.shared.utils.io import json_to_string
+
+logger = logging.getLogger(__name__)
+
+
+def run_cmdline(model_path: Text) -> None:
+    """Loops over CLI input, passing each message to a loaded NLU model."""
+    agent = Agent.load(model_path)
+
+    print_success("NLU model loaded. Type a message and press enter to parse it.")
+    while True:
+        print_success("Next message:")
+        try:
+            message = input().strip()
+        except (EOFError, KeyboardInterrupt):
+            print_info("Wrapping up command line chat...")
+            break
+
+        result = asyncio.run(agent.parse_message(message))
+
+        print(json_to_string(result))