rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/anonymization/anonymization_rule_executor.py

@@ -0,0 +1,260 @@
+import logging
+import typing
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Text, Union
+
+from faker import Faker
+from presidio_analyzer import AnalyzerEngine
+from presidio_analyzer.nlp_engine import (
+    SpacyNlpEngine,
+    StanzaNlpEngine,
+    TransformersNlpEngine,
+)
+from presidio_anonymizer import AnonymizerEngine
+from presidio_anonymizer.entities import OperatorConfig
+from rasa.shared.exceptions import RasaException
+
+from rasa.utils.singleton import Singleton
+
+if typing.TYPE_CHECKING:
+    from presidio_analyzer.nlp_engine.nlp_engine import NlpEngine
+
+DEFAULT_PRESIDIO_LANG_CODE = "en"
+DEFAULT_PRESIDIO_MODEL_NAME = "en_core_web_lg"
+DEFAULT_PRESIDIO_MODEL_PROVIDER = "spacy"
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class AnonymizationRule:
+    """A rule for anonymizing a given text."""
+
+    entity_name: Text
+    substitution: Text = "mask"
+    value: Optional[Text] = None
+
+
+@dataclass
+class AnonymizationRuleList:
+    """A list of anonymization rules."""
+
+    id: Text
+    rule_list: List[AnonymizationRule]
+    language: Text = DEFAULT_PRESIDIO_LANG_CODE
+    model_provider: Text = DEFAULT_PRESIDIO_MODEL_PROVIDER
+    models: Union[Text, Dict[Text, Text]] = DEFAULT_PRESIDIO_MODEL_NAME
+
+
+class AnonymizationAnalyzer(metaclass=Singleton):
+    """Anonymization analyzer."""
+
+    presidio_analyzer_engine = None
+
+    def __init__(self, anonymization_rule_list: AnonymizationRuleList):
+        """Initialise the anonymization analyzer."""
+        if self.presidio_analyzer_engine is None:
+            self.presidio_analyzer_engine = self._get_analyzer_engine(
+                anonymization_rule_list
+            )
+
+    @staticmethod
+    def _get_analyzer_engine(
+        anonymization_rule_list: AnonymizationRuleList,
+    ) -> AnalyzerEngine:
+        """Returns an analyzer engine for all the anonymization rule lists."""
+        try:
+            nlp_engine = AnonymizationAnalyzer._build_presidio_nlp_engine(
+                anonymization_rule_list
+            )
+        except (OSError, ImportError) as exception:
+            raise RasaException(
+                "Failed to load Presidio nlp engine. "
+                "Please check that you have provided "
+                "a valid model name."
+            ) from exception
+
+        return AnalyzerEngine(
+            nlp_engine=nlp_engine,
+            supported_languages=[anonymization_rule_list.language],
+        )
+
+    @staticmethod
+    def _build_presidio_nlp_engine(
+        anonymization_rule_list: AnonymizationRuleList,
+    ) -> "NlpEngine":
+        """Creates an instance of the Presidio nlp engine."""
+        if anonymization_rule_list.model_provider == "transformers":
+            nlp_engine = TransformersNlpEngine(
+                models={
+                    anonymization_rule_list.language: anonymization_rule_list.models
+                },
+            )
+        elif anonymization_rule_list.model_provider == "stanza":
+            nlp_engine = StanzaNlpEngine(
+                models={
+                    anonymization_rule_list.language: anonymization_rule_list.models
+                },
+            )
+        else:
+            nlp_engine = SpacyNlpEngine(
+                models={
+                    anonymization_rule_list.language: anonymization_rule_list.models
+                },
+            )
+
+        return nlp_engine
+
+
+class AnonymizationRuleExecutor:
+    """Executes a given anonymization rule set on a given text."""
+
+    def __init__(self, anonymization_rule_list: AnonymizationRuleList):
+        """Initialize the anonymization rule executor."""
+        self.anonymization_rule_list = anonymization_rule_list
+
+        is_valid_rule_list = self._validate_anonymization_rule_list(
+            anonymization_rule_list
+        )
+
+        self.analyzer = (
+            AnonymizationAnalyzer(anonymization_rule_list)
+            if is_valid_rule_list
+            else None
+        )
+
+        self.anonymizer_engine = AnonymizerEngine()  # type: ignore
+
+    @staticmethod
+    def _validate_anonymization_rule_list(
+        anonymization_rule_list: AnonymizationRuleList,
+    ) -> bool:
+        """Validates the given anonymization rule list object."""
+        if (
+            anonymization_rule_list.language != DEFAULT_PRESIDIO_LANG_CODE
+            and anonymization_rule_list.models == DEFAULT_PRESIDIO_MODEL_NAME
+        ):
+            logger.debug(
+                f"Anonymization rule list language is "
+                f"'{anonymization_rule_list.language}', "
+                f"but no specific model name was provided. "
+                f"You must specify the spaCy model name in the"
+                f"endpoints yaml file. "
+                f"Cannot proceed with anonymization."
+            )
+            return False
+
+        return True
+
+    def run(self, text: Text) -> Optional[Text]:
+        """Anonymizes the given text using the given anonymization rule list."""
+        if (
+            self.analyzer is None
+            or not self.anonymization_rule_list
+            or self.analyzer.presidio_analyzer_engine is None
+        ):
+            return text
+
+        if not self.anonymization_rule_list.rule_list:
+            return text
+
+        analyzer_results = self.analyzer.presidio_analyzer_engine.analyze(
+            text=text,
+            entities=[
+                rule.entity_name for rule in self.anonymization_rule_list.rule_list
+            ],
+            language=self.anonymization_rule_list.language,
+        )
+
+        operators = self.get_operators()
+
+        anonymized_text = self.anonymizer_engine.anonymize(
+            text=text,
+            analyzer_results=analyzer_results,
+            operators=operators,
+        )
+
+        return anonymized_text.text
+
+    @staticmethod
+    @typing.no_type_check  # faker is not typed correctly
+    def _get_supported_faker_entities() -> Dict[Text, Any]:
+        faker = Faker(["en_US", "es_ES", "it_IT"])
+
+        # Presidio entities: https://microsoft.github.io/presidio/supported_entities/
+        # Faker providers: https://faker.readthedocs.io/en/master/providers.html
+        # Unsupported entities by faker:
+        # CRYPTO, NRP, MEDICAL_LICENSE, US_BANK_NUMBER, US_DRIVER_LICENSE
+        # UK_NHS, IT_FISCAL_CODE, IT_DRIVER_LICENSE, IT_PASSPORT, IT_IDENTITY_CARD
+        # SG_NRIC_FIN, AU_ABN, AU_ACN, AU_TFN, AU_MEDICARE
+        supported_entities = {
+            "PERSON": lambda value: faker["en_US"].name(),
+            "PHONE_NUMBER": lambda value: faker["en_US"].phone_number(),
+            "EMAIL_ADDRESS": lambda value: faker["en_US"].ascii_email(),
+            "CREDIT_CARD": lambda value: faker["en_US"].credit_card_number(),
+            "IBAN_CODE": lambda value: faker["en_US"].iban(),
+            "DATE_TIME": lambda value: faker["en_US"].date(),
+            "IP_ADDRESS": lambda value: faker["en_US"].ipv4(),
+            "URL": lambda value: faker["en_US"].url(),
+            # This faker method returns a tuple of
+            # (latitude, longitude, place name, country code, timezone)
+            "LOCATION": lambda value: faker["en_US"].location_on_land()[2],
+            # USA
+            "US_ITIN": lambda value: faker["en_US"].itin(),
+            "US_PASSPORT": lambda value: faker["en_US"].passport_number(),
+            "US_SSN": lambda value: faker["en_US"].ssn(),
+            # Spain
+            "ES_NIF": lambda value: faker["es_ES"].nif(),
+            # Italy
+            "IT_VAT_CODE": lambda value: faker["it_IT"].company_vat(),
+        }
+
+        return supported_entities
+
+    @staticmethod
+    def _mask_anonymize(value: Text) -> Text:
+        return "*" * len(value)
+
+    def get_substitution_func(self, rule: AnonymizationRule) -> Optional[Any]:
+        """Returns a function that anonymizes the given text.
+
+        Args:
+            rule: The anonymization rule to use.
+
+        Returns:
+            A function that anonymizes the given text.
+        """
+        if rule.substitution == "faker":
+            supported_faker_entities = self._get_supported_faker_entities()
+
+            if rule.entity_name not in supported_faker_entities:
+                logger.debug(
+                    f"Unsupported faker entity: {rule.entity_name}. "
+                    f"Supported entities are: {supported_faker_entities.keys()}"
+                    f"Using mask anonymization instead."
+                )
+                func = self._mask_anonymize
+            else:
+                func = supported_faker_entities.get(rule.entity_name)
+        elif rule.substitution == "mask":
+            func = self._mask_anonymize
+        else:
+            raise RasaException(f"Unknown substitution type: {rule.substitution}")
+
+        return func
+
+    def get_operators(self) -> Dict[Text, OperatorConfig]:
+        """Returns a dictionary of operators for the given anonymization rule list."""
+        operators = {}
+
+        for rule in self.anonymization_rule_list.rule_list:
+            if rule.substitution == "text":
+                operators[rule.entity_name] = OperatorConfig(
+                    "replace", {"new_value": rule.value}
+                )
+            else:
+                operators[rule.entity_name] = OperatorConfig(
+                    "custom", {"lambda": self.get_substitution_func(rule)}
+                )
+
+        return operators

rasa/anonymization/anonymization_rule_orchestrator.py

@@ -0,0 +1,120 @@
+import copy
+import json
+import logging
+from typing import Any, Dict, List, Optional, Text
+
+from rasa.core.brokers.broker import EventBroker
+from rasa.core.brokers.kafka import KafkaEventBroker
+
+from rasa.anonymization.anonymization_rule_executor import (
+    AnonymizationRuleExecutor,
+    AnonymizationRuleList,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AnonymizationRuleOrchestrator:
+    """Orchestrates the execution of anonymization rules."""
+
+    def __init__(
+        self,
+        event_broker: Optional[EventBroker],
+        anonymization_rule_list: AnonymizationRuleList,
+    ):
+        """Initializes the orchestrator."""
+        self.event_broker = (
+            event_broker if isinstance(event_broker, KafkaEventBroker) else None
+        )
+        self.anonymization_rule_executor = AnonymizationRuleExecutor(
+            anonymization_rule_list
+        )
+
+    def anonymize_event(self, event: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Anonymizes one of the supported event types.
+
+        The supported event types: user, bot, slot, entities.
+        """
+        event_copy = copy.deepcopy(event)
+        event_type = event["event"]
+
+        if event_type == "user" or event_type == "bot":
+            if event["text"] is None:
+                return event_copy
+
+            anonymized_text = self.anonymization_rule_executor.run(event["text"])
+            event_copy["text"] = anonymized_text
+
+            if "parse_data" in event_copy:
+                event_copy["parse_data"]["text"] = anonymized_text
+
+                entities = event_copy["parse_data"]["entities"]
+
+                if entities:
+                    anonymized_entities = self._anonymize_entities(entities)
+                    event_copy["parse_data"]["entities"] = anonymized_entities
+
+        elif event_type == "slot":
+            slot_value = event["value"]
+
+            if slot_value is None:
+                return event_copy
+            elif not isinstance(slot_value, str):
+                slot_value = json.dumps(slot_value)
+
+            anonymized_value = self.anonymization_rule_executor.run(slot_value)
+            event_copy["value"] = anonymized_value
+
+        elif event_type == "entities":
+            entities = event_copy["entities"]
+            anonymized_entities = self._anonymize_entities(entities)
+
+            event_copy["entities"] = anonymized_entities
+        else:
+            logger.debug(f"Unsupported event type for anonymization: {event_type}.")
+
+        return event_copy
+
+    def publish_event(
+        self, anonymized_event: Dict[Text, Any], is_anonymized: bool
+    ) -> None:
+        """Publishes the anonymized event to the event broker."""
+        if self.event_broker is None:
+            return None
+
+        # this assumes that the event broker's topic attribute
+        # is set to the correct anonymization topic
+        if is_anonymized:
+            event_type = anonymized_event["event"]
+            sender_id = anonymized_event["sender_id"]
+            logger.debug(
+                f"Publishing event of type '{event_type}' from "
+                f"sender ID '{sender_id}' to "
+                f"Kafka topic '{self.event_broker.topic}'. "
+                f"The anonymization pipeline used rule named "
+                f"'{self.anonymization_rule_executor.anonymization_rule_list.id}'."
+            )
+
+        self.event_broker.publish(anonymized_event)
+
+    def _anonymize_entities(
+        self, entities: List[Dict[Text, Any]]
+    ) -> List[Dict[Text, Any]]:
+        """Anonymizes entities."""
+        anonymized_entities = []
+
+        for entity in entities:
+            entity_value = entity.get("value")
+
+            if entity_value is not None:
+                value = self.anonymization_rule_executor.run(str(entity_value))
+                entity["value"] = value
+                anonymized_entities.append(entity)
+
+        return anonymized_entities
+
+    def anonymize_log_message(self, message: Text) -> Any:
+        """Anonymizes log messages."""
+        if self.event_broker:
+            return None
+        return self.anonymization_rule_executor.run(message)

rasa/anonymization/schemas/config.yml

@@ -0,0 +1,47 @@
+allowempty: True
+mapping:
+  anonymization:
+    type: "map"
+    mapping:
+      rule_lists:
+        type: "seq"
+        matching: "any"
+        required: True
+        sequence:
+        - type: "map"
+          mapping:
+            id:
+              type: "str"
+              required: True
+            rules:
+              type: "seq"
+              required: True
+              matching: "any"
+              sequence:
+              - type: "map"
+                mapping:
+                  entity:
+                    type: "str"
+                    required: True
+                  substitution:
+                    type: "str"
+                    enum: ["text", "mask", "faker"]
+                    required: False
+                  value:
+                    type: "str"
+                    required: False
+
+      metadata:
+        type: "map"
+        required: True
+        mapping:
+          language:
+            type: "str"
+            required: True
+          model_name:
+            type: "any"
+            required: True
+          model_provider:
+            type: "str"
+            required: True
+            enum: ["spacy", "stanza", "transformers"]

rasa/anonymization/utils.py

@@ -0,0 +1,118 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Text
+
+from rasa.shared.exceptions import RasaException, YamlException
+from rasa.shared.utils.yaml import (
+    read_config_file,
+    read_yaml_file,
+    validate_yaml_content_using_schema,
+)
+
+logger = logging.getLogger(__name__)
+SCHEMA_FILE = Path(__file__).parent / "schemas" / "config.yml"
+
+
+def read_endpoint_config(
+    filename: Optional[Text], endpoint_type: Text
+) -> Optional[Dict[Text, Any]]:
+    """Reads a configuration file from disk and extract one config."""
+    if not filename:
+        return None
+
+    try:
+        content = read_config_file(filename, reader_type=["safe", "rt"])
+
+        if content.get(endpoint_type) is None:
+            return None
+
+        return content
+    except FileNotFoundError:
+        logger.error(
+            "Failed to read configuration from {}. No such file.".format(
+                os.path.abspath(filename)
+            ),
+        )
+        return None
+
+
+def extract_anonymization_traits(
+    anonymization_config: Optional[Dict[Text, Any]], endpoint_type: Text
+) -> Dict[Text, Any]:
+    """Extract anonymization traits from the anonymization config.
+
+    Args:
+        anonymization_config: The anonymization config.
+        endpoint_type: The endpoint type.
+
+    Returns:
+        A dictionary containing the anonymization traits.
+    """
+    if not anonymization_config:
+        return {"enabled": False}
+
+    anonymization_value = anonymization_config.get(endpoint_type)
+    if not anonymization_value:
+        logger.debug("Anonymization config found but nothing is defined.")
+        return {"enabled": False}
+
+    try:
+        validate_anonymization_yaml(anonymization_config)
+    except RasaException as exception:
+        logger.debug(exception)
+        return {"enabled": False}
+
+    rule_lists = anonymization_value.get("rule_lists", [])
+
+    traits = {
+        "enabled": True,
+        "metadata": anonymization_value.get("metadata", {}),
+        "number_of_rule_lists": len(rule_lists),
+        "number_of_rules": 0,
+        "substitutions": {
+            "mask": 0,
+            "faker": 0,
+            "text": 0,
+            "not_defined": 0,
+        },
+        "entities": set(),
+    }
+
+    for rule_list in rule_lists:
+        traits["number_of_rules"] += len(rule_list.get("rules", []))
+        for rule in rule_list.get("rules", []):
+            traits["substitutions"][rule.get("substitution", "not_defined")] += 1
+            traits["entities"].add(rule.get("entity"))
+
+    entities = list(traits["entities"])
+    entities.sort()
+    traits["entities"] = entities
+
+    return traits
+
+
+def validate_anonymization_yaml(yaml_content: Dict[Text, Any]) -> None:
+    """Checks if the yaml_content adheres to the anonymization rule schema.
+
+    If the yaml_content is not in the right format, an exception will be raised.
+    """
+    schema = read_yaml_file(SCHEMA_FILE, reader_type=("safe", "rt"))
+    try:
+        validate_yaml_content_using_schema(yaml_content, schema)
+    except YamlException as exception:
+        raise RasaException(
+            f"Invalid configuration for `anonymization_rules` : {exception}"
+        ) from exception
+
+    rule_lists = yaml_content.get("anonymization", {}).get("rule_lists", [])
+
+    rule_set = set()
+    for rule_list in rule_lists:
+        rule_id = rule_list.get("id")
+        if rule_id in rule_set:
+            raise RasaException(
+                f"Invalid configuration for `anonymization_rules` :"
+                f"Duplicate rule id: '{rule_id}' encountered in rule_lists"
+            )
+        rule_set.add(rule_id)