rasa-pro 3.9.18__py3-none-any.whl

rasa/anonymization/anonymization_pipeline.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+import copy
+import json
+import logging
+import queue
+from typing import Any, Dict, List, Optional, Text
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from rasa.core.brokers.kafka import KafkaEventBroker
+from rasa.shared.core.events import Event
+from rasa.utils.endpoints import EndpointConfig, read_endpoint_config
+
+from rasa.anonymization.anonymisation_rule_yaml_reader import (
+    AnonymizationRulesYamlReader,
+)
+from rasa.anonymization.anonymization_rule_executor import AnonymizationRuleList
+from rasa.anonymization.anonymization_rule_orchestrator import (
+    AnonymizationRuleOrchestrator,
+)
+from rasa.utils.singleton import Singleton
+
+logger = logging.getLogger(__name__)
+
+
+class AnonymizationPipeline:
+    def run(self, event: Dict[Text, Any]) -> None:
+        """Run the anonymization pipeline on the given event.
+
+        Args:
+            event: The event to anonymize
+        """
+        ...
+
+    def log_run(self, data: Any) -> Any:
+        """Anonymize the log data.
+
+        Args:
+            data: log data to anonymize
+
+        Returns:
+            Anonymized log data
+        """
+        ...
+
+    def stop(self) -> None:
+        """Stop the anonymization pipeline."""
+        ...
+
+
+class BackgroundAnonymizationPipeline(AnonymizationPipeline):
+    EVENT_QUEUE_PROCESSING_TIMEOUT_IN_SECONDS = 2.0
+
+    def __init__(self, anonymization_pipeline: AnonymizationPipeline):
+        self.anonymization_pipeline = anonymization_pipeline
+
+        # Order of the initialisation is important
+        # The event queue must be created before the scheduler
+        # The can_consume_event_queue must be set to True before the scheduler starts
+        self.event_queue: queue.Queue = queue.Queue()
+
+        # This flag is used to stop the scheduler
+        self.can_consume_from_event_queue = True
+        self.event_anonymisation_scheduler = BackgroundScheduler()
+        self.event_anonymisation_scheduler.add_job(
+            self._consumer_queue, max_instances=1
+        )
+        self.event_anonymisation_scheduler.start()
+
+    def stop(self) -> None:
+        logger.debug("Shutting down the anonymization pipeline...")
+        self.can_consume_from_event_queue = False
+        self.event_anonymisation_scheduler.shutdown()
+
+    def run(self, event: Dict[Text, Any]) -> None:
+        self.event_queue.put(event)
+
+    def log_run(self, data: Any) -> Any:
+        return self.anonymization_pipeline.log_run(data)
+
+    def _consumer_queue(self) -> None:
+        while self.can_consume_from_event_queue:
+            try:
+                # Wait for 2 seconds for an event to be added to the queue
+                # If no event is added to the queue, continue
+                # This is done to avoid the scheduler to be stuck in the while loop
+                # when we want to stop the scheduler
+                event = self.event_queue.get(
+                    timeout=self.EVENT_QUEUE_PROCESSING_TIMEOUT_IN_SECONDS
+                )
+                self.anonymization_pipeline.run(event)
+                self.event_queue.task_done()
+            except queue.Empty:
+                continue
+
+
+class SyncAnonymizationPipeline(AnonymizationPipeline):
+    """Pipeline for anonymizing events."""
+
+    def __init__(self, orchestrators: List[AnonymizationRuleOrchestrator]) -> None:
+        """Initializes the pipeline."""
+        self.orchestrators = orchestrators
+
+    def stop(self) -> None:
+        pass
+
+    def run(self, event: Dict[Text, Any]) -> None:
+        """Runs the anonymization pipeline."""
+        logger.debug("Running the anonymization pipeline for event...")
+
+        for orchestrator in self.orchestrators:
+            anonymized_event = orchestrator.anonymize_event(event)
+            is_anonymized = True if event != anonymized_event else False
+            orchestrator.publish_event(anonymized_event, is_anonymized)
+
+    def log_run(self, data: Any) -> Any:
+        """Runs the anonymization pipeline for logging."""
+        logger.debug("Running the anonymization pipeline for logs....")
+
+        anonymized_data = None
+
+        # this is to make sure that the original data is not modified
+        data_copy = copy.deepcopy(data)
+
+        for orchestrator in self.orchestrators:
+            # orchestrator for anonymizing logs has its event broker set to None
+            if isinstance(orchestrator.event_broker, KafkaEventBroker):
+                continue
+
+            if isinstance(data_copy, str):
+                anonymized_data = orchestrator.anonymize_log_message(data_copy)
+
+            elif isinstance(data_copy, list):
+                anonymized_data = [
+                    Event.from_parameters(orchestrator.anonymize_event(item.as_dict()))
+                    if isinstance(item, Event)
+                    else orchestrator.anonymize_log_message(str(item["value"]))
+                    if isinstance(item, dict) and item.get("value") is not None
+                    else item
+                    for item in data_copy
+                ]
+
+            elif isinstance(data_copy, dict) and "event" in data_copy:
+                anonymized_data = orchestrator.anonymize_event(data_copy)
+
+            elif isinstance(data_copy, dict):
+                anonymized_data = {}
+
+                for key, value in data_copy.items():
+                    try:
+                        serialized_value = json.dumps(value)
+                    except TypeError as error:
+                        logger.error(
+                            f"Failed to serialize value of type '{type(value)}' "
+                            f"for key '{key}' before anonymization. "
+                            f"Encountered error: {error}. "
+                            f"Setting value to None."
+                        )
+                        serialized_value = None
+
+                    anonymized_data[key] = (
+                        orchestrator.anonymize_log_message(serialized_value)
+                        if serialized_value is not None
+                        else None
+                    )
+
+            else:
+                logger.debug("Unsupported data type for logging anonymization.")
+
+            if anonymized_data:
+                return anonymized_data
+        return data
+
+
+class AnonymizationPipelineProvider(metaclass=Singleton):
+    """Represents a provider for anonymization pipeline."""
+
+    anonymization_pipeline: Optional[AnonymizationPipeline] = None
+
+    def register_anonymization_pipeline(self, pipeline: AnonymizationPipeline) -> None:
+        """Register an anonymization pipeline.
+
+        Args:
+            pipeline: The anonymization pipeline to register.
+        """
+        self.anonymization_pipeline = pipeline
+
+    def get_anonymization_pipeline(self) -> Optional[AnonymizationPipeline]:
+        """Get the anonymization pipeline.
+
+        Returns:
+            The anonymization pipeline.
+        """
+        return self.anonymization_pipeline
+
+
+def load_anonymization_pipeline(endpoints_file: Optional[Text]) -> None:
+    """Creates an anonymization pipeline."""
+    yaml_reader = AnonymizationRulesYamlReader(endpoints_filename=endpoints_file)
+    anonymization_rules = yaml_reader.read_anonymization_rules()
+
+    if anonymization_rules is None:
+        return None
+
+    event_broker_config = read_endpoint_config(
+        endpoints_file, endpoint_type="event_broker"
+    )
+    logging_config = read_endpoint_config(endpoints_file, endpoint_type="logger")
+
+    orchestrators = _load_orchestrators(
+        logging_config, event_broker_config, anonymization_rules
+    )
+
+    if not orchestrators:
+        return None
+
+    pipeline = SyncAnonymizationPipeline(orchestrators)
+    async_anonymization_pipeline = BackgroundAnonymizationPipeline(pipeline)
+    provider = AnonymizationPipelineProvider()
+    provider.register_anonymization_pipeline(async_anonymization_pipeline)
+
+    return None
+
+
+def create_event_broker(
+    topic_name: Text, event_broker_config: EndpointConfig
+) -> Optional[KafkaEventBroker]:
+    """Create a KafkaEventBroker object.
+
+    Returns None if the event broker config is not of type 'kafka'.
+    """
+    if event_broker_config.type != "kafka":
+        logger.warning(
+            f"Unsupported event broker config provided. "
+            f"Expected type 'kafka' but got "
+            f"'{event_broker_config.type}'. "
+            f"Setting event broker to None."
+        )
+        event_broker = None
+    else:
+        logger.debug(f"Setting topic to '{topic_name}'.")
+        event_broker_config.kwargs["topic"] = topic_name
+        event_broker = KafkaEventBroker(
+            event_broker_config.url, **event_broker_config.kwargs
+        )
+
+    return event_broker
+
+
+def _load_orchestrators(
+    logging_config: EndpointConfig,
+    event_broker_config: EndpointConfig,
+    anonymization_rules: List[AnonymizationRuleList],
+) -> List[AnonymizationRuleOrchestrator]:
+    orchestrators = []
+
+    if logging_config:
+        formatter = logging_config.kwargs.get("formatter", "")
+        logging_rule = formatter.get("anonymization_rules")
+
+        for rule_list in anonymization_rules:
+            if rule_list.id == logging_rule:
+                orchestrators.append(AnonymizationRuleOrchestrator(None, rule_list))
+
+    if event_broker_config:
+        anonymization_topics = event_broker_config.kwargs.get(
+            "anonymization_topics", []
+        )
+        visited_topics = []
+
+        for rule_list in anonymization_rules:
+            for topic in anonymization_topics:
+                topic_name = topic.get("name")
+
+                if topic_name in visited_topics:
+                    continue
+
+                if rule_list.id == topic.get("anonymization_rules"):
+                    event_broker = create_event_broker(topic_name, event_broker_config)
+
+                    orchestrators.append(
+                        AnonymizationRuleOrchestrator(event_broker, rule_list)
+                    )
+
+                    visited_topics.append(topic_name)
+    return orchestrators

rasa/anonymization/anonymization_rule_executor.py

@@ -0,0 +1,260 @@
+import logging
+import typing
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Text, Union
+
+from faker import Faker
+from presidio_analyzer import AnalyzerEngine
+from presidio_analyzer.nlp_engine import (
+    SpacyNlpEngine,
+    StanzaNlpEngine,
+    TransformersNlpEngine,
+)
+from presidio_anonymizer import AnonymizerEngine
+from presidio_anonymizer.entities import OperatorConfig
+from rasa.shared.exceptions import RasaException
+
+from rasa.utils.singleton import Singleton
+
+if typing.TYPE_CHECKING:
+    from presidio_analyzer.nlp_engine.nlp_engine import NlpEngine
+
+DEFAULT_PRESIDIO_LANG_CODE = "en"
+DEFAULT_PRESIDIO_MODEL_NAME = "en_core_web_lg"
+DEFAULT_PRESIDIO_MODEL_PROVIDER = "spacy"
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class AnonymizationRule:
+    """A rule for anonymizing a given text."""
+
+    entity_name: Text
+    substitution: Text = "mask"
+    value: Optional[Text] = None
+
+
+@dataclass
+class AnonymizationRuleList:
+    """A list of anonymization rules."""
+
+    id: Text
+    rule_list: List[AnonymizationRule]
+    language: Text = DEFAULT_PRESIDIO_LANG_CODE
+    model_provider: Text = DEFAULT_PRESIDIO_MODEL_PROVIDER
+    models: Union[Text, Dict[Text, Text]] = DEFAULT_PRESIDIO_MODEL_NAME
+
+
+class AnonymizationAnalyzer(metaclass=Singleton):
+    """Anonymization analyzer."""
+
+    presidio_analyzer_engine = None
+
+    def __init__(self, anonymization_rule_list: AnonymizationRuleList):
+        """Initialise the anonymization analyzer."""
+        if self.presidio_analyzer_engine is None:
+            self.presidio_analyzer_engine = self._get_analyzer_engine(
+                anonymization_rule_list
+            )
+
+    @staticmethod
+    def _get_analyzer_engine(
+        anonymization_rule_list: AnonymizationRuleList,
+    ) -> AnalyzerEngine:
+        """Returns an analyzer engine for all the anonymization rule lists."""
+        try:
+            nlp_engine = AnonymizationAnalyzer._build_presidio_nlp_engine(
+                anonymization_rule_list
+            )
+        except (OSError, ImportError) as exception:
+            raise RasaException(
+                "Failed to load Presidio nlp engine. "
+                "Please check that you have provided "
+                "a valid model name."
+            ) from exception
+
+        return AnalyzerEngine(
+            nlp_engine=nlp_engine,
+            supported_languages=[anonymization_rule_list.language],
+        )
+
+    @staticmethod
+    def _build_presidio_nlp_engine(
+        anonymization_rule_list: AnonymizationRuleList,
+    ) -> "NlpEngine":
+        """Creates an instance of the Presidio nlp engine."""
+        if anonymization_rule_list.model_provider == "transformers":
+            nlp_engine = TransformersNlpEngine(
+                models={
+                    anonymization_rule_list.language: anonymization_rule_list.models
+                },
+            )
+        elif anonymization_rule_list.model_provider == "stanza":
+            nlp_engine = StanzaNlpEngine(
+                models={
+                    anonymization_rule_list.language: anonymization_rule_list.models
+                },
+            )
+        else:
+            nlp_engine = SpacyNlpEngine(
+                models={
+                    anonymization_rule_list.language: anonymization_rule_list.models
+                },
+            )
+
+        return nlp_engine
+
+
+class AnonymizationRuleExecutor:
+    """Executes a given anonymization rule set on a given text."""
+
+    def __init__(self, anonymization_rule_list: AnonymizationRuleList):
+        """Initialize the anonymization rule executor."""
+        self.anonymization_rule_list = anonymization_rule_list
+
+        is_valid_rule_list = self._validate_anonymization_rule_list(
+            anonymization_rule_list
+        )
+
+        self.analyzer = (
+            AnonymizationAnalyzer(anonymization_rule_list)
+            if is_valid_rule_list
+            else None
+        )
+
+        self.anonymizer_engine = AnonymizerEngine()
+
+    @staticmethod
+    def _validate_anonymization_rule_list(
+        anonymization_rule_list: AnonymizationRuleList,
+    ) -> bool:
+        """Validates the given anonymization rule list object."""
+        if (
+            anonymization_rule_list.language != DEFAULT_PRESIDIO_LANG_CODE
+            and anonymization_rule_list.models == DEFAULT_PRESIDIO_MODEL_NAME
+        ):
+            logger.debug(
+                f"Anonymization rule list language is "
+                f"'{anonymization_rule_list.language}', "
+                f"but no specific model name was provided. "
+                f"You must specify the spaCy model name in the"
+                f"endpoints yaml file. "
+                f"Cannot proceed with anonymization."
+            )
+            return False
+
+        return True
+
+    def run(self, text: Text) -> Text:
+        """Anonymizes the given text using the given anonymization rule list."""
+        if (
+            self.analyzer is None
+            or not self.anonymization_rule_list
+            or self.analyzer.presidio_analyzer_engine is None
+        ):
+            return text
+
+        if not self.anonymization_rule_list.rule_list:
+            return text
+
+        analyzer_results = self.analyzer.presidio_analyzer_engine.analyze(
+            text=text,
+            entities=[
+                rule.entity_name for rule in self.anonymization_rule_list.rule_list
+            ],
+            language=self.anonymization_rule_list.language,
+        )
+
+        operators = self.get_operators()
+
+        anonymized_text = self.anonymizer_engine.anonymize(
+            text=text,
+            analyzer_results=analyzer_results,
+            operators=operators,
+        )
+
+        return anonymized_text.text
+
+    @staticmethod
+    @typing.no_type_check  # faker is not typed correctly
+    def _get_supported_faker_entities() -> Dict[Text, Any]:
+        faker = Faker(["en_US", "es_ES", "it_IT"])
+
+        # Presidio entities: https://microsoft.github.io/presidio/supported_entities/
+        # Faker providers: https://faker.readthedocs.io/en/master/providers.html
+        # Unsupported entities by faker:
+        # CRYPTO, NRP, MEDICAL_LICENSE, US_BANK_NUMBER, US_DRIVER_LICENSE
+        # UK_NHS, IT_FISCAL_CODE, IT_DRIVER_LICENSE, IT_PASSPORT, IT_IDENTITY_CARD
+        # SG_NRIC_FIN, AU_ABN, AU_ACN, AU_TFN, AU_MEDICARE
+        supported_entities = {
+            "PERSON": lambda value: faker["en_US"].name(),
+            "PHONE_NUMBER": lambda value: faker["en_US"].phone_number(),
+            "EMAIL_ADDRESS": lambda value: faker["en_US"].ascii_email(),
+            "CREDIT_CARD": lambda value: faker["en_US"].credit_card_number(),
+            "IBAN_CODE": lambda value: faker["en_US"].iban(),
+            "DATE_TIME": lambda value: faker["en_US"].date(),
+            "IP_ADDRESS": lambda value: faker["en_US"].ipv4(),
+            "URL": lambda value: faker["en_US"].url(),
+            # This faker method returns a tuple of
+            # (latitude, longitude, place name, country code, timezone)
+            "LOCATION": lambda value: faker["en_US"].location_on_land()[2],
+            # USA
+            "US_ITIN": lambda value: faker["en_US"].itin(),
+            "US_PASSPORT": lambda value: faker["en_US"].passport_number(),
+            "US_SSN": lambda value: faker["en_US"].ssn(),
+            # Spain
+            "ES_NIF": lambda value: faker["es_ES"].nif(),
+            # Italy
+            "IT_VAT_CODE": lambda value: faker["it_IT"].company_vat(),
+        }
+
+        return supported_entities
+
+    @staticmethod
+    def _mask_anonymize(value: Text) -> Text:
+        return "*" * len(value)
+
+    def get_substitution_func(self, rule: AnonymizationRule) -> Optional[Any]:
+        """Returns a function that anonymizes the given text.
+
+        Args:
+            rule: The anonymization rule to use.
+
+        Returns:
+            A function that anonymizes the given text.
+        """
+        if rule.substitution == "faker":
+            supported_faker_entities = self._get_supported_faker_entities()
+
+            if rule.entity_name not in supported_faker_entities:
+                logger.debug(
+                    f"Unsupported faker entity: {rule.entity_name}. "
+                    f"Supported entities are: {supported_faker_entities.keys()}"
+                    f"Using mask anonymization instead."
+                )
+                func = self._mask_anonymize
+            else:
+                func = supported_faker_entities.get(rule.entity_name)
+        elif rule.substitution == "mask":
+            func = self._mask_anonymize
+        else:
+            raise RasaException(f"Unknown substitution type: {rule.substitution}")
+
+        return func
+
+    def get_operators(self) -> Dict[Text, OperatorConfig]:
+        """Returns a dictionary of operators for the given anonymization rule list."""
+        operators = {}
+
+        for rule in self.anonymization_rule_list.rule_list:
+            if rule.substitution == "text":
+                operators[rule.entity_name] = OperatorConfig(
+                    "replace", {"new_value": rule.value}
+                )
+            else:
+                operators[rule.entity_name] = OperatorConfig(
+                    "custom", {"lambda": self.get_substitution_func(rule)}
+                )
+
+        return operators

rasa/anonymization/anonymization_rule_orchestrator.py

@@ -0,0 +1,120 @@
+import copy
+import json
+import logging
+from typing import Any, Dict, List, Optional, Text
+
+from rasa.core.brokers.broker import EventBroker
+from rasa.core.brokers.kafka import KafkaEventBroker
+
+from rasa.anonymization.anonymization_rule_executor import (
+    AnonymizationRuleExecutor,
+    AnonymizationRuleList,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AnonymizationRuleOrchestrator:
+    """Orchestrates the execution of anonymization rules."""
+
+    def __init__(
+        self,
+        event_broker: Optional[EventBroker],
+        anonymization_rule_list: AnonymizationRuleList,
+    ):
+        """Initializes the orchestrator."""
+        self.event_broker = (
+            event_broker if isinstance(event_broker, KafkaEventBroker) else None
+        )
+        self.anonymization_rule_executor = AnonymizationRuleExecutor(
+            anonymization_rule_list
+        )
+
+    def anonymize_event(self, event: Dict[Text, Any]) -> Dict[Text, Any]:
+        """Anonymizes one of the supported event types.
+
+        The supported event types: user, bot, slot, entities.
+        """
+        event_copy = copy.deepcopy(event)
+        event_type = event["event"]
+
+        if event_type == "user" or event_type == "bot":
+            if event["text"] is None:
+                return event_copy
+
+            anonymized_text = self.anonymization_rule_executor.run(event["text"])
+            event_copy["text"] = anonymized_text
+
+            if "parse_data" in event_copy:
+                event_copy["parse_data"]["text"] = anonymized_text
+
+                entities = event_copy["parse_data"]["entities"]
+
+                if entities:
+                    anonymized_entities = self._anonymize_entities(entities)
+                    event_copy["parse_data"]["entities"] = anonymized_entities
+
+        elif event_type == "slot":
+            slot_value = event["value"]
+
+            if slot_value is None:
+                return event_copy
+            elif not isinstance(slot_value, str):
+                slot_value = json.dumps(slot_value)
+
+            anonymized_value = self.anonymization_rule_executor.run(slot_value)
+            event_copy["value"] = anonymized_value
+
+        elif event_type == "entities":
+            entities = event_copy["entities"]
+            anonymized_entities = self._anonymize_entities(entities)
+
+            event_copy["entities"] = anonymized_entities
+        else:
+            logger.debug(f"Unsupported event type for anonymization: {event_type}.")
+
+        return event_copy
+
+    def publish_event(
+        self, anonymized_event: Dict[Text, Any], is_anonymized: bool
+    ) -> None:
+        """Publishes the anonymized event to the event broker."""
+        if self.event_broker is None:
+            return None
+
+        # this assumes that the event broker's topic attribute
+        # is set to the correct anonymization topic
+        if is_anonymized:
+            event_type = anonymized_event["event"]
+            sender_id = anonymized_event["sender_id"]
+            logger.debug(
+                f"Publishing event of type '{event_type}' from "
+                f"sender ID '{sender_id}' to "
+                f"Kafka topic '{self.event_broker.topic}'. "
+                f"The anonymization pipeline used rule named "
+                f"'{self.anonymization_rule_executor.anonymization_rule_list.id}'."
+            )
+
+        self.event_broker.publish(anonymized_event)
+
+    def _anonymize_entities(
+        self, entities: List[Dict[Text, Any]]
+    ) -> List[Dict[Text, Any]]:
+        """Anonymizes entities."""
+        anonymized_entities = []
+
+        for entity in entities:
+            entity_value = entity.get("value")
+
+            if entity_value is not None:
+                value = self.anonymization_rule_executor.run(str(entity_value))
+                entity["value"] = value
+                anonymized_entities.append(entity)
+
+        return anonymized_entities
+
+    def anonymize_log_message(self, message: Text) -> Any:
+        """Anonymizes log messages."""
+        if self.event_broker:
+            return None
+        return self.anonymization_rule_executor.run(message)