rasa-pro 3.13.0.dev5__py3-none-any.whl → 3.13.0.dev8__py3-none-any.whl

rasa/anonymization/__init__.py

@@ -1,2 +0,0 @@
-ENV_LOG_LEVEL_PRESIDIO = "LOG_LEVEL_PRESIDIO"
-ENV_LOG_LEVEL_FAKER = "LOG_LEVEL_FAKER"

rasa/anonymization/anonymisation_rule_yaml_reader.py

@@ -1,91 +0,0 @@
-from typing import List, Optional, Text
-
-import pycountry
-
-from rasa.anonymization.anonymization_rule_executor import (
-    AnonymizationRule,
-    AnonymizationRuleList,
-)
-from rasa.anonymization.utils import (
-    read_endpoint_config,
-    validate_anonymization_yaml,
-)
-from rasa.shared.exceptions import RasaException
-
-KEY_ANONYMIZATION_RULES = "anonymization"
-
-
-class AnonymizationRulesYamlReader:
-    """Reads anonymization rules in YAML."""
-
-    def __init__(self, endpoints_filename: Optional[Text] = None) -> None:
-        """Initializes the reader with the endpoints' filename."""
-        self.endpoints_filename = endpoints_filename
-
-    def read_anonymization_rules(self) -> List[AnonymizationRuleList]:
-        """Reads Anonymization rules from a YAML file.
-
-        Returns:
-            Parsed Anonymization rules.
-        """
-        yaml_content = read_endpoint_config(
-            self.endpoints_filename, KEY_ANONYMIZATION_RULES
-        )
-        if yaml_content is None:
-            return []
-
-        validate_anonymization_yaml(yaml_content)
-
-        anonymization_rules = []
-
-        for key, value in yaml_content.items():
-            if key == KEY_ANONYMIZATION_RULES:
-                metadata = value.get("metadata", {})
-                rule_lists = value.get("rule_lists", [])
-
-                lang_code = metadata.get("language")
-                self.validate_language(lang_code)
-
-                model_provider = metadata.get("model_provider")
-                model_name = metadata.get("model_name")
-
-                for rule in rule_lists:
-                    identifier = rule.get("id")
-                    rules = rule.get("rules", [])
-                    rule_list = []
-
-                    for item in rules:
-                        entity_name = item.get("entity")
-                        substitution = item.get("substitution", "mask")
-                        value = item.get("value")
-
-                        rule_obj = AnonymizationRule(
-                            entity_name=entity_name,
-                            substitution=substitution,
-                            value=value,
-                        )
-                        rule_list.append(rule_obj)
-                    anonymization_rule_list_obj = AnonymizationRuleList(
-                        id=identifier,
-                        rule_list=rule_list,
-                        language=lang_code,
-                        model_provider=model_provider,
-                        models=model_name,
-                    )
-
-                    anonymization_rules.append(anonymization_rule_list_obj)
-
-        return anonymization_rules
-
-    def validate_language(self, lang_code: Text) -> None:
-        """Checks if the language is a valid ISO 639-2 code."""
-        language = pycountry.languages.get(alpha_2=lang_code)
-        if language is None:
-            raise RasaException(
-                f"Provided language code '{lang_code}' is invalid. "
-                f"In order to proceed with anonymization, "
-                f"please provide a valid ISO 639-2 language code in "
-                f"the {self.endpoints_filename} file."
-            )
-
-        return None

rasa/anonymization/anonymization_pipeline.py

@@ -1,286 +0,0 @@
-from __future__ import annotations
-
-import copy
-import json
-import logging
-import queue
-from typing import Any, Dict, List, Optional, Text
-
-from apscheduler.schedulers.background import BackgroundScheduler
-
-from rasa.anonymization.anonymisation_rule_yaml_reader import (
-    AnonymizationRulesYamlReader,
-)
-from rasa.anonymization.anonymization_rule_executor import AnonymizationRuleList
-from rasa.anonymization.anonymization_rule_orchestrator import (
-    AnonymizationRuleOrchestrator,
-)
-from rasa.core.brokers.kafka import KafkaEventBroker
-from rasa.shared.core.events import Event
-from rasa.utils.endpoints import EndpointConfig, read_endpoint_config
-from rasa.utils.singleton import Singleton
-
-logger = logging.getLogger(__name__)
-
-
-class AnonymizationPipeline:
-    def run(self, event: Dict[Text, Any]) -> None:
-        """Run the anonymization pipeline on the given event.
-
-        Args:
-            event: The event to anonymize
-        """
-        ...
-
-    def log_run(self, data: Any) -> Any:
-        """Anonymize the log data.
-
-        Args:
-            data: log data to anonymize
-
-        Returns:
-            Anonymized log data
-        """
-        ...
-
-    def stop(self) -> None:
-        """Stop the anonymization pipeline."""
-        ...
-
-
-class BackgroundAnonymizationPipeline(AnonymizationPipeline):
-    EVENT_QUEUE_PROCESSING_TIMEOUT_IN_SECONDS = 2.0
-
-    def __init__(self, anonymization_pipeline: AnonymizationPipeline):
-        self.anonymization_pipeline = anonymization_pipeline
-
-        # Order of the initialisation is important
-        # The event queue must be created before the scheduler
-        # The can_consume_event_queue must be set to True before the scheduler starts
-        self.event_queue: queue.Queue = queue.Queue()
-
-        # This flag is used to stop the scheduler
-        self.can_consume_from_event_queue = True
-        self.event_anonymisation_scheduler = BackgroundScheduler()
-        self.event_anonymisation_scheduler.add_job(
-            self._consumer_queue, max_instances=1
-        )
-        self.event_anonymisation_scheduler.start()
-
-    def stop(self) -> None:
-        logger.debug("Shutting down the anonymization pipeline...")
-        self.can_consume_from_event_queue = False
-        self.event_anonymisation_scheduler.shutdown()
-
-    def run(self, event: Dict[Text, Any]) -> None:
-        self.event_queue.put(event)
-
-    def log_run(self, data: Any) -> Any:
-        return self.anonymization_pipeline.log_run(data)
-
-    def _consumer_queue(self) -> None:
-        while self.can_consume_from_event_queue:
-            try:
-                # Wait for 2 seconds for an event to be added to the queue
-                # If no event is added to the queue, continue
-                # This is done to avoid the scheduler to be stuck in the while loop
-                # when we want to stop the scheduler
-                event = self.event_queue.get(
-                    timeout=self.EVENT_QUEUE_PROCESSING_TIMEOUT_IN_SECONDS
-                )
-                self.anonymization_pipeline.run(event)
-                self.event_queue.task_done()
-            except queue.Empty:
-                continue
-
-
-class SyncAnonymizationPipeline(AnonymizationPipeline):
-    """Pipeline for anonymizing events."""
-
-    def __init__(self, orchestrators: List[AnonymizationRuleOrchestrator]) -> None:
-        """Initializes the pipeline."""
-        self.orchestrators = orchestrators
-
-    def stop(self) -> None:
-        pass
-
-    def run(self, event: Dict[Text, Any]) -> None:
-        """Runs the anonymization pipeline."""
-        logger.debug("Running the anonymization pipeline for event...")
-
-        for orchestrator in self.orchestrators:
-            anonymized_event = orchestrator.anonymize_event(event)
-            is_anonymized = True if event != anonymized_event else False
-            orchestrator.publish_event(anonymized_event, is_anonymized)
-
-    def log_run(self, data: Any) -> Any:
-        """Runs the anonymization pipeline for logging."""
-        logger.debug("Running the anonymization pipeline for logs....")
-
-        anonymized_data = None
-
-        # this is to make sure that the original data is not modified
-        data_copy = copy.deepcopy(data)
-
-        for orchestrator in self.orchestrators:
-            # orchestrator for anonymizing logs has its event broker set to None
-            if isinstance(orchestrator.event_broker, KafkaEventBroker):
-                continue
-
-            if isinstance(data_copy, str):
-                anonymized_data = orchestrator.anonymize_log_message(data_copy)
-
-            elif isinstance(data_copy, list):
-                anonymized_data = [
-                    Event.from_parameters(orchestrator.anonymize_event(item.as_dict()))
-                    if isinstance(item, Event)
-                    else orchestrator.anonymize_log_message(str(item["value"]))
-                    if isinstance(item, dict) and item.get("value") is not None
-                    else item
-                    for item in data_copy
-                ]
-
-            elif isinstance(data_copy, dict) and "event" in data_copy:
-                anonymized_data = orchestrator.anonymize_event(data_copy)
-
-            elif isinstance(data_copy, dict):
-                anonymized_data = {}
-
-                for key, value in data_copy.items():
-                    try:
-                        serialized_value = json.dumps(value)
-                    except TypeError as error:
-                        logger.error(
-                            f"Failed to serialize value of type '{type(value)}' "
-                            f"for key '{key}' before anonymization. "
-                            f"Encountered error: {error}. "
-                            f"Setting value to None."
-                        )
-                        serialized_value = None
-
-                    anonymized_data[key] = (
-                        orchestrator.anonymize_log_message(serialized_value)
-                        if serialized_value is not None
-                        else None
-                    )
-
-            else:
-                logger.debug("Unsupported data type for logging anonymization.")
-
-            if anonymized_data:
-                return anonymized_data
-        return data
-
-
-class AnonymizationPipelineProvider(metaclass=Singleton):
-    """Represents a provider for anonymization pipeline."""
-
-    anonymization_pipeline: Optional[AnonymizationPipeline] = None
-
-    def register_anonymization_pipeline(self, pipeline: AnonymizationPipeline) -> None:
-        """Register an anonymization pipeline.
-
-        Args:
-            pipeline: The anonymization pipeline to register.
-        """
-        self.anonymization_pipeline = pipeline
-
-    def get_anonymization_pipeline(self) -> Optional[AnonymizationPipeline]:
-        """Get the anonymization pipeline.
-
-        Returns:
-            The anonymization pipeline.
-        """
-        return self.anonymization_pipeline
-
-
-def load_anonymization_pipeline(endpoints_file: Optional[Text]) -> None:
-    """Creates an anonymization pipeline."""
-    yaml_reader = AnonymizationRulesYamlReader(endpoints_filename=endpoints_file)
-    anonymization_rules = yaml_reader.read_anonymization_rules()
-
-    if anonymization_rules is None:
-        return None
-
-    event_broker_config = read_endpoint_config(
-        endpoints_file, endpoint_type="event_broker"
-    )
-    logging_config = read_endpoint_config(endpoints_file, endpoint_type="logger")
-
-    orchestrators = _load_orchestrators(
-        logging_config, event_broker_config, anonymization_rules
-    )
-
-    if not orchestrators:
-        return None
-
-    pipeline = SyncAnonymizationPipeline(orchestrators)
-    async_anonymization_pipeline = BackgroundAnonymizationPipeline(pipeline)
-    provider = AnonymizationPipelineProvider()
-    provider.register_anonymization_pipeline(async_anonymization_pipeline)
-
-    return None
-
-
-def create_event_broker(
-    topic_name: Text, event_broker_config: EndpointConfig
-) -> Optional[KafkaEventBroker]:
-    """Create a KafkaEventBroker object.
-
-    Returns None if the event broker config is not of type 'kafka'.
-    """
-    if event_broker_config.type != "kafka":
-        logger.warning(
-            f"Unsupported event broker config provided. "
-            f"Expected type 'kafka' but got "
-            f"'{event_broker_config.type}'. "
-            f"Setting event broker to None."
-        )
-        event_broker = None
-    else:
-        logger.debug(f"Setting topic to '{topic_name}'.")
-        event_broker_config.kwargs["topic"] = topic_name
-        event_broker = KafkaEventBroker(
-            event_broker_config.url, **event_broker_config.kwargs
-        )
-
-    return event_broker
-
-
-def _load_orchestrators(
-    logging_config: EndpointConfig,
-    event_broker_config: EndpointConfig,
-    anonymization_rules: List[AnonymizationRuleList],
-) -> List[AnonymizationRuleOrchestrator]:
-    orchestrators = []
-
-    if logging_config:
-        formatter = logging_config.kwargs.get("formatter", "")
-        logging_rule = formatter.get("anonymization_rules")
-
-        for rule_list in anonymization_rules:
-            if rule_list.id == logging_rule:
-                orchestrators.append(AnonymizationRuleOrchestrator(None, rule_list))
-
-    if event_broker_config:
-        anonymization_topics = event_broker_config.kwargs.get(
-            "anonymization_topics", []
-        )
-        visited_topics = []
-
-        for rule_list in anonymization_rules:
-            for topic in anonymization_topics:
-                topic_name = topic.get("name")
-
-                if topic_name in visited_topics:
-                    continue
-
-                if rule_list.id == topic.get("anonymization_rules"):
-                    event_broker = create_event_broker(topic_name, event_broker_config)
-
-                    orchestrators.append(
-                        AnonymizationRuleOrchestrator(event_broker, rule_list)
-                    )
-
-                    visited_topics.append(topic_name)
-    return orchestrators

rasa/anonymization/anonymization_rule_executor.py

@@ -1,266 +0,0 @@
-import logging
-import typing
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Text, Union
-
-from faker import Faker
-
-from rasa.shared.exceptions import RasaException
-from rasa.utils.singleton import Singleton
-
-if typing.TYPE_CHECKING:
-    from presidio_analyzer import AnalyzerEngine
-    from presidio_analyzer.nlp_engine.nlp_engine import NlpEngine
-    from presidio_anonymizer.entities import OperatorConfig
-
-DEFAULT_PRESIDIO_LANG_CODE = "en"
-DEFAULT_PRESIDIO_MODEL_NAME = "en_core_web_lg"
-DEFAULT_PRESIDIO_MODEL_PROVIDER = "spacy"
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class AnonymizationRule:
-    """A rule for anonymizing a given text."""
-
-    entity_name: Text
-    substitution: Text = "mask"
-    value: Optional[Text] = None
-
-
-@dataclass
-class AnonymizationRuleList:
-    """A list of anonymization rules."""
-
-    id: Text
-    rule_list: List[AnonymizationRule]
-    language: Text = DEFAULT_PRESIDIO_LANG_CODE
-    model_provider: Text = DEFAULT_PRESIDIO_MODEL_PROVIDER
-    models: Union[Text, Dict[Text, Text]] = DEFAULT_PRESIDIO_MODEL_NAME
-
-
-class AnonymizationAnalyzer(metaclass=Singleton):
-    """Anonymization analyzer."""
-
-    presidio_analyzer_engine = None
-
-    def __init__(self, anonymization_rule_list: AnonymizationRuleList):
-        """Initialise the anonymization analyzer."""
-        if self.presidio_analyzer_engine is None:
-            self.presidio_analyzer_engine = self._get_analyzer_engine(
-                anonymization_rule_list
-            )
-
-    @staticmethod
-    def _get_analyzer_engine(
-        anonymization_rule_list: AnonymizationRuleList,
-    ) -> "AnalyzerEngine":
-        """Returns an analyzer engine for all the anonymization rule lists."""
-        from presidio_analyzer import AnalyzerEngine
-
-        try:
-            nlp_engine = AnonymizationAnalyzer._build_presidio_nlp_engine(
-                anonymization_rule_list
-            )
-        except (OSError, ImportError) as exception:
-            raise RasaException(
-                "Failed to load Presidio nlp engine. "
-                "Please check that you have provided "
-                "a valid model name."
-            ) from exception
-
-        return AnalyzerEngine(
-            nlp_engine=nlp_engine,
-            supported_languages=[anonymization_rule_list.language],
-        )
-
-    @staticmethod
-    def _build_presidio_nlp_engine(
-        anonymization_rule_list: AnonymizationRuleList,
-    ) -> "NlpEngine":
-        """Creates an instance of the Presidio nlp engine."""
-        from presidio_analyzer.nlp_engine import (
-            SpacyNlpEngine,
-            StanzaNlpEngine,
-            TransformersNlpEngine,
-        )
-
-        if anonymization_rule_list.model_provider == "transformers":
-            nlp_engine = TransformersNlpEngine(
-                models={
-                    anonymization_rule_list.language: anonymization_rule_list.models
-                },
-            )
-        elif anonymization_rule_list.model_provider == "stanza":
-            nlp_engine = StanzaNlpEngine(
-                models={
-                    anonymization_rule_list.language: anonymization_rule_list.models
-                },
-            )
-        else:
-            nlp_engine = SpacyNlpEngine(
-                models={
-                    anonymization_rule_list.language: anonymization_rule_list.models
-                },
-            )
-
-        return nlp_engine
-
-
-class AnonymizationRuleExecutor:
-    """Executes a given anonymization rule set on a given text."""
-
-    def __init__(self, anonymization_rule_list: AnonymizationRuleList):
-        """Initialize the anonymization rule executor."""
-        from presidio_anonymizer import AnonymizerEngine
-
-        self.anonymization_rule_list = anonymization_rule_list
-
-        is_valid_rule_list = self._validate_anonymization_rule_list(
-            anonymization_rule_list
-        )
-
-        self.analyzer = (
-            AnonymizationAnalyzer(anonymization_rule_list)
-            if is_valid_rule_list
-            else None
-        )
-
-        self.anonymizer_engine = AnonymizerEngine()  # type: ignore
-
-    @staticmethod
-    def _validate_anonymization_rule_list(
-        anonymization_rule_list: AnonymizationRuleList,
-    ) -> bool:
-        """Validates the given anonymization rule list object."""
-        if (
-            anonymization_rule_list.language != DEFAULT_PRESIDIO_LANG_CODE
-            and anonymization_rule_list.models == DEFAULT_PRESIDIO_MODEL_NAME
-        ):
-            logger.debug(
-                f"Anonymization rule list language is "
-                f"'{anonymization_rule_list.language}', "
-                f"but no specific model name was provided. "
-                f"You must specify the spaCy model name in the"
-                f"endpoints yaml file. "
-                f"Cannot proceed with anonymization."
-            )
-            return False
-
-        return True
-
-    def run(self, text: Text) -> Optional[Text]:
-        """Anonymizes the given text using the given anonymization rule list."""
-        if (
-            self.analyzer is None
-            or not self.anonymization_rule_list
-            or self.analyzer.presidio_analyzer_engine is None
-        ):
-            return text
-
-        if not self.anonymization_rule_list.rule_list:
-            return text
-
-        analyzer_results = self.analyzer.presidio_analyzer_engine.analyze(
-            text=text,
-            entities=[
-                rule.entity_name for rule in self.anonymization_rule_list.rule_list
-            ],
-            language=self.anonymization_rule_list.language,
-        )
-
-        operators = self.get_operators()
-
-        anonymized_text = self.anonymizer_engine.anonymize(
-            text=text,
-            analyzer_results=analyzer_results,
-            operators=operators,
-        )
-
-        return anonymized_text.text
-
-    @staticmethod
-    @typing.no_type_check  # faker is not typed correctly
-    def _get_supported_faker_entities() -> Dict[Text, Any]:
-        faker = Faker(["en_US", "es_ES", "it_IT"])
-
-        # Presidio entities: https://microsoft.github.io/presidio/supported_entities/
-        # Faker providers: https://faker.readthedocs.io/en/master/providers.html
-        # Unsupported entities by faker:
-        # CRYPTO, NRP, MEDICAL_LICENSE, US_BANK_NUMBER, US_DRIVER_LICENSE
-        # UK_NHS, IT_FISCAL_CODE, IT_DRIVER_LICENSE, IT_PASSPORT, IT_IDENTITY_CARD
-        # SG_NRIC_FIN, AU_ABN, AU_ACN, AU_TFN, AU_MEDICARE
-        supported_entities = {
-            "PERSON": lambda value: faker["en_US"].name(),
-            "PHONE_NUMBER": lambda value: faker["en_US"].phone_number(),
-            "EMAIL_ADDRESS": lambda value: faker["en_US"].ascii_email(),
-            "CREDIT_CARD": lambda value: faker["en_US"].credit_card_number(),
-            "IBAN_CODE": lambda value: faker["en_US"].iban(),
-            "DATE_TIME": lambda value: faker["en_US"].date(),
-            "IP_ADDRESS": lambda value: faker["en_US"].ipv4(),
-            "URL": lambda value: faker["en_US"].url(),
-            # This faker method returns a tuple of
-            # (latitude, longitude, place name, country code, timezone)
-            "LOCATION": lambda value: faker["en_US"].location_on_land()[2],
-            # USA
-            "US_ITIN": lambda value: faker["en_US"].itin(),
-            "US_PASSPORT": lambda value: faker["en_US"].passport_number(),
-            "US_SSN": lambda value: faker["en_US"].ssn(),
-            # Spain
-            "ES_NIF": lambda value: faker["es_ES"].nif(),
-            # Italy
-            "IT_VAT_CODE": lambda value: faker["it_IT"].company_vat(),
-        }
-
-        return supported_entities
-
-    @staticmethod
-    def _mask_anonymize(value: Text) -> Text:
-        return "*" * len(value)
-
-    def get_substitution_func(self, rule: AnonymizationRule) -> Optional[Any]:
-        """Returns a function that anonymizes the given text.
-
-        Args:
-            rule: The anonymization rule to use.
-
-        Returns:
-            A function that anonymizes the given text.
-        """
-        if rule.substitution == "faker":
-            supported_faker_entities = self._get_supported_faker_entities()
-
-            if rule.entity_name not in supported_faker_entities:
-                logger.debug(
-                    f"Unsupported faker entity: {rule.entity_name}. "
-                    f"Supported entities are: {supported_faker_entities.keys()}"
-                    f"Using mask anonymization instead."
-                )
-                func = self._mask_anonymize
-            else:
-                func = supported_faker_entities.get(rule.entity_name)
-        elif rule.substitution == "mask":
-            func = self._mask_anonymize
-        else:
-            raise RasaException(f"Unknown substitution type: {rule.substitution}")
-
-        return func
-
-    def get_operators(self) -> Dict[Text, "OperatorConfig"]:
-        """Returns a dictionary of operators for the given anonymization rule list."""
-        from presidio_anonymizer.entities import OperatorConfig
-
-        operators = {}
-
-        for rule in self.anonymization_rule_list.rule_list:
-            if rule.substitution == "text":
-                operators[rule.entity_name] = OperatorConfig(
-                    "replace", {"new_value": rule.value}
-                )
-            else:
-                operators[rule.entity_name] = OperatorConfig(
-                    "custom", {"lambda": self.get_substitution_func(rule)}
-                )
-
-        return operators