rasa-pro 3.13.0.dev7__py3-none-any.whl → 3.13.0.dev8__py3-none-any.whl

rasa/privacy/privacy_config.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import os
+import sys
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+import structlog
+from apscheduler.triggers.cron import CronTrigger
+from pydantic import BaseModel, ConfigDict
+
+from rasa.constants import PACKAGE_NAME
+from rasa.privacy.constants import (
+    ANONYMIZATION_KEY,
+    DELETION_KEY,
+    KEEP_LEFT_KEY,
+    KEEP_RIGHT_KEY,
+    PRIVACY_CONFIG_SCHEMA,
+    REDACTION_CHAR_KEY,
+    SLOT_KEY,
+    TRACKER_STORE_SETTINGS,
+    USER_CHAT_INACTIVITY_IN_MINUTES_ENV_VAR_NAME,
+)
+from rasa.shared.exceptions import RasaException
+from rasa.shared.utils.io import read_json_file
+from rasa.shared.utils.yaml import (
+    YamlValidationException,
+    validate_data_with_jsonschema,
+)
+
+if TYPE_CHECKING:
+    from rasa.shared.core.domain import Domain
+
+structlogger = structlog.get_logger(__name__)
+
+
+class AnonymizationType(Enum):
+    """Enum for the anonymization types."""
+
+    REDACT = "redact"
+    """Replaces the PII plaintext value with the same character
+    for the entire or partial length of the value."""
+    MASK = "mask"
+    """Replaces the PII plaintext value with the uppercase slot name
+    in square brackets, e.g. [CREDIT_CARD_NUMBER]."""
+
+
+class AnonymizationMethod(BaseModel):
+    """Class for configuring the anonymization method."""
+
+    method_type: AnonymizationType
+    """The anonymization method to be used."""
+    redaction_char: str
+    """The character to use for redaction."""
+    keep_left: Optional[int] = None
+    """The number of characters to be kept intact on the left side."""
+    keep_right: Optional[int] = None
+    """The number of characters to be kept intact on the right side."""
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> AnonymizationMethod:
+        """Create an AnonymizationMethod object from parsed data."""
+        method_type = AnonymizationType(
+            data.get("type", AnonymizationType.REDACT.value)
+        )
+        redaction_char = data.get(REDACTION_CHAR_KEY, "*")
+        keep_left = data.get(KEEP_LEFT_KEY)
+        keep_right = data.get(KEEP_RIGHT_KEY)
+
+        return cls(
+            method_type=method_type,
+            redaction_char=redaction_char,
+            keep_left=keep_left,
+            keep_right=keep_right,
+        )
+
+
+class PrivacyPolicy(BaseModel):
+    """Parent class for configuring privacy policies."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    min_after_session_end: int
+    """Minimum time in minutes after session end before the policy is executed."""
+    cron: CronTrigger
+    """Cron trigger for periodic execution of the privacy policy."""
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> PrivacyPolicy:
+        """Create an AnonymizationPolicy object from parsed data."""
+        min_after_session_end = data.get("min_after_session_end", 1)
+        validate_min_after_session_end(min_after_session_end)
+
+        cron_expression = get_cron_trigger(data.get("cron"))
+
+        return cls(
+            min_after_session_end=min_after_session_end,
+            cron=cron_expression,
+        )
+
+
+class DeletionPolicy(PrivacyPolicy):
+    """Class for configuring periodic deletion in the tracker store."""
+
+    type: str = "deletion"
+
+
+class AnonymizationPolicy(PrivacyPolicy):
+    """Class for configuring periodic anonymization in the tracker store."""
+
+    type: str = "anonymization"
+
+
+class TrackerStoreSettings(BaseModel):
+    """Class for configuring tracker store settings."""
+
+    deletion_policy: Optional[DeletionPolicy] = None
+    """The deletion policy to be used."""
+    anonymization_policy: Optional[AnonymizationPolicy] = None
+    """The anonymization policy to be used."""
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> TrackerStoreSettings:
+        """Create a TrackerStoreSettings object from parsed data."""
+        deletion_policy = data.get(DELETION_KEY)
+        anonymization_policy = data.get(ANONYMIZATION_KEY)
+
+        deletion_policy = (
+            DeletionPolicy.from_dict(deletion_policy) if deletion_policy else None
+        )
+        anonymization_policy = (
+            AnonymizationPolicy.from_dict(anonymization_policy)
+            if anonymization_policy
+            else None
+        )
+
+        validate_policies(deletion_policy, anonymization_policy)
+
+        return cls(
+            deletion_policy=deletion_policy,
+            anonymization_policy=anonymization_policy,
+        )
+
+
+class PrivacyConfig(BaseModel):
+    """Class for configuring PII management."""
+
+    anonymization_rules: Dict[str, AnonymizationMethod]
+    """"Mapping of slot names to rules for anonymizing sensitive information."""
+    tracker_store_settings: Optional[TrackerStoreSettings] = None
+    """The tracker store settings to be used for periodic jobs
+    anonymizing and deleting conversation data in the tracker store."""
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> PrivacyConfig:
+        """Create a PrivacyConfig object from parsed privacy config."""
+        # Validate the data against the schema
+        validate_privacy_config(data)
+
+        anonymization_rules = {
+            rule[SLOT_KEY]: AnonymizationMethod.from_dict(
+                rule.get(ANONYMIZATION_KEY, {})
+            )
+            for rule in data.get("rules", [])
+        }
+
+        tracker_store_settings = data.get(TRACKER_STORE_SETTINGS, {})
+        tracker_store_settings = (
+            TrackerStoreSettings.from_dict(tracker_store_settings)
+            if tracker_store_settings
+            else None
+        )
+        return cls(
+            anonymization_rules=anonymization_rules,
+            tracker_store_settings=tracker_store_settings,
+        )
+
+
+def validate_privacy_config(data: Dict[str, Any]) -> None:
+    """Validate the privacy configuration."""
+    import importlib_resources
+
+    schema_file = str(
+        importlib_resources.files(PACKAGE_NAME).joinpath(PRIVACY_CONFIG_SCHEMA)
+    )
+    schema_content = read_json_file(schema_file)
+    try:
+        validate_data_with_jsonschema(data, schema_content)
+    except YamlValidationException as exception:
+        validation_errors = (
+            [error.message for error in exception.validation_errors]
+            if exception.validation_errors
+            else []
+        )
+        exception_message = exception.message
+        structlogger.error(
+            "privacy_config.invalid_privacy_config",
+            validation_errors=validation_errors,
+            event_info=f"Invalid privacy config: {exception_message}. "
+            f"Please check the configuration file.",
+        )
+        sys.exit(1)
+
+
+def get_cron_trigger(cron_expression: str) -> CronTrigger:
+    """Validate the crontab expression."""
+    try:
+        cron = CronTrigger.from_crontab(cron_expression)
+    except Exception as exc:
+        structlogger.error(
+            "privacy_config.invalid_cron_expression",
+            cron=cron_expression,
+        )
+        raise RasaException from exc
+
+    return cron
+
+
+def validate_min_after_session_end(min_after_session_end: int) -> None:
+    """Validate the minimum time after session end."""
+    try:
+        inactivity_period = int(
+            os.getenv(USER_CHAT_INACTIVITY_IN_MINUTES_ENV_VAR_NAME, "30")
+        )
+    except (ValueError, TypeError) as exc:
+        raise RasaException(
+            f"Invalid value for {USER_CHAT_INACTIVITY_IN_MINUTES_ENV_VAR_NAME} "
+            f"env var: {exc}."
+        )
+
+    if min_after_session_end < inactivity_period:
+        raise RasaException(
+            f"Minimum time in minutes after session end must be greater than "
+            f"{USER_CHAT_INACTIVITY_IN_MINUTES_ENV_VAR_NAME} env var value."
+        )
+
+
+def validate_policies(
+    deletion_policy: Optional[DeletionPolicy],
+    anonymization_policy: Optional[AnonymizationPolicy],
+) -> None:
+    """Validate the deletion and anonymization policies' configurations."""
+    if not deletion_policy or not anonymization_policy:
+        return None
+
+    if (
+        deletion_policy.min_after_session_end
+        <= anonymization_policy.min_after_session_end
+    ):
+        raise RasaException(
+            "Minimum time in minutes after session end for deletion policy "
+            "must be greater than that of the anonymization policy."
+        )
+
+    if deletion_policy.cron.fields == anonymization_policy.cron.fields:
+        raise RasaException(
+            "Cron expressions for the deletion and anonymization policies "
+            "must be different."
+        )
+
+    return None
+
+
+def validate_sensitive_slots(sensitive_slots: List[str], domain: "Domain") -> None:
+    """Validate the sensitive slots defined in the privacy config against the domain."""
+    all_slot_names = [slot.name for slot in domain.slots]
+    all_good = True
+    for sensitive_slot in sensitive_slots:
+        if sensitive_slot not in all_slot_names:
+            structlogger.error(
+                "privacy_config.invalid_sensitive_slot",
+                sensitive_slot=sensitive_slot,
+                event_info="Sensitive slot not found in the domain.",
+            )
+            all_good = False
+
+    if not all_good:
+        raise RasaException(
+            "Sensitive slots defined in the privacy config do not match "
+            "the slots defined in the domain. Please check the slot names."
+        )

rasa/privacy/privacy_config_schema.json

@@ -0,0 +1,86 @@
+{
+  "type": "object",
+  "required": [
+    "rules"
+  ],
+  "properties": {
+    "rules": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "slot": {
+            "type": "string"
+          },
+          "anonymization": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "redact",
+                  "mask"
+                ]
+              },
+              "redaction_char": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 1
+              },
+              "keep_left": {
+                "type": "number",
+                "exclusiveMinimum": 0
+              },
+              "keep_right": {
+                "type": "number",
+                "exclusiveMinimum": 0
+              }
+            },
+            "required": [
+              "type"
+            ]
+          }
+        },
+        "required": [
+          "slot",
+          "anonymization"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "tracker_store_settings": {
+      "type": "object",
+      "properties": {
+        "deletion": {
+          "$ref": "#/$defs/trigger_settings"
+        },
+        "anonymization": {
+          "$ref": "#/$defs/trigger_settings"
+        }
+      }
+    }
+  },
+  "$defs": {
+    "trigger_settings": {
+      "type": "object",
+      "properties": {
+        "min_after_session_end": {
+          "type": "number",
+          "exclusiveMinimum": 0
+        },
+        "cron": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "dependentRequired": {
+        "min_after_session_end": [
+          "cron"
+        ],
+        "cron": [
+          "min_after_session_end"
+        ]
+      }
+    }
+  }
+}

rasa/privacy/privacy_filter.py

@@ -0,0 +1,340 @@
+import copy
+import datetime
+import json
+import os
+from typing import Any, Dict, List, Optional
+
+import structlog
+
+from rasa.privacy.constants import (
+    DEFAULT_PII_MODEL,
+    ENTITIES_KEY,
+    ENTITY_LABEL_KEY,
+    GLINER_LABELS,
+    GLINER_MODEL_PATH_ENV_VAR_NAME,
+    HUGGINGFACE_CACHE_DIR_ENV_VAR_NAME,
+    TEXT_KEY,
+    VALUE_KEY,
+)
+from rasa.privacy.privacy_config import AnonymizationMethod, AnonymizationType
+from rasa.shared.core.events import BotUttered, Event, SlotSet, UserUttered
+
+structlogger = structlog.get_logger(__name__)
+
+
+class PrivacyFilter:
+    """A class to anonymise sensitive information."""
+
+    def __init__(self, anonymization_rules: Dict[str, AnonymizationMethod]) -> None:
+        """Initialise the PrivacyFilter."""
+        self.anonymization_rules = anonymization_rules
+        self.labels = GLINER_LABELS
+        self.model = self._load_gliner_model()
+
+    def anonymize(
+        self, events: List[Event], prior_sensitive_slot_events: List[Event]
+    ) -> List[Event]:
+        """Anonymize sensitive information in the events of the current turn.
+
+        The order of priority for PII detection is:
+        - firstly, the slot-based approach i.e. identify any defined slots in
+        the anonymization rules that could have been set in this turn and
+        anonymise the plaintext slot values in all 3 event types
+        (UserUttered, BotUttered, SlotSet)
+        - secondly, the GLiNER model based approach i.e. identify any PII entities
+         and anonymise the text in UserUttered events or values of
+         SlotSet events that fill from_text slots.
+        """
+        anonymized_events: List[Event] = []
+        anonymized_slots = self._anonymize_sensitive_slots(
+            (events + prior_sensitive_slot_events)
+        )
+
+        for event in events:
+            anonymized_event = self._anonymize_event(event, anonymized_slots)
+            anonymized_events.append(anonymized_event)
+
+        return anonymized_events
+
+    @staticmethod
+    def _load_gliner_model() -> Optional[Any]:
+        """Load the GLiNER model for PII detection."""
+        local_model_path = os.getenv(GLINER_MODEL_PATH_ENV_VAR_NAME)
+        cache_dir = os.getenv(HUGGINGFACE_CACHE_DIR_ENV_VAR_NAME)
+        model_path = local_model_path or DEFAULT_PII_MODEL
+
+        try:
+            from gliner import GLiNER
+
+            return GLiNER.from_pretrained(
+                model_path,
+                cache_dir=cache_dir,
+            )
+        except ImportError:
+            structlogger.warning(
+                "rasa.privacy.privacy_filter.gliner_import_error",
+                event_info="Optional GLiNER library is not installed. "
+                "Please install it if you wish to use additional "
+                "PII detection to the slot based approach.",
+            )
+            return None
+
+    def _anonymize_sensitive_slots(self, events: List[Event]) -> Dict[str, SlotSet]:
+        """Identify and anonymize sensitive slot events.
+
+        Returns a dictionary where the keys represent a concatenation of the slot key
+        and its original value, and the values are the anonymized SlotSet events.
+        """
+        sensitive_slots = self._find_sensitive_slots(events)
+
+        if not sensitive_slots:
+            structlogger.debug("rasa.privacy.privacy_filter.no_sensitive_slots_found")
+            return {}
+
+        anonymized_slots = {}
+        for slot in sensitive_slots:
+            slot_value = (
+                slot.value if isinstance(slot.value, str) else json.dumps(slot.value)
+            )
+            anonymized_slots[f"{slot.key}:{slot_value}"] = (
+                self._anonymize_sensitive_slot_event(slot)
+            )
+
+        return anonymized_slots
+
+    def _find_sensitive_slots(self, processed_events: List[Event]) -> List[SlotSet]:
+        """Find all slot events that contain sensitive information.
+
+        These sensitive slots are defined in the anonymization rules and
+        have a non-empty value.
+        """
+        return [
+            copy.deepcopy(slot_event)
+            for slot_event in processed_events
+            if isinstance(slot_event, SlotSet)
+            and slot_event.key in self.anonymization_rules
+            and bool(slot_event.value)
+        ]
+
+    def _anonymize_sensitive_slot_event(
+        self,
+        slot_event: SlotSet,
+    ) -> SlotSet:
+        """Anonymize the sensitive slot event if it contains sensitive information.
+
+        A sensitive slot event is defined as a SlotSet event that has a key
+        in the anonymization rules and a non-empty value.
+        """
+        slot_value = slot_event.value
+        if not bool(slot_value):
+            return slot_event
+
+        anonymized_value = self._anonymize_value(slot_event)
+        slot_event.value = anonymized_value
+
+        return slot_event
+
+    def _anonymize_event(
+        self, event: Event, anonymized_slots: Dict[str, SlotSet]
+    ) -> Event:
+        if isinstance(event, SlotSet):
+            return self._anonymize_slot_event(event, anonymized_slots)
+        elif isinstance(event, UserUttered):
+            return self._anonymize_user_event(event, anonymized_slots)
+        elif isinstance(event, BotUttered):
+            return self._anonymize_bot_event(event, anonymized_slots)
+        else:
+            return event
+
+    def _anonymize_slot_event(
+        self,
+        event: SlotSet,
+        anonymized_slots: Dict[str, SlotSet],
+    ) -> SlotSet:
+        """Anonymize the slot event if it contains sensitive information."""
+        event_value = (
+            event.value if isinstance(event.value, str) else json.dumps(event.value)
+        )
+        # obtain the anonymized slot event, otherwise return the original event
+        slot_event = anonymized_slots.get(f"{event.key}:{event_value}", event)
+
+        # apply the edge case anonymization
+        slot_value = (
+            slot_event.value
+            if isinstance(slot_event.value, str)
+            else json.dumps(slot_event.value)
+        )
+        anonymized_value = self._anonymize_edge_cases(slot_value, anonymized_slots)
+
+        slot_event.value = (
+            anonymized_value
+            if isinstance(slot_event.value, str)
+            else json.loads(anonymized_value)
+        )
+        slot_event.anonymized_at = datetime.datetime.now(datetime.timezone.utc)
+        return slot_event
+
+    def _anonymize_user_event(
+        self,
+        user_event: UserUttered,
+        anonymized_slots: Dict[str, SlotSet],
+    ) -> UserUttered:
+        """Anonymize the user event if it contains sensitive information."""
+        if not user_event.text:
+            structlogger.debug(
+                "rasa.privacy.privacy_filter.user_event_no_text",
+            )
+            return user_event
+
+        original_parse_data: Dict[str, Any] = (
+            copy.deepcopy(user_event.parse_data) if user_event.parse_data else {}
+        )
+        anonymized_parse_data: Dict[str, Any] = {}
+
+        for key, slot in anonymized_slots.items():
+            original_slot_value = key.split(":", 1)[1]
+            anonymized_text = user_event.text.replace(original_slot_value, slot.value)
+            user_event.text = anonymized_text
+
+            anonymized_parse_data[TEXT_KEY] = anonymized_text
+            for entity in original_parse_data.get(ENTITIES_KEY, []):
+                entity_value = entity[VALUE_KEY]
+                if entity_value == original_slot_value:
+                    anonymized_entities: List[Dict[str, Any]] = (
+                        anonymized_parse_data.get(ENTITIES_KEY, [])
+                    )
+                    anonymized_entities.append({**entity, VALUE_KEY: slot.value})
+                    anonymized_parse_data[ENTITIES_KEY] = anonymized_entities
+
+        user_event.parse_data = anonymized_parse_data  # type: ignore[assignment]
+        user_event.text = self._anonymize_edge_cases(user_event.text, anonymized_slots)
+        # cover the edge case anonymization for the parse data text field
+        parse_data_text = user_event.parse_data.get(TEXT_KEY, "")
+        user_event.parse_data[TEXT_KEY] = self._anonymize_edge_cases(  # type: ignore[literal-required]
+            parse_data_text, anonymized_slots
+        )
+
+        user_event.anonymized_at = datetime.datetime.now(datetime.timezone.utc)
+
+        return user_event
+
+    def _anonymize_bot_event(
+        self,
+        bot_event: BotUttered,
+        anonymized_slots: Dict[str, SlotSet],
+    ) -> BotUttered:
+        """Anonymize the bot event if it contains sensitive information."""
+        if not bot_event.text:
+            structlogger.debug(
+                "rasa.privacy.privacy_filter.bot_event_no_text",
+            )
+            return bot_event
+
+        for key, slot in anonymized_slots.items():
+            original_slot_value = key.split(":", 1)[1]
+            anonymized_text = bot_event.text.replace(original_slot_value, slot.value)
+            bot_event.text = anonymized_text
+
+        bot_event.text = self._anonymize_edge_cases(bot_event.text, anonymized_slots)
+        bot_event.anonymized_at = datetime.datetime.now(datetime.timezone.utc)
+        return bot_event
+
+    def _anonymize_value(self, slot: SlotSet) -> str:
+        """Anonymize the given slot value using the specified anonymization method."""
+        slot_name = slot.key
+        slot_value = slot.value
+        anonymization_method = self.anonymization_rules[slot_name]
+
+        if anonymization_method.method_type == AnonymizationType.REDACT:
+            return self._redact(slot_value, anonymization_method)
+
+        if anonymization_method.method_type == AnonymizationType.MASK:
+            return self._mask(slot_name)
+
+        # we won't reach this case, because the json schema specifies
+        # the allowed methods, this is to satisfy the type checker
+        return ""
+
+    @staticmethod
+    def _redact(slot_value: Any, anonymization_method: AnonymizationMethod) -> str:
+        """Redact the given slot value using the specified anonymization method."""
+        if anonymization_method.keep_left is not None:
+            left_part = slot_value[: anonymization_method.keep_left]
+        else:
+            left_part = ""
+
+        if anonymization_method.keep_right is not None:
+            right_part = slot_value[-anonymization_method.keep_right :]
+        else:
+            right_part = ""
+
+        return (
+            left_part
+            + anonymization_method.redaction_char
+            * (len(slot_value) - len(left_part) - len(right_part))
+            + right_part
+        )
+
+    @staticmethod
+    def _mask(slot_name: str) -> str:
+        """Mask the given slot value using the slot name."""
+        return f"[{slot_name.upper()}]"
+
+    @staticmethod
+    def _strip_square_brackets(string: str) -> str:
+        """Strip square brackets from the start and end of the string if present."""
+        if len(string) >= 2 and string[0] == "[" and string[-1] == "]":
+            return string[1:-1]
+        return string
+
+    def _anonymize_edge_cases(
+        self, text: str, anonymized_slots: Dict[str, SlotSet]
+    ) -> str:
+        """Anonymize edge cases in the text using GLiNER model.
+
+        This method is used to detect PII entities in the text that are not
+        covered by the slot-based anonymization rules. For example, when
+        the user message contains PII entities that are not defined as slots,
+        or when the slot is filled from a text input that could contain multiple
+        PII entities, such as a from_text slot.
+
+        This method uses the GLiNER model to predict entities in the text
+        and replaces them with masked values.
+        If the GLiNER model is not loaded, it will skip this step and return
+        the original text.
+        """
+        if self.model is None:
+            structlogger.debug(
+                "rasa.privacy.privacy_filter.gliner_model_not_loaded",
+                event_info="GLiNER model is not loaded, skipping PII detection.",
+            )
+            return text
+
+        entities = self.model.predict_entities(text, self.labels, threshold=0.85)
+
+        all_anonymized_slot_values = [
+            self._strip_square_brackets(str(slot.value))
+            for slot in anonymized_slots.values()
+        ]
+
+        for entity in entities:
+            structlogger.debug(
+                "rasa.privacy.privacy_filter.pii_entity_found",
+                entity=entity[ENTITY_LABEL_KEY],
+            )
+
+            entity_value = entity[TEXT_KEY]
+
+            if entity_value in all_anonymized_slot_values:
+                # the entity that was found is already anonymized,
+                # we shouldn't override the already anonymized value
+                # with a masked value
+                structlogger.debug(
+                    "rasa.privacy.privacy_filter.pii_entity_already_anonymized",
+                    entity=entity[ENTITY_LABEL_KEY],
+                )
+                continue
+
+            text = text.replace(entity_value, self._mask(entity[ENTITY_LABEL_KEY]))
+
+        return text