ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/api/service/privacy_request/duplication_detection.py

@@ -0,0 +1,424 @@
+from datetime import datetime, timedelta, timezone
+from typing import Optional
+from uuid import UUID
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.models.privacy_request.duplicate_group import (
+    DuplicateGroup,
+    generate_rule_version,
+)
+from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import ActionType
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+    GroupOperator,
+    Operator,
+)
+from fides.api.task.conditional_dependencies.sql_translator import (
+    SQLConditionTranslator,
+)
+from fides.config.config_proxy import ConfigProxy
+from fides.config.duplicate_detection_settings import DuplicateDetectionSettings
+
+ACTIONED_REQUEST_STATUSES = [
+    PrivacyRequestStatus.approved,
+    PrivacyRequestStatus.in_processing,
+    PrivacyRequestStatus.requires_manual_finalization,
+    PrivacyRequestStatus.requires_input,
+    PrivacyRequestStatus.paused,
+    PrivacyRequestStatus.awaiting_email_send,
+    PrivacyRequestStatus.error,
+]
+
+
+class DuplicateDetectionService:
+    def __init__(self, db: Session):
+        self.db = db
+        self._config = ConfigProxy(db).privacy_request_duplicate_detection
+
+    def is_enabled(self) -> bool:
+        return self._config.enabled
+
+    def _create_identity_conditions(
+        self, current_request: PrivacyRequest
+    ) -> list[Condition]:
+        """Creates conditions for matching identity fields.
+
+        For identity field matching using the EAV pattern in ProvidedIdentity, we need to match both field_name
+        and hashed_value. This function creates the required nested conditions for each identity field.
+        Also adds a condition for the policy_id to ensure that we are only matching requests for the same policy.
+        """
+        conditions: list[Condition] = []
+        current_identities: dict[str, str] = {
+            pi.field_name: pi.hashed_value
+            for pi in current_request.provided_identities  # type: ignore [attr-defined]
+            if pi.field_name in self._config.match_identity_fields
+        }
+        if len(current_identities) != len(self._config.match_identity_fields):
+            missing_fields = [
+                field
+                for field in self._config.match_identity_fields
+                if field not in current_identities.keys()
+            ]
+            logger.debug(
+                f"Some identity fields were not found in the current request: {missing_fields}"
+            )
+            return []
+
+        for field_name, hashed_value in current_identities.items():
+            identity_condition = ConditionGroup(
+                logical_operator=GroupOperator.and_,
+                conditions=[
+                    ConditionLeaf(
+                        field_address="privacyrequest.provided_identities.field_name",
+                        operator=Operator.eq,
+                        value=field_name,
+                    ),
+                    ConditionLeaf(
+                        field_address="privacyrequest.provided_identities.hashed_value",
+                        operator=Operator.eq,
+                        value=hashed_value,
+                    ),
+                ],
+            )
+            conditions.append(identity_condition)
+        policy_condition = ConditionLeaf(
+            field_address="privacyrequest.policy_id",
+            operator=Operator.eq,
+            value=current_request.policy_id,
+        )
+        conditions.append(policy_condition)
+        return conditions
+
+    def _create_time_window_condition(self, time_window_days: int) -> Condition:
+        """Creates a condition for matching requests within a time window."""
+        cutoff_date = datetime.now(timezone.utc) - timedelta(days=time_window_days)
+        condition = ConditionLeaf(
+            field_address="privacyrequest.created_at",
+            operator=Operator.gte,
+            value=cutoff_date.isoformat(),
+        )
+        return condition
+
+    def create_duplicate_detection_conditions(
+        self,
+        current_request: PrivacyRequest,
+    ) -> Optional[ConditionGroup]:
+        """
+        Create conditions for duplicate detection based on configuration.
+
+        Args:
+            current_request: The current privacy request to find duplicates for
+            config: Duplicate detection configuration settings
+
+        Returns:
+            A ConditionGroup with AND operator, or None if no conditions can be created
+        """
+        if len(self._config.match_identity_fields) == 0:
+            return None
+
+        identity_conditions = self._create_identity_conditions(current_request)
+        if not identity_conditions:
+            return None  # Only proceed if we have identity conditions
+
+        time_window_condition = self._create_time_window_condition(
+            self._config.time_window_days
+        )
+
+        # Combine all conditions with AND operator
+        all_conditions: list[Condition] = [*identity_conditions, time_window_condition]
+        return ConditionGroup(
+            logical_operator=GroupOperator.and_, conditions=all_conditions
+        )
+
+    def find_duplicate_privacy_requests(
+        self,
+        current_request: PrivacyRequest,
+    ) -> list[PrivacyRequest]:
+        """
+        Find potential duplicate privacy requests based on duplicate detection configuration.
+
+        Uses the SQLConditionTranslator to build queries from conditions, which handles
+        the ProvidedIdentity relationship using SQLAlchemy's .any() method.
+
+        Args:
+            current_request: The privacy request to check for duplicates
+            config: Duplicate detection configuration settings
+
+        Returns:
+            List of PrivacyRequest objects that match the duplicate criteria,
+            does not include the current request
+        """
+        condition = self.create_duplicate_detection_conditions(current_request)
+
+        if condition is None:
+            return []
+
+        translator = SQLConditionTranslator(self.db)
+        query = translator.generate_query_from_condition(condition)
+
+        query = query.filter(PrivacyRequest.id != current_request.id).filter(
+            PrivacyRequest.deleted_at.is_(None)
+        )
+        return query.all()
+
+    def generate_dedup_key(self, request: PrivacyRequest) -> str:
+        """
+        Generate a dedup key for a request based on the duplicate detection settings.
+        """
+        current_identities: dict[str, str] = {
+            pi.field_name: pi.hashed_value
+            for pi in request.provided_identities  # type: ignore [attr-defined]
+            if pi.field_name in self._config.match_identity_fields
+        }
+        if len(current_identities) != len(self._config.match_identity_fields):
+            raise ValueError(
+                "This request does not contain the required identity fields for duplicate detection."
+            )
+        return "|".join(
+            [
+                current_identities[field]
+                for field in sorted(self._config.match_identity_fields)
+            ]
+        )
+
+    def update_duplicate_group_ids(
+        self,
+        request: PrivacyRequest,
+        duplicates: list[PrivacyRequest],
+        duplicate_group_id: UUID,
+    ) -> None:
+        """
+        Update the duplicate request group ids for a request and its duplicates.
+        Args:
+            request: The privacy request to update
+            duplicates: The list of duplicate requests to update
+            duplicate_group_id: The duplicate request group id to update
+        """
+        update_all = [request] + duplicates
+        try:
+            for privacy_request in update_all:
+                privacy_request.duplicate_request_group_id = duplicate_group_id  # type: ignore [assignment]
+        except Exception as e:
+            logger.error(f"Failed to update duplicate request group ids: {e}")
+            raise e
+
+    def mark_as_duplicate(self, request: PrivacyRequest, message: str) -> None:
+        """
+        Mark a request as a duplicate.
+        """
+        request.update(self.db, data={"status": PrivacyRequestStatus.duplicate})
+        logger.debug(message)
+        self.add_error_execution_log(request, message)
+
+    def add_error_execution_log(self, request: PrivacyRequest, message: str) -> None:
+        request.add_error_execution_log(
+            db=self.db,
+            connection_key=None,
+            dataset_name="Duplicate Request Detection",
+            collection_name=None,
+            message=message,
+            action_type=(
+                request.policy.get_action_type()  # type: ignore [arg-type]
+                if request.policy
+                else ActionType.access
+            ),
+        )
+
+    def add_success_execution_log(self, request: PrivacyRequest, message: str) -> None:
+        request.add_success_execution_log(
+            db=self.db,
+            connection_key=None,
+            dataset_name="Duplicate Request Detection",
+            collection_name=None,
+            message=message,
+            action_type=(
+                request.policy.get_action_type()  # type: ignore [arg-type]
+                if request.policy
+                else ActionType.access
+            ),
+        )
+
+    def verified_identity_cases(
+        self, request: PrivacyRequest, duplicates: list[PrivacyRequest]
+    ) -> bool:
+        """
+        Apply verified identity rules to determine if a request is a duplicate request.
+        - If this request does not have a verified identity, it may be a duplicate if another request in the group is verified.
+        - If this is the first request to be verified, it is not a duplicate request
+        - If other requests identities were verified before this request, it is a duplicate request
+        Args:
+            request: The privacy request to check
+            duplicates: The list of duplicate requests
+
+        Returns:
+            True if the request is a duplicate request, False otherwise
+        """
+        verified_in_group = [
+            duplicate for duplicate in duplicates if duplicate.identity_verified_at
+        ]
+
+        # The request identity is not verified.
+        if not request.identity_verified_at:
+            if len(verified_in_group) > 0:
+                message = f"Request {request.id} is a duplicate: it is duplicating request(s) {[duplicate.id for duplicate in verified_in_group]}."
+                self.mark_as_duplicate(request, message)
+                return True
+
+            canonical_request = min(duplicates, key=lambda x: x.created_at)  # type: ignore [arg-type, return-value]
+            canonical_request_created_at = canonical_request.created_at or datetime.now(
+                timezone.utc
+            )
+            request_created_at = request.created_at or datetime.now(timezone.utc)
+            if request_created_at < canonical_request_created_at:
+                message = f"Request {request.id} is not a duplicate: it is the first request to be created in the group."
+                logger.debug(message)
+                self.add_success_execution_log(request, message)
+                return False
+
+            message = f"Request {request.id} is a duplicate: it is duplicating request(s) ['{canonical_request.id}']."
+            self.mark_as_duplicate(request, message)
+            return True
+
+        # The request identity is verified.
+        if not verified_in_group:
+            message = f"Request {request.id} is not a duplicate: it is the first request to be verified in the group."
+            logger.debug(message)
+            for duplicate in duplicates:
+                dup_message = f"Request {duplicate.id} is a duplicate: it is duplicating request(s) ['{request.id}']."
+                self.mark_as_duplicate(duplicate, dup_message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        # If this request is the first with a verified identity, it is not a duplicate.
+        canonical_request = min(verified_in_group, key=lambda x: x.identity_verified_at)  # type: ignore [arg-type, return-value]
+        canonical_request_verified_at = (
+            canonical_request.identity_verified_at or datetime.now(timezone.utc)
+        )
+        request_verified_at = request.identity_verified_at or datetime.now(timezone.utc)
+        if request_verified_at < canonical_request_verified_at:
+            message = f"Request {request.id} is not a duplicate: it is the first request to be verified in the group."
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            for duplicate in duplicates:
+                dup_message = f"Request {duplicate.id} is a duplicate: it is duplicating request(s) ['{request.id}']."
+                self.mark_as_duplicate(duplicate, dup_message)
+            return False
+        message = f"Request {request.id} is a duplicate: it is duplicating request(s) ['{canonical_request.id}']."
+        self.mark_as_duplicate(request, message)
+        return True
+
+    # pylint: disable=too-many-return-statements
+    def is_duplicate_request(self, request: PrivacyRequest) -> bool:
+        """
+        Determine if a request is a duplicate request and assigns a duplicate request group id.
+
+        The hierarchy is:
+        1. Actioned requests: if this request duplicates an actioned request, it is a duplicate.
+        2. Verified identity requests:
+          a. if this request has a verified identity:
+            - If none of the duplicates have a verified identity, it is not a duplicate.
+            - If duplicates have verified identities, but this request is the first with a verified identity, it is not a duplicate.
+          b. if this request does not have a verified identity:
+            - If no duplicates have a verified identity, and this was the first created request, it is not a duplicate.
+        3. First created request: if this is the first created request in the group, it is not a duplicate.
+        4. If no canonical requests are found (meaning all requests are marked as duplicates), this request is not a duplicate.
+            - Could occur if configuration changes and previous requests were already marked as duplicates.
+
+        Args:
+            request: The privacy request to check
+            config: Duplicate detection configuration settings
+        Returns:
+            True if the request is a duplicate request, False otherwise
+        """
+        if request.policy.get_action_type() == ActionType.consent:
+            message = f"Consent request {request.id} is not a duplicate."
+            logger.info(message)
+            self.add_success_execution_log(request, message)
+            return False
+        duplicates = self.find_duplicate_privacy_requests(request)
+        rule_version = generate_rule_version(
+            DuplicateDetectionSettings(
+                enabled=self._config.enabled,
+                time_window_days=self._config.time_window_days,
+                match_identity_fields=self._config.match_identity_fields,
+            )
+        )
+        try:
+            dedup_key = self.generate_dedup_key(request)
+        except ValueError as e:
+            message = f"Request {request.id} is not a duplicate: {e}"
+            logger.warning(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        _, duplicate_group = DuplicateGroup.get_or_create(
+            db=self.db, data={"rule_version": rule_version, "dedup_key": dedup_key}
+        )
+        if duplicate_group is None:
+            message = f"Failed to create duplicate group for request {request.id} with dedup key {dedup_key}"
+            logger.error(message)
+            self.add_error_execution_log(request, message)
+            return False
+
+        self.update_duplicate_group_ids(request, duplicates, duplicate_group.id)  # type: ignore [arg-type]
+
+        if request.status in ACTIONED_REQUEST_STATUSES:
+            message = (
+                f"Request {request.id} is not a duplicate: it is an actioned request."
+            )
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        # if this is the only request in the group, it is not a duplicate
+        if len(duplicates) == 0:
+            message = f"Request {request.id} is not a duplicate."
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        if request.status == PrivacyRequestStatus.duplicate:
+            return True
+
+        # only compare to non-duplicate/complete requests for the following cases
+        canonical_requests = [
+            duplicate
+            for duplicate in duplicates
+            if duplicate.status
+            not in [PrivacyRequestStatus.duplicate, PrivacyRequestStatus.complete]
+        ]
+        # If no non-duplicate requests are found, this request is not a duplicate.
+        if len(canonical_requests) == 0:
+            message = f"Request {request.id} is not a duplicate."
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        # If any requests in group are actioned, this request is a duplicate.
+        actioned_in_group = [
+            duplicate
+            for duplicate in canonical_requests
+            if duplicate.status in ACTIONED_REQUEST_STATUSES
+        ]
+        if len(actioned_in_group) > 0:
+            message = f"Request {request.id} is a duplicate: it is duplicating actioned request(s) {[duplicate.id for duplicate in actioned_in_group]}."
+            self.mark_as_duplicate(request, message)
+            return True
+        # Check against verified identity rules.
+        return self.verified_identity_cases(request, canonical_requests)
+
+
+def check_for_duplicates(db: Session, privacy_request: PrivacyRequest) -> None:
+    duplicate_detection_service = DuplicateDetectionService(db)
+    if duplicate_detection_service.is_enabled():
+        logger.info(
+            "Duplicate detection is enabled. Checking if privacy request is a duplicate."
+        )
+        if duplicate_detection_service.is_duplicate_request(privacy_request):
+            logger.info("Terminating privacy request: request is a duplicate.")

fides/api/service/privacy_request/request_runner_service.py

@@ -2,10 +2,13 @@
 import time
 from copy import deepcopy
 from datetime import datetime, timedelta
-from typing import Any, Optional, Tuple
+from typing import Any, Optional
 
-import requests
+import sqlalchemy.exc
 from loguru import logger
+
+# pylint: disable=no-name-in-module
+from psycopg2.errors import InternalError_  # type: ignore[import-untyped]
 from pydantic import ValidationError as PydanticValidationError
 from sqlalchemy.orm import Query, Session
 
@@ -22,7 +25,6 @@ from fides.api.common_exceptions import (
     ValidationError,
 )
 from fides.api.db.session import get_db_session
-from fides.api.graph.config import CollectionAddress
 from fides.api.graph.graph import DatasetGraph
 from fides.api.models.attachment import Attachment, AttachmentReferenceType
 from fides.api.models.audit_log import AuditLog, AuditLogAction
@@ -53,7 +55,7 @@ from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.schemas.redis_cache import Identity
 from fides.api.schemas.storage.storage import StorageType
-from fides.api.service.connectors import FidesConnector, get_connector
+from fides.api.service.connectors import get_connector
 from fides.api.service.connectors.consent_email_connector import (
     CONSENT_EMAIL_CONNECTOR_TYPES,
 )
@@ -69,6 +71,7 @@ from fides.api.service.privacy_request.attachment_handling import (
     get_attachments_content,
     process_attachments_for_upload,
 )
+from fides.api.service.privacy_request.duplication_detection import check_for_duplicates
 from fides.api.service.storage.storage_uploader_service import upload
 from fides.api.task.filter_results import filter_data_categories
 from fides.api.task.graph_runners import access_runner, consent_runner, erasure_runner
@@ -85,11 +88,7 @@ from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
 from fides.api.util.memory_watchdog import memory_limiter
-from fides.common.api.v1.urn_registry import (
-    PRIVACY_CENTER_DSR_PACKAGE,
-    PRIVACY_REQUEST_TRANSFER_TO_PARENT,
-    V1_URL_PREFIX,
-)
+from fides.common.api.v1.urn_registry import PRIVACY_CENTER_DSR_PACKAGE
 from fides.config import CONFIG
 from fides.config.config_proxy import ConfigProxy
 
@@ -276,6 +275,11 @@ def upload_access_results(
     return download_urls
 
 
+@log_context(
+    capture_args={
+        "privacy_request_id": LoggerContextKeys.privacy_request_id,
+    }
+)
 def save_access_results(
     session: Session,
     privacy_request: PrivacyRequest,
@@ -284,12 +288,34 @@ def save_access_results(
 ) -> None:
     """Save the results we uploaded to the user for later retrieval"""
     # Save the results we uploaded to the user for later retrieval
-    privacy_request.save_filtered_access_results(session, rule_filtered_results)
     # Saving access request URL's on the privacy request in case DSR 3.0
     # exits processing before the email is sent
     privacy_request.access_result_urls = {"access_result_urls": download_urls}
     privacy_request.save(session)
 
+    # Try to save the backup results, but don't fail the DSR if this fails
+    try:
+        privacy_request.save_filtered_access_results(session, rule_filtered_results)
+        logger.info("Successfully saved backup filtered access results to database")
+    except (
+        InternalError_,  # invalid memory alloc request size 1073741824
+        sqlalchemy.exc.StatementError,  # SQL statement errors
+        # Python memory errors
+        MemoryError,  # system out of memory
+        OverflowError,  # numeric overflow during serialization
+    ) as exc:
+        logger.warning(
+            "Failed to save backup of DSR results to database after successful S3 upload. "
+            "DSR will continue processing. Error: {}",
+            str(exc),
+        )
+    except Exception as exc:
+        logger.error(
+            "Failed to save backup of DSR results to database after successful S3 upload. "
+            "DSR will continue processing. Unexpected Error: {}",
+            str(exc),
+        )
+
 
 @log_context(
     capture_args={
@@ -307,8 +333,11 @@ def upload_and_save_access_results(  # pylint: disable=R0912
 ) -> list[str]:
     """Process the data uploads after the access portion of the privacy request has completed"""
     download_urls: list[str] = []
-    # Remove manual webhook attachments from the list of attachments
-    # This is done because the manual webhook attachments are already included in the manual_data
+    # Remove manual webhook attachments and request task attachments from the list of attachments
+    # This is done because:
+    # - manual webhook attachments are already included in the manual_data
+    # - manual task submission attachments are already included in the manual_data
+    # - request task attachments (from async polling) are already embedded in the dataset results
     loaded_attachments = [
         attachment
         for attachment in privacy_request.attachments
@@ -317,6 +346,7 @@ def upload_and_save_access_results(  # pylint: disable=R0912
             in [
                 AttachmentReferenceType.access_manual_webhook,
                 AttachmentReferenceType.manual_task_submission,
+                AttachmentReferenceType.request_task,
             ]
             for ref in attachment.references
         )
@@ -417,6 +447,10 @@ def run_privacy_request(
                 logger.info("Terminating privacy request: request deleted.")
                 return
 
+            check_for_duplicates(db=session, privacy_request=privacy_request)
+            if privacy_request.status == PrivacyRequestStatus.duplicate:
+                return
+
             logger.info("Dispatching privacy request")
             privacy_request.start_processing(session)
 
@@ -922,78 +956,6 @@ def mark_paused_privacy_request_as_expired(privacy_request_id: str) -> None:
     db.close()
 
 
-def _retrieve_child_results(  # pylint: disable=R0911
-    fides_connector: Tuple[str, ConnectionConfig],
-    rule_key: str,
-    access_result: dict[str, list[Row]],
-) -> Optional[list[dict[str, Optional[list[Row]]]]]:
-    """Get child access request results to add to upload."""
-    try:
-        connector = FidesConnector(fides_connector[1])
-    except Exception as e:
-        logger.error(
-            "Error create client for child server {}: {}", fides_connector[0], e
-        )
-        return None
-
-    results = []
-
-    for key, rows in access_result.items():
-        address = CollectionAddress.from_string(key)
-        privacy_request_id = None
-        if address.dataset == fides_connector[0]:
-            if not rows:
-                logger.info("No rows found for result entry {}", key)
-                continue
-            privacy_request_id = rows[0]["id"]
-
-        if not privacy_request_id:
-            logger.error(
-                "No privacy request found for connector key {}", fides_connector[0]
-            )
-            continue
-
-        try:
-            client = connector.create_client()
-        except requests.exceptions.HTTPError as e:
-            logger.error(
-                "Error logger into to child server for privacy request {}: {}",
-                privacy_request_id,
-                e,
-            )
-            continue
-
-        try:
-            request = client.authenticated_request(
-                method="get",
-                path=f"{V1_URL_PREFIX}{PRIVACY_REQUEST_TRANSFER_TO_PARENT.format(privacy_request_id=privacy_request_id, rule_key=rule_key)}",
-                headers={"Authorization": f"Bearer {client.token}"},
-            )
-            response = client.session.send(request)
-        except requests.exceptions.HTTPError as e:
-            logger.error(
-                "Error retrieving data from child server for privacy request {}: {}",
-                privacy_request_id,
-                e,
-            )
-            continue
-
-        if response.status_code != 200:
-            logger.error(
-                "Error retrieving data from child server for privacy request {}: {}",
-                privacy_request_id,
-                response.json(),
-            )
-            continue
-
-        results.append(response.json())
-
-    if not results:
-        return None
-
-    return results
-
-
 def get_consent_email_connection_configs(db: Session) -> Query:
     """Return enabled consent email connection configs."""
     return db.query(ConnectionConfig).filter(