ethyca-fides 2.73.2b0__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/api/models/privacy_request/privacy_request.py

@@ -4,12 +4,12 @@ from __future__ import annotations
 
 import json
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
 
 from celery.result import AsyncResult
 from loguru import logger
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String
-from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict, MutableList
 from sqlalchemy.orm import Query, RelationshipProperty, Session, backref, relationship
@@ -69,6 +69,7 @@ from fides.api.models.pre_approval_webhook import (
     PreApprovalWebhook,
     PreApprovalWebhookReply,
 )
+from fides.api.models.privacy_request.duplicate_group import DuplicateGroup
 from fides.api.models.privacy_request.execution_log import (
     COMPLETED_EXECUTION_LOG_STATUSES,
     EXITED_EXECUTION_LOG_STATUSES,
@@ -225,7 +226,16 @@ class PrivacyRequest(
     location = Column(String, nullable=True)
 
     # Duplicate detection - group ID to link duplicates together
-    duplicate_request_group_id = Column(String, index=True, nullable=True)
+    duplicate_request_group_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("duplicate_group.id"),
+        nullable=True,
+    )
+    duplicate_group = relationship(
+        DuplicateGroup,
+        back_populates="privacy_requests",
+        uselist=False,
+    )
 
     # A PrivacyRequest can be soft deleted, so we store when it was deleted
     deleted_at = Column(DateTime(timezone=True), nullable=True)
@@ -604,7 +614,10 @@ class PrivacyRequest(
         """Verify the identification code supplied by the user
         If verified, change the status of the request to "pending", and set the datetime the identity was verified.
         """
-        if not self.status == PrivacyRequestStatus.identity_unverified:
+        if not self.status in [
+            PrivacyRequestStatus.identity_unverified,
+            PrivacyRequestStatus.duplicate,
+        ]:
             raise IdentityVerificationException(
                 f"Invalid identity verification request. Privacy request '{self.id}' status = {self.status.value}."  # type: ignore # pylint: disable=no-member
             )
@@ -682,12 +695,29 @@ class PrivacyRequest(
                 },
             )
 
-    def get_cached_identity_data(self) -> Dict[str, Any]:
-        """Retrieves any identity data pertaining to this request from the cache"""
+    def identity_prefix_cache_and_keys(self) -> Tuple[str, FidesopsRedis, List[str]]:
+        """Returns the prefix and cache keys for the identity data for this request"""
         prefix = f"id-{self.id}-identity-*"
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+        return prefix, cache, keys
+
+    def verify_cache_for_identity_data(self) -> bool:
+        """Verifies if the identity data is cached for this request"""
+        _, _, keys = self.identity_prefix_cache_and_keys()
+        return len(keys) > 0
+
+    def get_cached_identity_data(self) -> Dict[str, Any]:
+        """Retrieves any identity data pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
+        prefix, cache, keys = self.identity_prefix_cache_and_keys()
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            identity = self.get_persisted_identity()
+            self.cache_identity(identity)
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:
@@ -705,10 +735,25 @@ class PrivacyRequest(
 
     def get_cached_custom_privacy_request_fields(self) -> Dict[str, Any]:
         """Retrieves any custom fields pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
         prefix = f"id-{self.id}-custom-privacy-request-field-*"
+
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            custom_privacy_request_fields = (
+                self.get_persisted_custom_privacy_request_fields()
+            )
+            self.cache_custom_privacy_request_fields(
+                {
+                    key: CustomPrivacyRequestFieldSchema(**value)
+                    for key, value in custom_privacy_request_fields.items()
+                }
+            )
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:

fides/api/schemas/privacy_request.py

@@ -1,9 +1,10 @@
 from datetime import datetime
 from enum import Enum as EnumType
 from typing import Any, Dict, List, Optional, Type, Union
+from uuid import UUID
 
 from fideslang.validation import FidesKey
-from pydantic import ConfigDict, Field, field_serializer, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
 
 from fides.api.custom_types import SafeStr
 from fides.api.graph.config import CollectionAddress
@@ -309,6 +310,11 @@ class PrivacyRequestStatus(str, EnumType):
     duplicate = "duplicate"  # Request identified as duplicate of another request
 
 
+class IdentityValue(BaseModel):
+    label: str
+    value: Optional[str] = None
+
+
 class PrivacyRequestResponse(FidesSchema):
     """Schema to check the status of a PrivacyRequest"""
 
@@ -329,7 +335,7 @@ class PrivacyRequestResponse(FidesSchema):
     # as it is an API response field, and we don't want to reveal any more
     # about our PII structure than is explicitly stored in the cache on request
     # creation.
-    identity: Optional[Dict[str, Union[Optional[str], Dict[str, Any]]]] = None
+    identity: Optional[Dict[str, Union[Optional[str], IdentityValue]]] = None
     custom_privacy_request_fields: Optional[Dict[str, Any]] = None
     policy: PolicySchema
     action_required_details: Optional[CheckpointActionRequiredDetails] = None
@@ -343,7 +349,7 @@ class PrivacyRequestResponse(FidesSchema):
     deleted_by: Optional[str] = None
     finalized_at: Optional[datetime] = None
     finalized_by: Optional[str] = None
-    duplicate_request_group_id: Optional[str] = None
+    duplicate_request_group_id: Optional[UUID] = None
 
     model_config = ConfigDict(from_attributes=True, use_enum_values=True)
 
@@ -370,6 +376,12 @@ class DenyPrivacyRequests(ReviewPrivacyRequestIds):
     reason: Optional[SafeStr] = None
 
 
+class CancelPrivacyRequests(ReviewPrivacyRequestIds):
+    """Pass in a list of privacy request ids and cancellation reason"""
+
+    reason: Optional[SafeStr] = None
+
+
 class BulkPostPrivacyRequests(BulkResponse):
     """Schema with mixed success/failure responses for Bulk Create of PrivacyRequest responses."""
 
@@ -458,7 +470,7 @@ class PrivacyRequestFilter(FidesSchema):
     errored_gt: Optional[datetime] = None
     external_id: Optional[str] = None
     location: Optional[str] = None
-    action_type: Optional[ActionType] = None
+    action_type: Optional[Union[ActionType, List[ActionType]]] = None
     verbose: Optional[bool] = False
     include_identities: Optional[bool] = False
     include_custom_privacy_request_fields: Optional[bool] = False

fides/api/service/async_dsr/strategies/async_dsr_strategy_polling.py

@@ -551,13 +551,20 @@ class AsyncPollingStrategy(AsyncDSRStrategy):
                 self.result_request.result_path,
             )
 
-        # Ensure we have the expected polling result type
-        if not isinstance(polling_result, PollingResult):
-            raise PrivacyRequestError("Polling result must be PollingResult instance")
-
-        # Store results on the sub-request
-        self._store_sub_request_result(polling_result, sub_request, polling_task)
+        # Checks if we have a polling result, response could be empty in case there was no data to access
+        if polling_result:
+            # Ensure we have the expected polling result type
+            if not isinstance(polling_result, PollingResult):
+                raise PrivacyRequestError(
+                    "Polling result must be PollingResult instance"
+                )
 
+            # Store results on the sub-request
+            self._store_sub_request_result(polling_result, sub_request, polling_task)
+        else:
+            logger.info(
+                f"No result response for sub-request {sub_request.id} for task {polling_task.id}"
+            )
         # Mark as complete using existing method
         sub_request.update_status(self.session, ExecutionLogStatus.complete.value)
 

fides/api/service/privacy_request/duplication_detection.py

@@ -0,0 +1,424 @@
+from datetime import datetime, timedelta, timezone
+from typing import Optional
+from uuid import UUID
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.models.privacy_request.duplicate_group import (
+    DuplicateGroup,
+    generate_rule_version,
+)
+from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import ActionType
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+    GroupOperator,
+    Operator,
+)
+from fides.api.task.conditional_dependencies.sql_translator import (
+    SQLConditionTranslator,
+)
+from fides.config.config_proxy import ConfigProxy
+from fides.config.duplicate_detection_settings import DuplicateDetectionSettings
+
+ACTIONED_REQUEST_STATUSES = [
+    PrivacyRequestStatus.approved,
+    PrivacyRequestStatus.in_processing,
+    PrivacyRequestStatus.requires_manual_finalization,
+    PrivacyRequestStatus.requires_input,
+    PrivacyRequestStatus.paused,
+    PrivacyRequestStatus.awaiting_email_send,
+    PrivacyRequestStatus.error,
+]
+
+
+class DuplicateDetectionService:
+    def __init__(self, db: Session):
+        self.db = db
+        self._config = ConfigProxy(db).privacy_request_duplicate_detection
+
+    def is_enabled(self) -> bool:
+        return self._config.enabled
+
+    def _create_identity_conditions(
+        self, current_request: PrivacyRequest
+    ) -> list[Condition]:
+        """Creates conditions for matching identity fields.
+
+        For identity field matching using the EAV pattern in ProvidedIdentity, we need to match both field_name
+        and hashed_value. This function creates the required nested conditions for each identity field.
+        Also adds a condition for the policy_id to ensure that we are only matching requests for the same policy.
+        """
+        conditions: list[Condition] = []
+        current_identities: dict[str, str] = {
+            pi.field_name: pi.hashed_value
+            for pi in current_request.provided_identities  # type: ignore [attr-defined]
+            if pi.field_name in self._config.match_identity_fields
+        }
+        if len(current_identities) != len(self._config.match_identity_fields):
+            missing_fields = [
+                field
+                for field in self._config.match_identity_fields
+                if field not in current_identities.keys()
+            ]
+            logger.debug(
+                f"Some identity fields were not found in the current request: {missing_fields}"
+            )
+            return []
+
+        for field_name, hashed_value in current_identities.items():
+            identity_condition = ConditionGroup(
+                logical_operator=GroupOperator.and_,
+                conditions=[
+                    ConditionLeaf(
+                        field_address="privacyrequest.provided_identities.field_name",
+                        operator=Operator.eq,
+                        value=field_name,
+                    ),
+                    ConditionLeaf(
+                        field_address="privacyrequest.provided_identities.hashed_value",
+                        operator=Operator.eq,
+                        value=hashed_value,
+                    ),
+                ],
+            )
+            conditions.append(identity_condition)
+        policy_condition = ConditionLeaf(
+            field_address="privacyrequest.policy_id",
+            operator=Operator.eq,
+            value=current_request.policy_id,
+        )
+        conditions.append(policy_condition)
+        return conditions
+
+    def _create_time_window_condition(self, time_window_days: int) -> Condition:
+        """Creates a condition for matching requests within a time window."""
+        cutoff_date = datetime.now(timezone.utc) - timedelta(days=time_window_days)
+        condition = ConditionLeaf(
+            field_address="privacyrequest.created_at",
+            operator=Operator.gte,
+            value=cutoff_date.isoformat(),
+        )
+        return condition
+
+    def create_duplicate_detection_conditions(
+        self,
+        current_request: PrivacyRequest,
+    ) -> Optional[ConditionGroup]:
+        """
+        Create conditions for duplicate detection based on configuration.
+
+        Args:
+            current_request: The current privacy request to find duplicates for
+            config: Duplicate detection configuration settings
+
+        Returns:
+            A ConditionGroup with AND operator, or None if no conditions can be created
+        """
+        if len(self._config.match_identity_fields) == 0:
+            return None
+
+        identity_conditions = self._create_identity_conditions(current_request)
+        if not identity_conditions:
+            return None  # Only proceed if we have identity conditions
+
+        time_window_condition = self._create_time_window_condition(
+            self._config.time_window_days
+        )
+
+        # Combine all conditions with AND operator
+        all_conditions: list[Condition] = [*identity_conditions, time_window_condition]
+        return ConditionGroup(
+            logical_operator=GroupOperator.and_, conditions=all_conditions
+        )
+
+    def find_duplicate_privacy_requests(
+        self,
+        current_request: PrivacyRequest,
+    ) -> list[PrivacyRequest]:
+        """
+        Find potential duplicate privacy requests based on duplicate detection configuration.
+
+        Uses the SQLConditionTranslator to build queries from conditions, which handles
+        the ProvidedIdentity relationship using SQLAlchemy's .any() method.
+
+        Args:
+            current_request: The privacy request to check for duplicates
+            config: Duplicate detection configuration settings
+
+        Returns:
+            List of PrivacyRequest objects that match the duplicate criteria,
+            does not include the current request
+        """
+        condition = self.create_duplicate_detection_conditions(current_request)
+
+        if condition is None:
+            return []
+
+        translator = SQLConditionTranslator(self.db)
+        query = translator.generate_query_from_condition(condition)
+
+        query = query.filter(PrivacyRequest.id != current_request.id).filter(
+            PrivacyRequest.deleted_at.is_(None)
+        )
+        return query.all()
+
+    def generate_dedup_key(self, request: PrivacyRequest) -> str:
+        """
+        Generate a dedup key for a request based on the duplicate detection settings.
+        """
+        current_identities: dict[str, str] = {
+            pi.field_name: pi.hashed_value
+            for pi in request.provided_identities  # type: ignore [attr-defined]
+            if pi.field_name in self._config.match_identity_fields
+        }
+        if len(current_identities) != len(self._config.match_identity_fields):
+            raise ValueError(
+                "This request does not contain the required identity fields for duplicate detection."
+            )
+        return "|".join(
+            [
+                current_identities[field]
+                for field in sorted(self._config.match_identity_fields)
+            ]
+        )
+
+    def update_duplicate_group_ids(
+        self,
+        request: PrivacyRequest,
+        duplicates: list[PrivacyRequest],
+        duplicate_group_id: UUID,
+    ) -> None:
+        """
+        Update the duplicate request group ids for a request and its duplicates.
+        Args:
+            request: The privacy request to update
+            duplicates: The list of duplicate requests to update
+            duplicate_group_id: The duplicate request group id to update
+        """
+        update_all = [request] + duplicates
+        try:
+            for privacy_request in update_all:
+                privacy_request.duplicate_request_group_id = duplicate_group_id  # type: ignore [assignment]
+        except Exception as e:
+            logger.error(f"Failed to update duplicate request group ids: {e}")
+            raise e
+
+    def mark_as_duplicate(self, request: PrivacyRequest, message: str) -> None:
+        """
+        Mark a request as a duplicate.
+        """
+        request.update(self.db, data={"status": PrivacyRequestStatus.duplicate})
+        logger.debug(message)
+        self.add_error_execution_log(request, message)
+
+    def add_error_execution_log(self, request: PrivacyRequest, message: str) -> None:
+        request.add_error_execution_log(
+            db=self.db,
+            connection_key=None,
+            dataset_name="Duplicate Request Detection",
+            collection_name=None,
+            message=message,
+            action_type=(
+                request.policy.get_action_type()  # type: ignore [arg-type]
+                if request.policy
+                else ActionType.access
+            ),
+        )
+
+    def add_success_execution_log(self, request: PrivacyRequest, message: str) -> None:
+        request.add_success_execution_log(
+            db=self.db,
+            connection_key=None,
+            dataset_name="Duplicate Request Detection",
+            collection_name=None,
+            message=message,
+            action_type=(
+                request.policy.get_action_type()  # type: ignore [arg-type]
+                if request.policy
+                else ActionType.access
+            ),
+        )
+
+    def verified_identity_cases(
+        self, request: PrivacyRequest, duplicates: list[PrivacyRequest]
+    ) -> bool:
+        """
+        Apply verified identity rules to determine if a request is a duplicate request.
+        - If this request does not have a verified identity, it may be a duplicate if another request in the group is verified.
+        - If this is the first request to be verified, it is not a duplicate request
+        - If other requests identities were verified before this request, it is a duplicate request
+        Args:
+            request: The privacy request to check
+            duplicates: The list of duplicate requests
+
+        Returns:
+            True if the request is a duplicate request, False otherwise
+        """
+        verified_in_group = [
+            duplicate for duplicate in duplicates if duplicate.identity_verified_at
+        ]
+
+        # The request identity is not verified.
+        if not request.identity_verified_at:
+            if len(verified_in_group) > 0:
+                message = f"Request {request.id} is a duplicate: it is duplicating request(s) {[duplicate.id for duplicate in verified_in_group]}."
+                self.mark_as_duplicate(request, message)
+                return True
+
+            canonical_request = min(duplicates, key=lambda x: x.created_at)  # type: ignore [arg-type, return-value]
+            canonical_request_created_at = canonical_request.created_at or datetime.now(
+                timezone.utc
+            )
+            request_created_at = request.created_at or datetime.now(timezone.utc)
+            if request_created_at < canonical_request_created_at:
+                message = f"Request {request.id} is not a duplicate: it is the first request to be created in the group."
+                logger.debug(message)
+                self.add_success_execution_log(request, message)
+                return False
+
+            message = f"Request {request.id} is a duplicate: it is duplicating request(s) ['{canonical_request.id}']."
+            self.mark_as_duplicate(request, message)
+            return True
+
+        # The request identity is verified.
+        if not verified_in_group:
+            message = f"Request {request.id} is not a duplicate: it is the first request to be verified in the group."
+            logger.debug(message)
+            for duplicate in duplicates:
+                dup_message = f"Request {duplicate.id} is a duplicate: it is duplicating request(s) ['{request.id}']."
+                self.mark_as_duplicate(duplicate, dup_message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        # If this request is the first with a verified identity, it is not a duplicate.
+        canonical_request = min(verified_in_group, key=lambda x: x.identity_verified_at)  # type: ignore [arg-type, return-value]
+        canonical_request_verified_at = (
+            canonical_request.identity_verified_at or datetime.now(timezone.utc)
+        )
+        request_verified_at = request.identity_verified_at or datetime.now(timezone.utc)
+        if request_verified_at < canonical_request_verified_at:
+            message = f"Request {request.id} is not a duplicate: it is the first request to be verified in the group."
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            for duplicate in duplicates:
+                dup_message = f"Request {duplicate.id} is a duplicate: it is duplicating request(s) ['{request.id}']."
+                self.mark_as_duplicate(duplicate, dup_message)
+            return False
+        message = f"Request {request.id} is a duplicate: it is duplicating request(s) ['{canonical_request.id}']."
+        self.mark_as_duplicate(request, message)
+        return True
+
+    # pylint: disable=too-many-return-statements
+    def is_duplicate_request(self, request: PrivacyRequest) -> bool:
+        """
+        Determine if a request is a duplicate request and assigns a duplicate request group id.
+
+        The hierarchy is:
+        1. Actioned requests: if this request duplicates an actioned request, it is a duplicate.
+        2. Verified identity requests:
+          a. if this request has a verified identity:
+            - If none of the duplicates have a verified identity, it is not a duplicate.
+            - If duplicates have verified identities, but this request is the first with a verified identity, it is not a duplicate.
+          b. if this request does not have a verified identity:
+            - If no duplicates have a verified identity, and this was the first created request, it is not a duplicate.
+        3. First created request: if this is the first created request in the group, it is not a duplicate.
+        4. If no canonical requests are found (meaning all requests are marked as duplicates), this request is not a duplicate.
+            - Could occur if configuration changes and previous requests were already marked as duplicates.
+
+        Args:
+            request: The privacy request to check
+            config: Duplicate detection configuration settings
+        Returns:
+            True if the request is a duplicate request, False otherwise
+        """
+        if request.policy.get_action_type() == ActionType.consent:
+            message = f"Consent request {request.id} is not a duplicate."
+            logger.info(message)
+            self.add_success_execution_log(request, message)
+            return False
+        duplicates = self.find_duplicate_privacy_requests(request)
+        rule_version = generate_rule_version(
+            DuplicateDetectionSettings(
+                enabled=self._config.enabled,
+                time_window_days=self._config.time_window_days,
+                match_identity_fields=self._config.match_identity_fields,
+            )
+        )
+        try:
+            dedup_key = self.generate_dedup_key(request)
+        except ValueError as e:
+            message = f"Request {request.id} is not a duplicate: {e}"
+            logger.warning(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        _, duplicate_group = DuplicateGroup.get_or_create(
+            db=self.db, data={"rule_version": rule_version, "dedup_key": dedup_key}
+        )
+        if duplicate_group is None:
+            message = f"Failed to create duplicate group for request {request.id} with dedup key {dedup_key}"
+            logger.error(message)
+            self.add_error_execution_log(request, message)
+            return False
+
+        self.update_duplicate_group_ids(request, duplicates, duplicate_group.id)  # type: ignore [arg-type]
+
+        if request.status in ACTIONED_REQUEST_STATUSES:
+            message = (
+                f"Request {request.id} is not a duplicate: it is an actioned request."
+            )
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        # if this is the only request in the group, it is not a duplicate
+        if len(duplicates) == 0:
+            message = f"Request {request.id} is not a duplicate."
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        if request.status == PrivacyRequestStatus.duplicate:
+            return True
+
+        # only compare to non-duplicate/complete requests for the following cases
+        canonical_requests = [
+            duplicate
+            for duplicate in duplicates
+            if duplicate.status
+            not in [PrivacyRequestStatus.duplicate, PrivacyRequestStatus.complete]
+        ]
+        # If no non-duplicate requests are found, this request is not a duplicate.
+        if len(canonical_requests) == 0:
+            message = f"Request {request.id} is not a duplicate."
+            logger.debug(message)
+            self.add_success_execution_log(request, message)
+            return False
+
+        # If any requests in group are actioned, this request is a duplicate.
+        actioned_in_group = [
+            duplicate
+            for duplicate in canonical_requests
+            if duplicate.status in ACTIONED_REQUEST_STATUSES
+        ]
+        if len(actioned_in_group) > 0:
+            message = f"Request {request.id} is a duplicate: it is duplicating actioned request(s) {[duplicate.id for duplicate in actioned_in_group]}."
+            self.mark_as_duplicate(request, message)
+            return True
+        # Check against verified identity rules.
+        return self.verified_identity_cases(request, canonical_requests)
+
+
+def check_for_duplicates(db: Session, privacy_request: PrivacyRequest) -> None:
+    duplicate_detection_service = DuplicateDetectionService(db)
+    if duplicate_detection_service.is_enabled():
+        logger.info(
+            "Duplicate detection is enabled. Checking if privacy request is a duplicate."
+        )
+        if duplicate_detection_service.is_duplicate_request(privacy_request):
+            logger.info("Terminating privacy request: request is a duplicate.")

fides/api/service/privacy_request/request_runner_service.py

@@ -71,6 +71,7 @@ from fides.api.service.privacy_request.attachment_handling import (
     get_attachments_content,
     process_attachments_for_upload,
 )
+from fides.api.service.privacy_request.duplication_detection import check_for_duplicates
 from fides.api.service.storage.storage_uploader_service import upload
 from fides.api.task.filter_results import filter_data_categories
 from fides.api.task.graph_runners import access_runner, consent_runner, erasure_runner
@@ -446,6 +447,10 @@ def run_privacy_request(
                 logger.info("Terminating privacy request: request deleted.")
                 return
 
+            check_for_duplicates(db=session, privacy_request=privacy_request)
+            if privacy_request.status == PrivacyRequestStatus.duplicate:
+                return
+
             logger.info("Dispatching privacy request")
             privacy_request.start_processing(session)
 

fides/api/service/saas_request/saas_request_override_factory.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from inspect import Signature, signature
-from typing import TYPE_CHECKING, Callable, Dict, List, Union
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union
 
 from loguru import logger
 
@@ -319,7 +319,12 @@ def validate_polling_result_override_function(f: Callable) -> None:
 
     # Check return type annotation - handle both direct class and string literals
     return_annotation = sig.return_annotation
-    if return_annotation not in (PollingResult, "PollingResult"):
+    if return_annotation not in (
+        PollingResult,
+        "PollingResult",
+        Optional[PollingResult],
+        "Optional[PollingResult]",
+    ):
         raise InvalidSaaSRequestOverrideException(
             "Polling result override function must return a PollingResult"
         )

fides/api/util/logger_context_utils.py

@@ -165,7 +165,9 @@ def request_details(
 
     if response is not None:
         if CONFIG.dev_mode and response.content:
-            details[LoggerContextKeys.response.value] = response.content.decode("utf-8")
+            details[LoggerContextKeys.response.value] = response.content.decode(
+                "utf-8", errors="replace"
+            )
 
         details[LoggerContextKeys.status_code.value] = response.status_code