ethyca-fides 2.66.2b0__py2.py3-none-any.whl → 2.66.2b2__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-25T16:16:16-0700",
+ "date": "2025-07-29T12:07:43-0300",
  "dirty": false,
  "error": null,
- "full-revisionid": "3805a2cc72d6f998672d84b64fd0f2c78a64f81d",
- "version": "2.66.2b0"
+ "full-revisionid": "4ba0450a533ee5c9b65ed8ae70df97cfb829ef8d",
+ "version": "2.66.2b2"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py

@@ -0,0 +1,65 @@
+"""add finalized fields to privacy request
+
+Revision ID: a7065df4dcf1
+Revises: 7e9a2b52f498
+Create Date: 2025-07-01 14:07:36.779437
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a7065df4dcf1"
+down_revision = "7e9a2b52f498"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "privacyrequest",
+        sa.Column("finalized_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column(
+        "privacyrequest", sa.Column("finalized_by", sa.String(), nullable=True)
+    )
+    op.create_foreign_key(
+        "privacyrequest_fidesuser_id_fkey",
+        "privacyrequest",
+        "fidesuser",
+        ["finalized_by"],
+        ["id"],
+        ondelete="SET NULL",
+    )
+    op.execute(
+        "alter type privacyrequeststatus add value 'requires_manual_finalization'"
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "privacyrequest_fidesuser_id_fkey", "privacyrequest", type_="foreignkey"
+    )
+    op.drop_column("privacyrequest", "finalized_by")
+    op.drop_column("privacyrequest", "finalized_at")
+
+    op.execute(
+        "delete from privacyrequest where status in ('requires_manual_finalization')"
+    )
+
+    op.execute("alter type privacyrequeststatus rename to privacyrequeststatus_old")
+    op.execute(
+        "create type privacyrequeststatus as enum('identity_unverified', 'requires_input', 'pending', 'in_processing', 'complete', 'pending', 'error', 'paused', 'approved', 'denied', 'canceled', 'awaiting_email_send')"
+    )
+    op.execute(
+        (
+            "alter table privacyrequest alter column status type privacyrequeststatus using "
+            "status::text::privacyrequeststatus"
+        )
+    )
+    op.execute("drop type privacyrequeststatus_old")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -4,7 +4,7 @@ import csv
 import io
 import json
 from collections import defaultdict
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import (
     Annotated,
     Any,
@@ -146,6 +146,7 @@ from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_BULK_SOFT_DELETE,
     PRIVACY_REQUEST_DENY,
     PRIVACY_REQUEST_FILTERED_RESULTS,
+    PRIVACY_REQUEST_FINALIZE,
     PRIVACY_REQUEST_MANUAL_WEBHOOK_ACCESS_INPUT,
     PRIVACY_REQUEST_MANUAL_WEBHOOK_ERASURE_INPUT,
     PRIVACY_REQUEST_NOTIFICATIONS,
@@ -1862,6 +1863,48 @@ def resume_privacy_request_from_requires_input(
     return privacy_request  # type: ignore[return-value]
 
 
+@router.post(
+    PRIVACY_REQUEST_FINALIZE,
+    status_code=HTTP_200_OK,
+    response_model=PrivacyRequestResponse,
+    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_REVIEW])],
+)
+def finalize_privacy_request(
+    privacy_request_id: str,
+    *,
+    db: Session = Depends(deps.get_db),
+    client: ClientDetail = Security(
+        verify_oauth_client,
+        scopes=[PRIVACY_REQUEST_REVIEW],
+    ),
+) -> PrivacyRequestResponse:
+    """
+    Finalizes a privacy request, moving it from the 'requires_finalization' state to 'complete'.
+    This is done by re-queueing the request, which will then hit the finalization logic in the
+    request runner service. This logic marks the privacy request as complete
+    and sends out any configured messaging to the user.
+    """
+    privacy_request = get_privacy_request_or_error(db, privacy_request_id)
+
+    if privacy_request.status != PrivacyRequestStatus.requires_manual_finalization:
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail=f"Cannot manually finalize privacy request '{privacy_request_id}': status is {privacy_request.status}, not requires_manual_finalization.",
+        )
+
+    # Set finalized_by and finalized_at here, so the request runner service knows not to
+    # put the request back into the requires_finalization state.
+    privacy_request.finalized_at = datetime.now(timezone.utc)
+    privacy_request.finalized_by = client.user_id
+    privacy_request.save(db=db)
+
+    queue_privacy_request(
+        privacy_request_id=privacy_request_id,
+    )
+
+    return privacy_request  # type: ignore[return-value]
+
+
 @router.get(
     REQUEST_TASKS,
     dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_READ])],

fides/api/graph/traversal.py

@@ -115,7 +115,7 @@ class BaseTraversal:
             )
 
         # Ensure manual_task collections execute right after ROOT
-        from fides.api.task.manual.manual_task_utils import ManualTaskAddress
+        from fides.api.task.manual.manual_task_address import ManualTaskAddress
 
         for addr in self.traversal_node_dict.keys():
             if ManualTaskAddress.is_manual_task_address(addr):

fides/api/models/privacy_request/execution_log.py

@@ -24,6 +24,7 @@ EXECUTION_CHECKPOINTS = [
     CurrentStep.finalize_consent,
     CurrentStep.email_post_send,
     CurrentStep.post_webhooks,
+    CurrentStep.finalization,
 ]
 
 COMPLETED_EXECUTION_LOG_STATUSES = [

fides/api/models/privacy_request/privacy_request.py

@@ -145,6 +145,12 @@ class PrivacyRequest(
         ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
         nullable=True,
     )
+    finalized_at = Column(DateTime(timezone=True), nullable=True)
+    finalized_by = Column(
+        String,
+        ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
+        nullable=True,
+    )
     submitted_by = Column(
         String,
         ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),

fides/api/schemas/policy.py

@@ -28,6 +28,7 @@ class CurrentStep(EnumType):
     finalize_consent = "finalize_consent"
     email_post_send = "email_post_send"
     post_webhooks = "post_webhooks"
+    finalization = "finalization"
 
 
 # action types we actively support in policies/requests

fides/api/schemas/privacy_request.py

@@ -124,6 +124,8 @@ class PrivacyRequestResubmit(PrivacyRequestCreate):
     identity_verified_at: Optional[datetime] = None
     custom_privacy_request_fields_approved_at: Optional[datetime] = None
     custom_privacy_request_fields_approved_by: Optional[str] = None
+    finalized_at: Optional[datetime] = None
+    finalized_by: Optional[str] = None
 
 
 class ConsentRequestCreate(FidesSchema):
@@ -287,6 +289,7 @@ class PrivacyRequestStatus(str, EnumType):
     complete = "complete"
     paused = "paused"
     awaiting_email_send = "awaiting_email_send"
+    requires_manual_finalization = "requires_manual_finalization"
     canceled = "canceled"
     error = "error"
 
@@ -320,6 +323,8 @@ class PrivacyRequestResponse(FidesSchema):
     source: Optional[PrivacyRequestSource] = None
     deleted_at: Optional[datetime] = None
     deleted_by: Optional[str] = None
+    finalized_at: Optional[datetime] = None
+    finalized_by: Optional[str] = None
     model_config = ConfigDict(from_attributes=True, use_enum_values=True)
 
 

fides/api/service/privacy_request/request_runner_service.py

@@ -480,12 +480,21 @@ def run_privacy_request(
                     connection_configs
                 )
 
+                # If the privacy request requires manual finalization and has not yet been finalized, we exit here
+                if (
+                    privacy_request.status
+                    == PrivacyRequestStatus.requires_manual_finalization
+                    and privacy_request.finalized_at is None
+                ):
+                    return
+
                 # Access CHECKPOINT
                 if (
                     policy.get_rules_for_action(action_type=ActionType.access)
                     or policy.get_rules_for_action(action_type=ActionType.erasure)
                 ) and can_run_checkpoint(
-                    request_checkpoint=CurrentStep.access, from_checkpoint=resume_step
+                    request_checkpoint=CurrentStep.access,
+                    from_checkpoint=resume_step,
                 ):
                     privacy_request.cache_failed_checkpoint_details(CurrentStep.access)
                     access_runner(
@@ -530,7 +539,8 @@ def run_privacy_request(
                 if policy.get_rules_for_action(
                     action_type=ActionType.erasure
                 ) and can_run_checkpoint(
-                    request_checkpoint=CurrentStep.erasure, from_checkpoint=resume_step
+                    request_checkpoint=CurrentStep.erasure,
+                    from_checkpoint=resume_step,
                 ):
                     privacy_request.cache_failed_checkpoint_details(CurrentStep.erasure)
                     _verify_masking_secrets(policy, privacy_request, resume_step)
@@ -656,58 +666,111 @@ def run_privacy_request(
                 if not proceed:
                     return
 
-            legacy_request_completion_enabled = ConfigProxy(
-                session
-            ).notifications.send_request_completion_notification
-
-            action_type = (
-                MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
-                if policy.get_rules_for_action(action_type=ActionType.access)
-                else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
-            )
-
-            if message_send_enabled(
-                session,
-                privacy_request.property_id,
-                action_type,
-                legacy_request_completion_enabled,
-            ) and not policy.get_rules_for_action(action_type=ActionType.consent):
-                try:
-                    if not access_result_urls:
-                        # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
-                        # from the database because the Privacy Request would have exited
-                        # processing and lost access to the access_result_urls in memory
-                        access_result_urls = (
-                            privacy_request.access_result_urls or {}
-                        ).get("access_result_urls", [])
-                    initiate_privacy_request_completion_email(
-                        session,
-                        policy,
-                        access_result_urls,
-                        identity_data,
-                        privacy_request.property_id,
-                    )
-                except (IdentityNotFoundException, MessageDispatchException) as e:
-                    privacy_request.error_processing(db=session)
-                    # If dev mode, log traceback
-                    _log_exception(e, CONFIG.dev_mode)
-                    return
-
-            # Only mark as complete if not in error state
-            if privacy_request.status != PrivacyRequestStatus.error:
-                privacy_request.finished_processing_at = datetime.utcnow()
-                AuditLog.create(
-                    db=session,
-                    data={
-                        "user_id": "system",
-                        "privacy_request_id": privacy_request.id,
-                        "action": AuditLogAction.finished,
-                        "message": "",
-                    },
+            # Request finalization CHECKPOINT
+            if can_run_checkpoint(
+                request_checkpoint=CurrentStep.finalization,
+                from_checkpoint=resume_step,
+            ):
+                privacy_request.cache_failed_checkpoint_details(
+                    CurrentStep.finalization,
                 )
-                privacy_request.status = PrivacyRequestStatus.complete
-                logger.info("Privacy request run completed.")
-                privacy_request.save(db=session)
+                if privacy_request.status != PrivacyRequestStatus.error:
+                    erasure_rules = policy.get_rules_for_action(
+                        action_type=ActionType.erasure
+                    )
+                    if (
+                        privacy_request.finalized_at is None
+                        and erasure_rules
+                        and CONFIG.execution.erasure_request_finalization_required
+                    ):
+                        logger.info(
+                            "Marking privacy request '{}' as requires manual finalization.",
+                            privacy_request.id,
+                        )
+                        privacy_request.status = (
+                            PrivacyRequestStatus.requires_manual_finalization
+                        )
+                        privacy_request.save(db=session)
+                        return
+
+                    # Finally, mark the request as complete
+                    if privacy_request.finalized_at:
+                        logger.info(
+                            "Marking privacy request '{}' as finalized.",
+                            privacy_request.id,
+                        )
+                        privacy_request.add_success_execution_log(
+                            session,
+                            connection_key=None,
+                            dataset_name="Request finalized",
+                            collection_name=None,
+                            message=f"Request finalized for privacy request: {privacy_request.id}",
+                            action_type=privacy_request.policy.get_action_type(),  # type: ignore
+                        )
+
+                    logger.info(
+                        "Marking privacy request '{}' as complete.",
+                        privacy_request.id,
+                    )
+                    AuditLog.create(
+                        db=session,
+                        data={
+                            "user_id": "system",
+                            "privacy_request_id": privacy_request.id,
+                            "action": AuditLogAction.finished,
+                            "message": "",
+                        },
+                    )
+                    privacy_request.status = PrivacyRequestStatus.complete
+                    privacy_request.finished_processing_at = datetime.utcnow()
+                    privacy_request.save(db=session)
+
+                    # Send a final email to the user confirming request completion
+                    if privacy_request.status == PrivacyRequestStatus.complete:
+                        legacy_request_completion_enabled = ConfigProxy(
+                            session
+                        ).notifications.send_request_completion_notification
+
+                        action_type = (
+                            MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
+                            if policy.get_rules_for_action(
+                                action_type=ActionType.access
+                            )
+                            else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
+                        )
+
+                        if message_send_enabled(
+                            session,
+                            privacy_request.property_id,
+                            action_type,
+                            legacy_request_completion_enabled,
+                        ) and not policy.get_rules_for_action(
+                            action_type=ActionType.consent
+                        ):
+                            if not access_result_urls:
+                                # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
+                                # from the database because the Privacy Request would have exited
+                                # processing and lost access to the access_result_urls in memory
+                                access_result_urls = (
+                                    privacy_request.access_result_urls or {}
+                                ).get("access_result_urls", [])
+
+                            try:
+                                initiate_privacy_request_completion_email(
+                                    session,
+                                    policy,
+                                    access_result_urls,
+                                    identity_data,
+                                    privacy_request.property_id,
+                                )
+                            except (
+                                IdentityNotFoundException,
+                                MessageDispatchException,
+                            ) as e:
+                                privacy_request.error_processing(db=session)
+                                # If dev mode, log traceback
+                                _log_exception(e, CONFIG.dev_mode)
+                                return
 
 
 def initiate_privacy_request_completion_email(

fides/api/task/create_request_tasks.py

@@ -33,8 +33,8 @@ from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    ManualTaskAddress,
     create_manual_task_instances_for_privacy_request,
 )
 from fides.api.util.logger_context_utils import log_context

fides/api/task/execute_request_tasks.py

@@ -29,8 +29,8 @@ from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
 )
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
-from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.task.task_resources import TaskResources
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.util.cache import cache_task_tracking_key

fides/api/task/filter_results.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from fides.api.graph.config import CollectionAddress, FieldPath
 from fides.api.graph.graph import DatasetGraph
-from fides.api.task.manual.manual_task_utils import ManualTaskAddress
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.util.collection_util import Row
 
 

fides/api/task/manual/manual_task_address.py

@@ -0,0 +1,46 @@
+from fides.api.graph.config import CollectionAddress
+
+
+class ManualTaskAddress:
+    """Utility class for creating and parsing manual task addresses"""
+
+    MANUAL_DATA_COLLECTION = "manual_data"
+
+    @staticmethod
+    def create(connection_config_key: str) -> CollectionAddress:
+        """Create a CollectionAddress for manual data: {connection_key}:manual_data"""
+        return CollectionAddress(
+            dataset=connection_config_key,
+            collection=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+        )
+
+    @staticmethod
+    def _is_manual_data_collection(collection_name: str) -> bool:
+        """Check if collection name represents manual task data"""
+        return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
+
+    @staticmethod
+    def is_manual_task_address(address: CollectionAddress) -> bool:
+        """Check if address represents manual task data"""
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            _, _, collection_part = address.partition(":")
+            if not collection_part:
+                return False
+            return ManualTaskAddress._is_manual_data_collection(collection_part)
+
+        # Handle CollectionAddress object
+        return ManualTaskAddress._is_manual_data_collection(address.collection)
+
+    @staticmethod
+    def get_connection_key(address: CollectionAddress) -> str:
+        """Extract connection config key from manual task address"""
+        if not ManualTaskAddress.is_manual_task_address(address):
+            raise ValueError(f"Not a manual task address: {address}")
+
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.split(":")[0]
+
+        # Handle CollectionAddress object
+        return address.dataset