ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl

fides/api/task/graph_task.py

@@ -421,6 +421,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         action_type: ActionType,
         ex: Optional[BaseException] = None,
         success_override_msg: Optional[str] = None,
+        record_count: Optional[int] = None,
     ) -> None:
         """On completion activities"""
         if ex:
@@ -440,8 +441,23 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 mark_current_and_downstream_nodes_as_failed(request_task, db)
         else:
             logger.info("Ending {}, {}", self.resources.request.id, self.key)
+
+            # Build standardized success message with record count
+            base_message = (
+                str(success_override_msg) if success_override_msg else "success"
+            )
+            if record_count is not None:
+                if action_type == ActionType.access:
+                    message = f"{base_message} - retrieved {record_count} records"
+                elif action_type == ActionType.erasure:
+                    message = f"{base_message} - masked {record_count} records"
+                else:
+                    message = f"{base_message} - processed {record_count} records"
+            else:
+                message = base_message
+
             self.update_status(
-                str(success_override_msg) if success_override_msg else "success",
+                message,
                 build_affected_field_logs(
                     self.execution_node, self.resources.policy, action_type
                 ),
@@ -637,7 +653,11 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(ActionType.access, success_override_msg=success_message)
+        self.log_end(
+            ActionType.access,
+            success_override_msg=success_message,
+            record_count=len(filtered_output),
+        )
         return filtered_output
 
     @retry(action_type=ActionType.erasure, default_return=0)
@@ -645,9 +665,15 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         self,
         retrieved_data: List[Row],
         *erasure_prereqs: int,  # TODO Remove when we stop support for DSR 2.0. DSR 3.0 enforces with downstream_tasks.
+        inputs: Optional[
+            List[List[Row]]
+        ] = None,  # Upstream data from corresponding access task
     ) -> int:
         """Run erasure request"""
 
+        if inputs is None:
+            inputs = []
+
         # if there is no primary key specified in the graph node configuration
         # note this in the execution log and perform no erasures on this node
         if (
@@ -694,6 +720,10 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             )
             return 0
 
+        formatted_input_data: NodeInput = self.pre_process_input_data(
+            *inputs, group_dependent_fields=True
+        )
+
         # Use execution context to capture postprocessor messages
         with collect_execution_log_messages() as messages:
             output = self.connector.mask_data(
@@ -702,6 +732,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 self.resources.request,
                 self.resources.privacy_request_task,
                 retrieved_data,
+                formatted_input_data,
             )
 
         if self.request_task.id:
@@ -720,7 +751,11 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(ActionType.erasure, success_override_msg=success_message)
+        self.log_end(
+            ActionType.erasure,
+            success_override_msg=success_message,
+            record_count=output,
+        )
         return output
 
     @retry(action_type=ActionType.consent, default_return=False)

fides/api/task/manual/manual_task_address.py

@@ -0,0 +1,46 @@
+from fides.api.graph.config import CollectionAddress
+
+
+class ManualTaskAddress:
+    """Utility class for creating and parsing manual task addresses"""
+
+    MANUAL_DATA_COLLECTION = "manual_data"
+
+    @staticmethod
+    def create(connection_config_key: str) -> CollectionAddress:
+        """Create a CollectionAddress for manual data: {connection_key}:manual_data"""
+        return CollectionAddress(
+            dataset=connection_config_key,
+            collection=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+        )
+
+    @staticmethod
+    def _is_manual_data_collection(collection_name: str) -> bool:
+        """Check if collection name represents manual task data"""
+        return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
+
+    @staticmethod
+    def is_manual_task_address(address: CollectionAddress) -> bool:
+        """Check if address represents manual task data"""
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            _, _, collection_part = address.partition(":")
+            if not collection_part:
+                return False
+            return ManualTaskAddress._is_manual_data_collection(collection_part)
+
+        # Handle CollectionAddress object
+        return ManualTaskAddress._is_manual_data_collection(address.collection)
+
+    @staticmethod
+    def get_connection_key(address: CollectionAddress) -> str:
+        """Extract connection config key from manual task address"""
+        if not ManualTaskAddress.is_manual_task_address(address):
+            raise ValueError(f"Not a manual task address: {address}")
+
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.split(":")[0]
+
+        # Handle CollectionAddress object
+        return address.dataset

fides/api/task/manual/manual_task_graph_task.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 from loguru import logger
 from sqlalchemy.orm import Session
@@ -18,9 +18,9 @@ from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.task.graph_task import GraphTask, retry
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    ManualTaskAddress,
-    get_manual_tasks_for_connection_config,
+    get_manual_task_for_connection_config,
 )
 from fides.api.util.collection_util import Row
 
@@ -29,7 +29,7 @@ class ManualTaskGraphTask(GraphTask):
     """GraphTask implementation for ManualTask execution"""
 
     @retry(action_type=ActionType.access, default_return=[])
-    def access_request(self, *inputs: List[Row]) -> List[Row]:
+    def access_request(self, *inputs: list[Row]) -> list[Row]:
         """
         Execute manual task logic following the standard GraphTask pattern:
         1. Create ManualTaskInstances if they don't exist
@@ -46,9 +46,9 @@ class ManualTaskGraphTask(GraphTask):
         connection_key = ManualTaskAddress.get_connection_key(collection_address)
 
         # Get manual tasks for this connection
-        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+        manual_task = get_manual_task_for_connection_config(db, connection_key)
 
-        if not manual_tasks:
+        if not manual_task:
             return []
 
         # Check if any manual tasks have ACCESS configs
@@ -56,7 +56,6 @@ class ManualTaskGraphTask(GraphTask):
 
         has_access_configs = [
             config
-            for manual_task in manual_tasks
             for config in manual_task.configs
             if config.is_current
             and config.config_type == ManualTaskConfigurationType.access_privacy_request
@@ -77,7 +76,7 @@ class ManualTaskGraphTask(GraphTask):
         # Check/create manual task instances for ACCESS configs only
         self._ensure_manual_task_instances(
             db,
-            manual_tasks,
+            manual_task,
             self.resources.request,
             ManualTaskConfigurationType.access_privacy_request,
         )
@@ -85,13 +84,13 @@ class ManualTaskGraphTask(GraphTask):
         # Check if all manual task instances have submissions for ACCESS configs only
         submitted_data = self._get_submitted_data(
             db,
-            manual_tasks,
+            manual_task,
             self.resources.request,
             ManualTaskConfigurationType.access_privacy_request,
         )
 
         if submitted_data is not None:
-            result: List[Row] = [submitted_data] if submitted_data else []
+            result: list[Row] = [submitted_data] if submitted_data else []
             self.request_task.access_data = result
 
             # Mark request task as complete and write execution log
@@ -111,67 +110,65 @@ class ManualTaskGraphTask(GraphTask):
     def _ensure_manual_task_instances(
         self,
         db: Session,
-        manual_tasks: List[ManualTask],
+        manual_task: ManualTask,
         privacy_request: PrivacyRequest,
         allowed_config_type: "ManualTaskConfigurationType",
     ) -> None:
         """Create ManualTaskInstances for configs matching `allowed_config_type` if they don't exist."""
 
-        for manual_task in manual_tasks:
-            # ------------------------------------------------------------------
-            # Check if instances already exist for this task & entity with the SAME config type
-            # This prevents duplicates when configurations are versioned after the privacy
-            # request has started, while allowing different config types (access vs erasure)
-            # to have separate instances.
-            # ------------------------------------------------------------------
-            existing_task_instance = (
-                db.query(ManualTaskInstance)
-                .join(ManualTaskInstance.config)  # Join to access config information
-                .filter(
-                    ManualTaskInstance.task_id == manual_task.id,
-                    ManualTaskInstance.entity_id == privacy_request.id,
-                    ManualTaskInstance.entity_type
-                    == ManualTaskEntityType.privacy_request,
-                    # Only check for instances of the same config type
-                    ManualTaskConfig.config_type == allowed_config_type,
-                )
-                .first()
+        # ------------------------------------------------------------------
+        # Check if instances already exist for this task & entity with the SAME config type
+        # This prevents duplicates when configurations are versioned after the privacy
+        # request has started, while allowing different config types (access vs erasure)
+        # to have separate instances.
+        # ------------------------------------------------------------------
+        existing_task_instance = (
+            db.query(ManualTaskInstance)
+            .join(ManualTaskInstance.config)  # Join to access config information
+            .filter(
+                ManualTaskInstance.task_id == manual_task.id,
+                ManualTaskInstance.entity_id == privacy_request.id,
+                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
+                # Only check for instances of the same config type
+                ManualTaskConfig.config_type == allowed_config_type,
             )
-            if existing_task_instance:
-                # An instance already exists for this privacy request and config type – no need
-                # to create another one tied to a newer config version.
+            .first()
+        )
+        if existing_task_instance:
+            # An instance already exists for this privacy request and config type – no need
+            # to create another one tied to a newer config version.
+            return
+
+        # Check each active config for instances (now we know none exist yet for this config type)
+        for config in manual_task.configs:
+            if not config.is_current or config.config_type != allowed_config_type:
+                # Skip configs that are not current or not relevant for this request type
                 continue
 
-            # Check each active config for instances (now we know none exist yet for this config type)
-            for config in manual_task.configs:
-                if not config.is_current or config.config_type != allowed_config_type:
-                    # Skip configs that are not current or not relevant for this request type
-                    continue
-
-                ManualTaskInstance.create(
-                    db=db,
-                    data={
-                        "task_id": manual_task.id,
-                        "config_id": config.id,
-                        "entity_id": privacy_request.id,
-                        "entity_type": ManualTaskEntityType.privacy_request.value,
-                        "status": StatusType.pending.value,
-                    },
-                )
+            ManualTaskInstance.create(
+                db=db,
+                data={
+                    "task_id": manual_task.id,
+                    "config_id": config.id,
+                    "entity_id": privacy_request.id,
+                    "entity_type": ManualTaskEntityType.privacy_request.value,
+                    "status": StatusType.pending.value,
+                },
+            )
 
     # pylint: disable=too-many-branches,too-many-nested-blocks
     def _get_submitted_data(
         self,
         db: Session,
-        manual_tasks: List[ManualTask],
+        manual_task: ManualTask,
         privacy_request: PrivacyRequest,
         allowed_config_type: "ManualTaskConfigurationType",
-    ) -> Optional[Dict[str, Any]]:
+    ) -> Optional[dict[str, Any]]:
         """
         Check if all manual task instances have submissions for ALL fields and return aggregated data
         Returns None if any field submissions are missing (all fields must be completed or skipped)
         """
-        aggregated_data: Dict[str, Any] = {}
+        aggregated_data: dict[str, Any] = {}
 
         def _format_size(size_bytes: int) -> str:
             units = ["B", "KB", "MB", "GB", "TB"]
@@ -182,79 +179,74 @@ class ManualTaskGraphTask(GraphTask):
                 size /= 1024.0
             return f"{size:.1f} PB"
 
-        for manual_task in manual_tasks:
-
-            candidate_instances: list[ManualTaskInstance] = (
-                db.query(ManualTaskInstance)
-                .filter(
-                    ManualTaskInstance.task_id == manual_task.id,
-                    ManualTaskInstance.entity_id == privacy_request.id,
-                    ManualTaskInstance.entity_type
-                    == ManualTaskEntityType.privacy_request,
-                )
-                .all()
+        candidate_instances: list[ManualTaskInstance] = (
+            db.query(ManualTaskInstance)
+            .filter(
+                ManualTaskInstance.task_id == manual_task.id,
+                ManualTaskInstance.entity_id == privacy_request.id,
+                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
             )
+            .all()
+        )
+
+        if not candidate_instances:
+            return None  # No instance yet for this manual task
+
+        for inst in candidate_instances:
+            # Skip instances tied to other request types
+            if not inst.config or inst.config.config_type != allowed_config_type:
+                continue
+
+            all_fields = inst.config.field_definitions or []
+
+            # Every field must have a submission
+            if not all(inst.get_submission_for_field(f.id) for f in all_fields):
+                return None  # At least one instance still incomplete
+
+            # Ensure status set
+            if inst.status != StatusType.completed:
+                inst.status = StatusType.completed
+                inst.save(db)
+
+            # Aggregate submission data from this instance
+            for submission in inst.submissions:
+                if not submission.field or not submission.field.field_key:
+                    continue
 
-            if not candidate_instances:
-                return None  # No instance yet for this manual task
+                field_key = submission.field.field_key
 
-            for inst in candidate_instances:
-                # Skip instances tied to other request types
-                if not inst.config or inst.config.config_type != allowed_config_type:
+                if not isinstance(submission.data, dict):
                     continue
 
-                all_fields = inst.config.field_definitions or []
-
-                # Every field must have a submission
-                if not all(inst.get_submission_for_field(f.id) for f in all_fields):
-                    return None  # At least one instance still incomplete
-
-                # Ensure status set
-                if inst.status != StatusType.completed:
-                    inst.status = StatusType.completed
-                    inst.save(db)
-
-                # Aggregate submission data from this instance
-                for submission in inst.submissions:
-                    if not submission.field or not submission.field.field_key:
-                        continue
-
-                    field_key = submission.field.field_key
-
-                    if not isinstance(submission.data, dict):
-                        continue
-
-                    data_dict: Dict[str, Any] = submission.data
-
-                    field_type = data_dict.get("field_type")
-
-                    if field_type == ManualTaskFieldType.attachment.value:
-                        attachment_map: Dict[str, Dict[str, Any]] = {}
-                        for attachment in submission.attachments or []:
-                            if (
-                                attachment.attachment_type
-                                == AttachmentType.include_with_access_package
-                            ):
-                                try:
-                                    size, url = attachment.retrieve_attachment()
-                                    attachment_map[attachment.file_name] = {
-                                        "url": str(url) if url else None,
-                                        "size": (
-                                            _format_size(size) if size else "Unknown"
-                                        ),
-                                    }
-                                except (
-                                    Exception
-                                ) as exc:  # pylint: disable=broad-exception-caught
-                                    logger.warning(
-                                        "Error retrieving attachment {}: {}",
-                                        attachment.file_name,
-                                        str(exc),
-                                    )
-
-                        aggregated_data[field_key] = attachment_map or None
-                    else:
-                        aggregated_data[field_key] = data_dict.get("value")
+                data_dict: dict[str, Any] = submission.data
+
+                field_type = data_dict.get("field_type")
+
+                if field_type == ManualTaskFieldType.attachment.value:
+                    attachment_map: dict[str, dict[str, Any]] = {}
+                    for attachment in submission.attachments or []:
+                        if (
+                            attachment.attachment_type
+                            == AttachmentType.include_with_access_package
+                        ):
+                            try:
+                                size, url = attachment.retrieve_attachment()
+                                attachment_map[attachment.file_name] = {
+                                    "url": str(url) if url else None,
+                                    "size": (_format_size(size) if size else "Unknown"),
+                                }
+                            except (
+                                Exception
+                            ) as exc:  # pylint: disable=broad-exception-caught
+                                logger.warning(
+                                    "Error retrieving attachment {}: {}",
+                                    attachment.file_name,
+                                    str(exc),
+                                )
+
+                    aggregated_data[field_key] = attachment_map or None
+                else:
+                    aggregated_data[field_key] = data_dict.get("value")
 
         return aggregated_data if aggregated_data else None
 
@@ -266,8 +258,9 @@ class ManualTaskGraphTask(GraphTask):
     @retry(action_type=ActionType.erasure, default_return=0)
     def erasure_request(
         self,
-        retrieved_data: List[Row],
+        retrieved_data: list[Row],
         *erasure_prereqs: int,  # noqa: D401, pylint: disable=unused-argument
+        inputs: Optional[list[list[Row]]] = None,
     ) -> int:
         """Execute manual-task-driven erasure logic.
 
@@ -285,8 +278,8 @@ class ManualTaskGraphTask(GraphTask):
         connection_key = ManualTaskAddress.get_connection_key(collection_address)
 
         # Fetch relevant manual tasks for this connection
-        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
-        if not manual_tasks:
+        manual_task = get_manual_task_for_connection_config(db, connection_key)
+        if not manual_task:
             # No manual tasks defined – nothing to erase
             self.log_end(ActionType.erasure)
             return 0
@@ -294,7 +287,6 @@ class ManualTaskGraphTask(GraphTask):
         # Check if any manual tasks have ERASURE configs
         has_erasure_configs = [
             config
-            for manual_task in manual_tasks
             for config in manual_task.configs
             if config.is_current
             and config.config_type
@@ -309,7 +301,7 @@ class ManualTaskGraphTask(GraphTask):
         # Create ManualTaskInstances for ERASURE configs only
         self._ensure_manual_task_instances(
             db,
-            manual_tasks,
+            manual_task,
             self.resources.request,
             ManualTaskConfigurationType.erasure_privacy_request,
         )
@@ -317,7 +309,7 @@ class ManualTaskGraphTask(GraphTask):
         # Check for full submissions – reuse helper used by access flow, filtering ERASURE configs
         submissions_complete = self._get_submitted_data(
             db,
-            manual_tasks,
+            manual_task,
             self.resources.request,
             ManualTaskConfigurationType.erasure_privacy_request,
         )