ethyca-fides 2.66.2b2__py2.py3-none-any.whl → 2.66.2rc0__py2.py3-none-any.whl

fides/api/task/manual/manual_task_graph_task.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any, Dict, List, Optional
 
 from loguru import logger
 from sqlalchemy.orm import Session
@@ -18,9 +18,9 @@ from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.task.graph_task import GraphTask, retry
-from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    get_manual_task_for_connection_config,
+    ManualTaskAddress,
+    get_manual_tasks_for_connection_config,
 )
 from fides.api.util.collection_util import Row
 
@@ -29,7 +29,7 @@ class ManualTaskGraphTask(GraphTask):
     """GraphTask implementation for ManualTask execution"""
 
     @retry(action_type=ActionType.access, default_return=[])
-    def access_request(self, *inputs: list[Row]) -> list[Row]:
+    def access_request(self, *inputs: List[Row]) -> List[Row]:
         """
         Execute manual task logic following the standard GraphTask pattern:
         1. Create ManualTaskInstances if they don't exist
@@ -46,9 +46,9 @@ class ManualTaskGraphTask(GraphTask):
         connection_key = ManualTaskAddress.get_connection_key(collection_address)
 
         # Get manual tasks for this connection
-        manual_task = get_manual_task_for_connection_config(db, connection_key)
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
 
-        if not manual_task:
+        if not manual_tasks:
             return []
 
         # Check if any manual tasks have ACCESS configs
@@ -56,6 +56,7 @@ class ManualTaskGraphTask(GraphTask):
 
         has_access_configs = [
             config
+            for manual_task in manual_tasks
             for config in manual_task.configs
             if config.is_current
             and config.config_type == ManualTaskConfigurationType.access_privacy_request
@@ -76,7 +77,7 @@ class ManualTaskGraphTask(GraphTask):
         # Check/create manual task instances for ACCESS configs only
         self._ensure_manual_task_instances(
             db,
-            manual_task,
+            manual_tasks,
             self.resources.request,
             ManualTaskConfigurationType.access_privacy_request,
         )
@@ -84,13 +85,13 @@ class ManualTaskGraphTask(GraphTask):
         # Check if all manual task instances have submissions for ACCESS configs only
         submitted_data = self._get_submitted_data(
             db,
-            manual_task,
+            manual_tasks,
             self.resources.request,
             ManualTaskConfigurationType.access_privacy_request,
         )
 
         if submitted_data is not None:
-            result: list[Row] = [submitted_data] if submitted_data else []
+            result: List[Row] = [submitted_data] if submitted_data else []
             self.request_task.access_data = result
 
             # Mark request task as complete and write execution log
@@ -110,65 +111,67 @@ class ManualTaskGraphTask(GraphTask):
     def _ensure_manual_task_instances(
         self,
         db: Session,
-        manual_task: ManualTask,
+        manual_tasks: List[ManualTask],
         privacy_request: PrivacyRequest,
         allowed_config_type: "ManualTaskConfigurationType",
     ) -> None:
         """Create ManualTaskInstances for configs matching `allowed_config_type` if they don't exist."""
 
-        # ------------------------------------------------------------------
-        # Check if instances already exist for this task & entity with the SAME config type
-        # This prevents duplicates when configurations are versioned after the privacy
-        # request has started, while allowing different config types (access vs erasure)
-        # to have separate instances.
-        # ------------------------------------------------------------------
-        existing_task_instance = (
-            db.query(ManualTaskInstance)
-            .join(ManualTaskInstance.config)  # Join to access config information
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
-                # Only check for instances of the same config type
-                ManualTaskConfig.config_type == allowed_config_type,
+        for manual_task in manual_tasks:
+            # ------------------------------------------------------------------
+            # Check if instances already exist for this task & entity with the SAME config type
+            # This prevents duplicates when configurations are versioned after the privacy
+            # request has started, while allowing different config types (access vs erasure)
+            # to have separate instances.
+            # ------------------------------------------------------------------
+            existing_task_instance = (
+                db.query(ManualTaskInstance)
+                .join(ManualTaskInstance.config)  # Join to access config information
+                .filter(
+                    ManualTaskInstance.task_id == manual_task.id,
+                    ManualTaskInstance.entity_id == privacy_request.id,
+                    ManualTaskInstance.entity_type
+                    == ManualTaskEntityType.privacy_request,
+                    # Only check for instances of the same config type
+                    ManualTaskConfig.config_type == allowed_config_type,
+                )
+                .first()
             )
-            .first()
-        )
-        if existing_task_instance:
-            # An instance already exists for this privacy request and config type – no need
-            # to create another one tied to a newer config version.
-            return
-
-        # Check each active config for instances (now we know none exist yet for this config type)
-        for config in manual_task.configs:
-            if not config.is_current or config.config_type != allowed_config_type:
-                # Skip configs that are not current or not relevant for this request type
+            if existing_task_instance:
+                # An instance already exists for this privacy request and config type – no need
+                # to create another one tied to a newer config version.
                 continue
 
-            ManualTaskInstance.create(
-                db=db,
-                data={
-                    "task_id": manual_task.id,
-                    "config_id": config.id,
-                    "entity_id": privacy_request.id,
-                    "entity_type": ManualTaskEntityType.privacy_request.value,
-                    "status": StatusType.pending.value,
-                },
-            )
+            # Check each active config for instances (now we know none exist yet for this config type)
+            for config in manual_task.configs:
+                if not config.is_current or config.config_type != allowed_config_type:
+                    # Skip configs that are not current or not relevant for this request type
+                    continue
+
+                ManualTaskInstance.create(
+                    db=db,
+                    data={
+                        "task_id": manual_task.id,
+                        "config_id": config.id,
+                        "entity_id": privacy_request.id,
+                        "entity_type": ManualTaskEntityType.privacy_request.value,
+                        "status": StatusType.pending.value,
+                    },
+                )
 
     # pylint: disable=too-many-branches,too-many-nested-blocks
     def _get_submitted_data(
         self,
         db: Session,
-        manual_task: ManualTask,
+        manual_tasks: List[ManualTask],
         privacy_request: PrivacyRequest,
         allowed_config_type: "ManualTaskConfigurationType",
-    ) -> Optional[dict[str, Any]]:
+    ) -> Optional[Dict[str, Any]]:
         """
         Check if all manual task instances have submissions for ALL fields and return aggregated data
         Returns None if any field submissions are missing (all fields must be completed or skipped)
         """
-        aggregated_data: dict[str, Any] = {}
+        aggregated_data: Dict[str, Any] = {}
 
         def _format_size(size_bytes: int) -> str:
             units = ["B", "KB", "MB", "GB", "TB"]
@@ -179,74 +182,79 @@ class ManualTaskGraphTask(GraphTask):
                 size /= 1024.0
             return f"{size:.1f} PB"
 
-        candidate_instances: list[ManualTaskInstance] = (
-            db.query(ManualTaskInstance)
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
+        for manual_task in manual_tasks:
+
+            candidate_instances: list[ManualTaskInstance] = (
+                db.query(ManualTaskInstance)
+                .filter(
+                    ManualTaskInstance.task_id == manual_task.id,
+                    ManualTaskInstance.entity_id == privacy_request.id,
+                    ManualTaskInstance.entity_type
+                    == ManualTaskEntityType.privacy_request,
+                )
+                .all()
             )
-            .all()
-        )
-
-        if not candidate_instances:
-            return None  # No instance yet for this manual task
-
-        for inst in candidate_instances:
-            # Skip instances tied to other request types
-            if not inst.config or inst.config.config_type != allowed_config_type:
-                continue
-
-            all_fields = inst.config.field_definitions or []
-
-            # Every field must have a submission
-            if not all(inst.get_submission_for_field(f.id) for f in all_fields):
-                return None  # At least one instance still incomplete
-
-            # Ensure status set
-            if inst.status != StatusType.completed:
-                inst.status = StatusType.completed
-                inst.save(db)
-
-            # Aggregate submission data from this instance
-            for submission in inst.submissions:
-                if not submission.field or not submission.field.field_key:
-                    continue
 
-                field_key = submission.field.field_key
+            if not candidate_instances:
+                return None  # No instance yet for this manual task
 
-                if not isinstance(submission.data, dict):
+            for inst in candidate_instances:
+                # Skip instances tied to other request types
+                if not inst.config or inst.config.config_type != allowed_config_type:
                     continue
 
-                data_dict: dict[str, Any] = submission.data
-
-                field_type = data_dict.get("field_type")
-
-                if field_type == ManualTaskFieldType.attachment.value:
-                    attachment_map: dict[str, dict[str, Any]] = {}
-                    for attachment in submission.attachments or []:
-                        if (
-                            attachment.attachment_type
-                            == AttachmentType.include_with_access_package
-                        ):
-                            try:
-                                size, url = attachment.retrieve_attachment()
-                                attachment_map[attachment.file_name] = {
-                                    "url": str(url) if url else None,
-                                    "size": (_format_size(size) if size else "Unknown"),
-                                }
-                            except (
-                                Exception
-                            ) as exc:  # pylint: disable=broad-exception-caught
-                                logger.warning(
-                                    "Error retrieving attachment {}: {}",
-                                    attachment.file_name,
-                                    str(exc),
-                                )
-
-                    aggregated_data[field_key] = attachment_map or None
-                else:
-                    aggregated_data[field_key] = data_dict.get("value")
+                all_fields = inst.config.field_definitions or []
+
+                # Every field must have a submission
+                if not all(inst.get_submission_for_field(f.id) for f in all_fields):
+                    return None  # At least one instance still incomplete
+
+                # Ensure status set
+                if inst.status != StatusType.completed:
+                    inst.status = StatusType.completed
+                    inst.save(db)
+
+                # Aggregate submission data from this instance
+                for submission in inst.submissions:
+                    if not submission.field or not submission.field.field_key:
+                        continue
+
+                    field_key = submission.field.field_key
+
+                    if not isinstance(submission.data, dict):
+                        continue
+
+                    data_dict: Dict[str, Any] = submission.data
+
+                    field_type = data_dict.get("field_type")
+
+                    if field_type == ManualTaskFieldType.attachment.value:
+                        attachment_map: Dict[str, Dict[str, Any]] = {}
+                        for attachment in submission.attachments or []:
+                            if (
+                                attachment.attachment_type
+                                == AttachmentType.include_with_access_package
+                            ):
+                                try:
+                                    size, url = attachment.retrieve_attachment()
+                                    attachment_map[attachment.file_name] = {
+                                        "url": str(url) if url else None,
+                                        "size": (
+                                            _format_size(size) if size else "Unknown"
+                                        ),
+                                    }
+                                except (
+                                    Exception
+                                ) as exc:  # pylint: disable=broad-exception-caught
+                                    logger.warning(
+                                        "Error retrieving attachment {}: {}",
+                                        attachment.file_name,
+                                        str(exc),
+                                    )
+
+                        aggregated_data[field_key] = attachment_map or None
+                    else:
+                        aggregated_data[field_key] = data_dict.get("value")
 
         return aggregated_data if aggregated_data else None
 
@@ -258,9 +266,8 @@ class ManualTaskGraphTask(GraphTask):
     @retry(action_type=ActionType.erasure, default_return=0)
     def erasure_request(
         self,
-        retrieved_data: list[Row],
+        retrieved_data: List[Row],
         *erasure_prereqs: int,  # noqa: D401, pylint: disable=unused-argument
-        inputs: Optional[list[list[Row]]] = None,
     ) -> int:
         """Execute manual-task-driven erasure logic.
 
@@ -278,8 +285,8 @@ class ManualTaskGraphTask(GraphTask):
         connection_key = ManualTaskAddress.get_connection_key(collection_address)
 
         # Fetch relevant manual tasks for this connection
-        manual_task = get_manual_task_for_connection_config(db, connection_key)
-        if not manual_task:
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+        if not manual_tasks:
             # No manual tasks defined – nothing to erase
             self.log_end(ActionType.erasure)
             return 0
@@ -287,6 +294,7 @@ class ManualTaskGraphTask(GraphTask):
         # Check if any manual tasks have ERASURE configs
         has_erasure_configs = [
             config
+            for manual_task in manual_tasks
             for config in manual_task.configs
             if config.is_current
             and config.config_type
@@ -301,7 +309,7 @@ class ManualTaskGraphTask(GraphTask):
         # Create ManualTaskInstances for ERASURE configs only
         self._ensure_manual_task_instances(
             db,
-            manual_task,
+            manual_tasks,
             self.resources.request,
             ManualTaskConfigurationType.erasure_privacy_request,
         )
@@ -309,7 +317,7 @@ class ManualTaskGraphTask(GraphTask):
         # Check for full submissions – reuse helper used by access flow, filtering ERASURE configs
         submissions_complete = self._get_submitted_data(
             db,
-            manual_task,
+            manual_tasks,
             self.resources.request,
             ManualTaskConfigurationType.erasure_privacy_request,
         )

fides/api/task/manual/manual_task_utils.py

@@ -1,3 +1,5 @@
+from typing import TYPE_CHECKING, List
+
 from sqlalchemy.orm import Session
 
 from fides.api.graph.config import (
@@ -8,7 +10,6 @@ from fides.api.graph.config import (
     ScalarField,
 )
 from fides.api.graph.graph import Node
-from fides.api.graph.traversal import TraversalNode
 from fides.api.models.connectionconfig import ConnectionConfig
 
 # Import application models
@@ -21,10 +22,51 @@ from fides.api.models.manual_task import (
 )
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
-from fides.api.task.manual.manual_task_address import ManualTaskAddress
 
+# TYPE_CHECKING import placed after all runtime imports to avoid lint issues
+if TYPE_CHECKING:  # pragma: no cover
+    from fides.api.graph.traversal import TraversalNode  # noqa: F401
+    from fides.api.models.policy import Policy  # noqa: F401
+
+
+class ManualTaskAddress:
+    """Utility class for creating and parsing manual task addresses"""
+
+    MANUAL_DATA_COLLECTION = "manual_data"
+
+    @staticmethod
+    def create(connection_config_key: str) -> CollectionAddress:
+        """Create a CollectionAddress for manual data: {connection_key}:manual_data"""
+        return CollectionAddress(
+            dataset=connection_config_key,
+            collection=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+        )
+
+    @staticmethod
+    def is_manual_task_address(address: CollectionAddress) -> bool:
+        """Check if address represents manual task data"""
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.endswith(f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}")
+
+        # Handle CollectionAddress object
+        return address.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
+
+    @staticmethod
+    def get_connection_key(address: CollectionAddress) -> str:
+        """Extract connection config key from manual task address"""
+        if not ManualTaskAddress.is_manual_task_address(address):
+            raise ValueError(f"Not a manual task address: {address}")
+
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.split(":")[0]
+
+        # Handle CollectionAddress object
+        return address.dataset
 
-def get_connection_configs_with_manual_tasks(db: Session) -> list[ConnectionConfig]:
+
+def get_connection_configs_with_manual_tasks(db: Session) -> List[ConnectionConfig]:
     """
     Get all connection configs that have manual tasks.
     """
@@ -37,7 +79,7 @@ def get_connection_configs_with_manual_tasks(db: Session) -> list[ConnectionConf
     )
 
 
-def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
+def get_manual_task_addresses(db: Session) -> List[CollectionAddress]:
     """
     Get manual task addresses for all connection configs that have manual tasks.
 
@@ -56,20 +98,26 @@ def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
     return manual_task_addresses
 
 
-def get_manual_task_for_connection_config(
+def get_manual_tasks_for_connection_config(
     db: Session, connection_config_key: str
-) -> ManualTask:
-    """Get the ManualTask for a specific connection config,
-    the manual task/connection config relationship is 1:1.
-    """
+) -> List[ManualTask]:
+    """Get all ManualTasks for a specific connection config"""
+    connection_config = (
+        db.query(ConnectionConfig)
+        .filter(ConnectionConfig.key == connection_config_key)
+        .first()
+    )
+
+    if not connection_config:
+        return []
+
     return (
         db.query(ManualTask)
-        .join(ConnectionConfig, ManualTask.parent_entity_id == ConnectionConfig.id)
         .filter(
-            ConnectionConfig.key == connection_config_key,
+            ManualTask.parent_entity_id == connection_config.id,
             ManualTask.parent_entity_type == "connection_config",
         )
-        .one_or_none()
+        .all()
     )
 
 
@@ -82,23 +130,24 @@ def create_manual_data_traversal_node(
     connection_key = address.dataset
 
     # Get manual tasks for this connection to determine fields
-    manual_task = get_manual_task_for_connection_config(db, connection_key)
+    manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
 
     # Create fields based on ManualTaskConfigFields
-    fields: list[Field] = []
-    for config in manual_task.configs:
-        for field in config.field_definitions:
-            # Create a scalar field for each manual task field
-            # Extract data categories from field metadata if available
-            field_metadata = field.field_metadata or {}
-            data_categories = field_metadata.get("data_categories", [])
-
-            scalar_field = ScalarField(
-                name=field.field_key,
-                data_categories=data_categories,
-                # Manual task fields don't have complex relationships
-            )
-            fields.append(scalar_field)
+    fields: List[Field] = []
+    for manual_task in manual_tasks:
+        for config in manual_task.configs:
+            for field in config.field_definitions:
+                # Create a scalar field for each manual task field
+                # Extract data categories from field metadata if available
+                field_metadata = field.field_metadata or {}
+                data_categories = field_metadata.get("data_categories", [])
+
+                scalar_field = ScalarField(
+                    name=field.field_key,
+                    data_categories=data_categories,
+                    # Manual task fields don't have complex relationships
+                )
+                fields.append(scalar_field)
 
     # Create a synthetic Collection
     collection = Collection(
@@ -116,6 +165,9 @@ def create_manual_data_traversal_node(
         after=set(),
     )
 
+    # Create Node and TraversalNode (import locally to avoid cyclic import)
+    from fides.api.graph.traversal import TraversalNode  # local import
+
     node = Node(dataset, collection)
     traversal_node = TraversalNode(node)
 
@@ -124,7 +176,7 @@ def create_manual_data_traversal_node(
 
 def create_manual_task_instances_for_privacy_request(
     db: Session, privacy_request: PrivacyRequest
-) -> list[ManualTaskInstance]:
+) -> List[ManualTaskInstance]:
     """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
     instances = []
 
@@ -220,7 +272,7 @@ def create_manual_task_instances_for_privacy_request(
 
 def get_manual_task_instances_for_privacy_request(
     db: Session, privacy_request: PrivacyRequest
-) -> list[ManualTaskInstance]:
+) -> List[ManualTaskInstance]:
     """Get all manual task instances for a privacy request."""
     return (
         db.query(ManualTaskInstance)
@@ -234,7 +286,7 @@ def get_manual_task_instances_for_privacy_request(
 
 def create_manual_task_artificial_graphs(
     db: Session,
-) -> list:
+) -> List:
     """
     Create artificial GraphDataset objects for manual tasks that can be included
     in the main dataset graph during the dataset configuration phase.
@@ -261,33 +313,34 @@ def create_manual_task_artificial_graphs(
         connection_key = address.dataset
 
         # Get manual tasks for this connection to determine fields
-        manual_task = get_manual_task_for_connection_config(db, connection_key)
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
 
         # Create fields based only on ManualTaskConfigFields
-        fields: list = []
+        fields: List = []
 
         # Manual task collections act as root nodes - they don't need identity dependencies
         # since they provide manually-entered data rather than consuming identity data.
-        current_configs = [
-            config for config in manual_task.configs if config.is_current
-        ]
-        for config in current_configs:
-            if config.config_type not in [
-                ManualTaskConfigurationType.access_privacy_request,
-                ManualTaskConfigurationType.erasure_privacy_request,
-            ]:
-                continue
-
-            for field in config.field_definitions:
-                # Create a scalar field for each manual task field
-                field_metadata = field.field_metadata or {}
-                data_categories = field_metadata.get("data_categories", [])
-
-                scalar_field = ScalarField(
-                    name=field.field_key,
-                    data_categories=data_categories,
-                )
-                fields.append(scalar_field)
+        for manual_task in manual_tasks:
+            current_configs = [
+                config for config in manual_task.configs if config.is_current
+            ]
+            for config in current_configs:
+                if config.config_type not in [
+                    ManualTaskConfigurationType.access_privacy_request,
+                    ManualTaskConfigurationType.erasure_privacy_request,
+                ]:
+                    continue
+
+                for field in config.field_definitions:
+                    # Create a scalar field for each manual task field
+                    field_metadata = field.field_metadata or {}
+                    data_categories = field_metadata.get("data_categories", [])
+
+                    scalar_field = ScalarField(
+                        name=field.field_key,
+                        data_categories=data_categories,
+                    )
+                    fields.append(scalar_field)
 
         if fields:  # Only create graph if there are fields
             # Create a synthetic Collection

fides/api/util/cache.py

@@ -387,8 +387,3 @@ def get_queue_counts() -> Dict[str, int]:
         logger.critical(exception)
         queue_counts = {}
     return queue_counts
-
-
-def get_all_masking_secret_keys(privacy_request_id: str) -> List[str]:
-    cache: FidesopsRedis = get_cache()
-    return cache.keys(f"id-{privacy_request_id}-masking-secret-*")