ethyca-fides 2.67.0rc2__py2.py3-none-any.whl → 2.67.1b1__py2.py3-none-any.whl

fides/api/task/manual/manual_task_graph_task.py

@@ -7,11 +7,11 @@ from fides.api.common_exceptions import AwaitingAsyncTaskCallback
 from fides.api.models.attachment import AttachmentType
 from fides.api.models.manual_task import (
     ManualTask,
-    ManualTaskConfig,
     ManualTaskConfigurationType,
     ManualTaskEntityType,
     ManualTaskFieldType,
     ManualTaskInstance,
+    ManualTaskSubmission,
     StatusType,
 )
 from fides.api.models.privacy_request import PrivacyRequest
@@ -23,6 +23,7 @@ from fides.api.task.manual.manual_task_utils import (
     get_manual_task_for_connection_config,
 )
 from fides.api.util.collection_util import Row
+from fides.api.util.storage_util import format_size
 
 
 class ManualTaskGraphTask(GraphTask):
@@ -122,29 +123,36 @@ class ManualTaskGraphTask(GraphTask):
         # request has started, while allowing different config types (access vs erasure)
         # to have separate instances.
         # ------------------------------------------------------------------
-        existing_task_instance = (
-            db.query(ManualTaskInstance)
-            .join(ManualTaskInstance.config)  # Join to access config information
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
-                # Only check for instances of the same config type
-                ManualTaskConfig.config_type == allowed_config_type,
-            )
-            .first()
+        existing_task_instance = next(
+            (
+                instance
+                for instance in privacy_request.manual_task_instances
+                if instance.task_id == manual_task.id
+                and instance.config.config_type == allowed_config_type
+            ),
+            None,
         )
         if existing_task_instance:
             # An instance already exists for this privacy request and config type – no need
             # to create another one tied to a newer config version.
             return
 
-        # Check each active config for instances (now we know none exist yet for this config type)
-        for config in manual_task.configs:
-            if not config.is_current or config.config_type != allowed_config_type:
-                # Skip configs that are not current or not relevant for this request type
-                continue
+        # If no existing instances, create a new one for the current config
+        # There will only be one config of each type per manual task
+        config = next(
+            (
+                config
+                for config in sorted(
+                    manual_task.configs,
+                    key=lambda c: c.version if hasattr(c, "version") else 0,
+                    reverse=True,
+                )
+                if config.is_current and config.config_type == allowed_config_type
+            ),
+            None,
+        )
 
+        if config:
             ManualTaskInstance.create(
                 db=db,
                 data={
@@ -156,7 +164,6 @@ class ManualTaskGraphTask(GraphTask):
                 },
             )
 
-    # pylint: disable=too-many-branches,too-many-nested-blocks
     def _get_submitted_data(
         self,
         db: Session,
@@ -168,93 +175,90 @@ class ManualTaskGraphTask(GraphTask):
         Check if all manual task instances have submissions for ALL fields and return aggregated data
         Returns None if any field submissions are missing (all fields must be completed or skipped)
         """
-        aggregated_data: dict[str, Any] = {}
-
-        def _format_size(size_bytes: int) -> str:
-            units = ["B", "KB", "MB", "GB", "TB"]
-            size = float(size_bytes)
-            for unit in units:
-                if size < 1024.0:
-                    return f"{size:.1f} {unit}"
-                size /= 1024.0
-            return f"{size:.1f} PB"
-
-        candidate_instances: list[ManualTaskInstance] = (
-            db.query(ManualTaskInstance)
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
-            )
-            .all()
-        )
+        candidate_instances: list[ManualTaskInstance] = [
+            instance
+            for instance in privacy_request.manual_task_instances
+            if instance.task_id == manual_task.id
+            and instance.config.config_type == allowed_config_type
+        ]
 
         if not candidate_instances:
             return None  # No instance yet for this manual task
 
+        # Check for incomplete fields and update status in single pass
         for inst in candidate_instances:
-            # Skip instances tied to other request types
-            if not inst.config or inst.config.config_type != allowed_config_type:
-                continue
-
-            all_fields = inst.config.field_definitions or []
-
-            # Every field must have a submission
-            if not all(inst.get_submission_for_field(f.id) for f in all_fields):
+            if inst.incomplete_fields:
                 return None  # At least one instance still incomplete
 
-            # Ensure status set
+            # Update status if needed
             if inst.status != StatusType.completed:
                 inst.status = StatusType.completed
                 inst.save(db)
 
-            # Aggregate submission data from this instance
-            for submission in inst.submissions:
-                if not submission.field or not submission.field.field_key:
-                    continue
+        # Aggregate submission data from all instances
+        aggregated_data = self._aggregate_submission_data(candidate_instances)
+        return aggregated_data or None
+
+    def _aggregate_submission_data(
+        self, instances: list[ManualTaskInstance]
+    ) -> dict[str, Any]:
+        """Aggregate submission data from all instances into a single dictionary."""
+        aggregated_data: dict[str, Any] = {}
+
+        for inst in instances:
+            # Filter valid submissions and process them
+            valid_submissions = (
+                submission
+                for submission in inst.submissions
+                if (
+                    submission.field
+                    and submission.field.field_key
+                    and isinstance(submission.data, dict)
+                )
+            )
 
+            for submission in valid_submissions:
                 field_key = submission.field.field_key
+                # We already checked isinstance(submission.data, dict) in valid_submissions
+                data_dict: dict[str, Any] = submission.data  # type: ignore[assignment]
+                field_type = data_dict.get("field_type")
 
-                if not isinstance(submission.data, dict):
-                    continue
+                # Process field data based on type
+                aggregated_data[field_key] = (
+                    self._process_attachment_field(submission)
+                    if field_type == ManualTaskFieldType.attachment.value
+                    else data_dict.get("value")
+                )
 
-                data_dict: dict[str, Any] = submission.data
+        return aggregated_data
 
-                field_type = data_dict.get("field_type")
+    def _process_attachment_field(
+        self, submission: ManualTaskSubmission
+    ) -> Optional[dict[str, dict[str, Any]]]:
+        """Process attachment field and return attachment map or None."""
+        attachment_map: dict[str, dict[str, Any]] = {}
 
-                if field_type == ManualTaskFieldType.attachment.value:
-                    attachment_map: dict[str, dict[str, Any]] = {}
-                    for attachment in submission.attachments or []:
-                        if (
-                            attachment.attachment_type
-                            == AttachmentType.include_with_access_package
-                        ):
-                            try:
-                                size, url = attachment.retrieve_attachment()
-                                attachment_map[attachment.file_name] = {
-                                    "url": str(url) if url else None,
-                                    "size": (_format_size(size) if size else "Unknown"),
-                                }
-                            except (
-                                Exception
-                            ) as exc:  # pylint: disable=broad-exception-caught
-                                logger.warning(
-                                    "Error retrieving attachment {}: {}",
-                                    attachment.file_name,
-                                    str(exc),
-                                )
-
-                    aggregated_data[field_key] = attachment_map or None
-                else:
-                    aggregated_data[field_key] = data_dict.get("value")
-
-        return aggregated_data if aggregated_data else None
+        for attachment in filter(
+            lambda a: a.attachment_type == AttachmentType.include_with_access_package,
+            submission.attachments,
+        ):
+            try:
+                size, url = attachment.retrieve_attachment()
+                attachment_map[attachment.file_name] = {
+                    "url": str(url) if url else None,
+                    "size": (format_size(size) if size else "Unknown"),
+                }
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                logger.warning(
+                    f"Error retrieving attachment {attachment.file_name}: {str(exc)}"
+                )
+        return attachment_map or None
 
     def dry_run_task(self) -> int:
         """Return estimated row count for dry run - manual tasks don't have predictable counts"""
         return 1  # Placeholder - manual tasks generate variable data
 
-    # NEW METHOD: Provide erasure support for manual tasks
+    # Provide erasure support for manual tasks
     @retry(action_type=ActionType.erasure, default_return=0)
     def erasure_request(
         self,

fides/api/task/manual/manual_task_utils.py

@@ -1,3 +1,4 @@
+from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.graph.config import (
@@ -12,15 +13,7 @@ from fides.api.graph.traversal import TraversalNode
 from fides.api.models.connectionconfig import ConnectionConfig
 
 # Import application models
-from fides.api.models.manual_task import (
-    ManualTask,
-    ManualTaskConfig,
-    ManualTaskConfigurationType,
-    ManualTaskEntityType,
-    ManualTaskInstance,
-)
-from fides.api.models.privacy_request import PrivacyRequest
-from fides.api.schemas.policy import ActionType
+from fides.api.models.manual_task import ManualTask, ManualTaskConfigurationType
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 
 
@@ -28,13 +21,18 @@ def get_connection_configs_with_manual_tasks(db: Session) -> list[ConnectionConf
     """
     Get all connection configs that have manual tasks.
     """
-    return (
+    logger.info("Querying for connection configs with manual tasks")
+    connection_configs = (
         db.query(ConnectionConfig)
         .join(ManualTask, ConnectionConfig.id == ManualTask.parent_entity_id)
         .filter(ManualTask.parent_entity_type == "connection_config")
         .filter(ConnectionConfig.disabled.is_(False))
         .all()
     )
+    logger.info(
+        f"Found {len(connection_configs)} connection configs with manual tasks: {[cc.key for cc in connection_configs]}"
+    )
+    return connection_configs
 
 
 def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
@@ -47,12 +45,19 @@ def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
     """
     # Get all connection configs that have manual tasks (excluding disabled ones)
     connection_configs_with_manual_tasks = get_connection_configs_with_manual_tasks(db)
+    logger.debug(
+        f"Found {len(connection_configs_with_manual_tasks)} connection configs with manual tasks"
+    )
 
     # Create addresses for all connections that have manual tasks
     manual_task_addresses = []
     for config in connection_configs_with_manual_tasks:
+        logger.info(f"Creating manual task address for connection config: {config.key}")
         manual_task_addresses.append(ManualTaskAddress.create(config.key))
 
+    logger.info(
+        f"Created {len(manual_task_addresses)} manual task addresses: {manual_task_addresses}"
+    )
     return manual_task_addresses
 
 
@@ -62,7 +67,11 @@ def get_manual_task_for_connection_config(
     """Get the ManualTask for a specific connection config,
     the manual task/connection config relationship is 1:1.
     """
-    return (
+    logger.info(
+        f"Looking for manual task for connection config: {connection_config_key}"
+    )
+
+    manual_task = (
         db.query(ManualTask)
         .join(ConnectionConfig, ManualTask.parent_entity_id == ConnectionConfig.id)
         .filter(
@@ -72,6 +81,17 @@ def get_manual_task_for_connection_config(
         .one_or_none()
     )
 
+    if manual_task:
+        logger.info(
+            f"Found manual task {manual_task.id} for connection {connection_config_key}"
+        )
+    else:
+        logger.warning(
+            f"No manual task found for connection config: {connection_config_key}"
+        )
+
+    return manual_task
+
 
 def create_manual_data_traversal_node(
     db: Session, address: CollectionAddress
@@ -122,116 +142,6 @@ def create_manual_data_traversal_node(
     return traversal_node
 
 
-def create_manual_task_instances_for_privacy_request(
-    db: Session, privacy_request: PrivacyRequest
-) -> list[ManualTaskInstance]:
-    """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
-    instances = []
-
-    # Get all connection configs that have manual tasks (excluding disabled ones)
-    connection_configs_with_manual_tasks = get_connection_configs_with_manual_tasks(db)
-
-    # Determine the privacy request type based on policy rules
-    has_access_rules = bool(
-        privacy_request.policy.get_rules_for_action(action_type=ActionType.access)
-    )
-    has_erasure_rules = bool(
-        privacy_request.policy.get_rules_for_action(action_type=ActionType.erasure)
-    )
-
-    for connection_config in connection_configs_with_manual_tasks:
-        manual_tasks = (
-            db.query(ManualTask)
-            .filter(
-                ManualTask.parent_entity_id == connection_config.id,
-                ManualTask.parent_entity_type == "connection_config",
-            )
-            .all()
-        )
-
-        for manual_task in manual_tasks:
-            # Get the active config for this manual task, filtered by request type
-            active_config_query = db.query(ManualTaskConfig).filter(
-                ManualTaskConfig.task_id == manual_task.id,
-                ManualTaskConfig.is_current.is_(True),
-            )
-
-            # Filter by configuration type based on privacy request type
-            if has_access_rules and has_erasure_rules:
-                # If both access and erasure rules exist, include both types
-                active_config_query = active_config_query.filter(
-                    ManualTaskConfig.config_type.in_(
-                        [
-                            ManualTaskConfigurationType.access_privacy_request,
-                            ManualTaskConfigurationType.erasure_privacy_request,
-                        ]
-                    )
-                )
-            elif has_access_rules:
-                # Only access rules - only include access configurations
-                active_config_query = active_config_query.filter(
-                    ManualTaskConfig.config_type
-                    == ManualTaskConfigurationType.access_privacy_request
-                )
-            elif has_erasure_rules:
-                # Only erasure rules - only include erasure configurations
-                active_config_query = active_config_query.filter(
-                    ManualTaskConfig.config_type
-                    == ManualTaskConfigurationType.erasure_privacy_request
-                )
-            else:
-                # No relevant rules - skip this manual task
-                continue
-
-            active_configs = active_config_query.all()
-
-            if not active_configs:
-                continue  # Skip if no active configs
-
-            # Create instances for each active config
-            for active_config in active_configs:
-                # Check if instance already exists for this config
-                existing_instance = (
-                    db.query(ManualTaskInstance)
-                    .filter(
-                        ManualTaskInstance.entity_id == privacy_request.id,
-                        ManualTaskInstance.entity_type == "privacy_request",
-                        ManualTaskInstance.task_id == manual_task.id,
-                        ManualTaskInstance.config_id == active_config.id,
-                    )
-                    .first()
-                )
-
-                if not existing_instance:
-                    instance = ManualTaskInstance(
-                        entity_id=privacy_request.id,
-                        entity_type=ManualTaskEntityType.privacy_request,
-                        task_id=manual_task.id,
-                        config_id=active_config.id,
-                    )
-                    db.add(instance)
-                    instances.append(instance)
-
-    if instances:
-        db.commit()
-
-    return instances
-
-
-def get_manual_task_instances_for_privacy_request(
-    db: Session, privacy_request: PrivacyRequest
-) -> list[ManualTaskInstance]:
-    """Get all manual task instances for a privacy request."""
-    return (
-        db.query(ManualTaskInstance)
-        .filter(
-            ManualTaskInstance.entity_id == privacy_request.id,
-            ManualTaskInstance.entity_type == "privacy_request",
-        )
-        .all()
-    )
-
-
 def create_manual_task_artificial_graphs(
     db: Session,
 ) -> list:
@@ -254,11 +164,18 @@ def create_manual_task_artificial_graphs(
         List of GraphDataset objects representing manual tasks as root nodes
     """
 
+    logger.debug("Creating manual task artificial graphs")
     manual_task_graphs = []
     manual_addresses = get_manual_task_addresses(db)
+    logger.debug(
+        f"Found {len(manual_addresses)} manual task addresses: {manual_addresses}"
+    )
 
     for address in manual_addresses:
         connection_key = address.dataset
+        logger.debug(
+            f"Processing manual task address: {address} for connection: {connection_key}"
+        )
 
         # Get manual tasks for this connection to determine fields
         manual_task = get_manual_task_for_connection_config(db, connection_key)
@@ -268,28 +185,47 @@ def create_manual_task_artificial_graphs(
 
         # Manual task collections act as root nodes - they don't need identity dependencies
         # since they provide manually-entered data rather than consuming identity data.
-        current_configs = [
-            config for config in manual_task.configs if config.is_current
-        ]
-        for config in current_configs:
-            if config.config_type not in [
-                ManualTaskConfigurationType.access_privacy_request,
-                ManualTaskConfigurationType.erasure_privacy_request,
-            ]:
-                continue
-
-            for field in config.field_definitions:
-                # Create a scalar field for each manual task field
-                field_metadata = field.field_metadata or {}
-                data_categories = field_metadata.get("data_categories", [])
-
-                scalar_field = ScalarField(
-                    name=field.field_key,
-                    data_categories=data_categories,
+        if manual_task:
+            logger.debug(
+                f"Processing manual task {manual_task.id} with {len(manual_task.configs)} configs"
+            )
+            current_configs = [
+                config
+                for config in manual_task.configs
+                if config.is_current
+                and config.config_type
+                in [
+                    ManualTaskConfigurationType.access_privacy_request,
+                    ManualTaskConfigurationType.erasure_privacy_request,
+                ]
+            ]
+            logger.debug(
+                f"Found {len(current_configs)} current configs for manual task {manual_task.id}"
+            )
+
+            for config in current_configs:
+                logger.debug(
+                    f"Processing config {config.id} with {len(config.field_definitions)} fields"
                 )
-                fields.append(scalar_field)
+                for field in config.field_definitions:
+                    # Create a scalar field for each manual task field
+                    field_metadata = field.field_metadata or {}
+                    data_categories = field_metadata.get("data_categories", [])
+
+                    scalar_field = ScalarField(
+                        name=field.field_key,
+                        data_categories=data_categories,
+                    )
+                    fields.append(scalar_field)
+        else:
+            logger.warning(
+                f"No manual task found for connection {connection_key}, skipping"
+            )
 
         if fields:  # Only create graph if there are fields
+            logger.debug(
+                f"Creating graph for connection {connection_key} with {len(fields)} fields"
+            )
             # Create a synthetic Collection
             collection = Collection(
                 name=ManualTaskAddress.MANUAL_DATA_COLLECTION,
@@ -307,5 +243,13 @@ def create_manual_task_artificial_graphs(
             )
 
             manual_task_graphs.append(graph_dataset)
+            logger.debug(
+                f"Successfully created manual task graph for connection {connection_key}"
+            )
+        else:
+            logger.warning(
+                f"No fields found for connection {connection_key}, skipping graph creation"
+            )
 
+    logger.debug(f"Created {len(manual_task_graphs)} manual task graphs")
     return manual_task_graphs

fides/api/util/cache.py

@@ -334,6 +334,62 @@ def cache_task_tracking_key(request_id: str, celery_task_id: str) -> None:
         )
 
 
+def get_privacy_request_retry_cache_key(privacy_request_id: str) -> str:
+    """Get cache key for tracking privacy request requeue retry attempts."""
+    return f"id-{privacy_request_id}-privacy-request-retry-count"
+
+
+def get_privacy_request_retry_count(privacy_request_id: str) -> int:
+    """Get the current retry count for a privacy request requeue attempts.
+
+    Raises Exception if cache operations fail, allowing callers to handle cache failures appropriately.
+    """
+    cache: FidesopsRedis = get_cache()
+    try:
+        retry_count = cache.get(get_privacy_request_retry_cache_key(privacy_request_id))
+        return int(retry_count) if retry_count else 0
+    except Exception as exc:
+        logger.error(
+            f"Failed to get retry count for privacy request {privacy_request_id}: {exc}"
+        )
+        raise
+
+
+def increment_privacy_request_retry_count(privacy_request_id: str) -> int:
+    """Increment and return the retry count for a privacy request requeue attempts.
+
+    Raises Exception if cache operations fail, allowing callers to handle cache failures appropriately.
+    """
+    cache: FidesopsRedis = get_cache()
+    cache_key = get_privacy_request_retry_cache_key(privacy_request_id)
+
+    try:
+        # Increment the counter, will be 1 if key doesn't exist
+        new_count = cache.incr(cache_key)
+        # Set expiry to prevent cache buildup (24 hours)
+        cache.expire(cache_key, 86400)
+        return new_count
+    except Exception as exc:
+        logger.error(
+            f"Failed to increment retry count for privacy request {privacy_request_id}: {exc}"
+        )
+        raise
+
+
+def reset_privacy_request_retry_count(privacy_request_id: str) -> None:
+    """Reset the retry count for a privacy request requeue attempts.
+
+    Silently fails if cache operations fail since this is cleanup.
+    """
+    cache: FidesopsRedis = get_cache()
+    try:
+        cache.delete(get_privacy_request_retry_cache_key(privacy_request_id))
+    except Exception as exc:
+        logger.warning(
+            f"Failed to reset retry count for privacy request {privacy_request_id}: {exc}"
+        )
+
+
 def celery_tasks_in_flight(celery_task_ids: List[str]) -> bool:
     """Returns True if supplied Celery Tasks appear to be in-flight"""
     if not celery_task_ids: