ethyca-fides 2.67.1rc1__py2.py3-none-any.whl → 2.67.2b1__py2.py3-none-any.whl

fides/api/models/privacy_request/privacy_request.py

@@ -47,8 +47,16 @@ from fides.api.models.attachment import (
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
 from fides.api.models.comment import Comment, CommentReference, CommentReferenceType
+from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.field_types import EncryptedLargeDataDescriptor
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskConfig,
+    ManualTaskConfigurationType,
+    ManualTaskEntityType,
+    ManualTaskInstance,
+)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.masking_secret import MaskingSecret
 from fides.api.models.policy import (
@@ -200,6 +208,14 @@ class PrivacyRequest(
         viewonly=True,
         uselist=True,
     )
+    manual_task_instances = relationship(
+        "ManualTaskInstance",
+        lazy="select",
+        passive_deletes="all",
+        primaryjoin="and_(ManualTaskInstance.entity_id==foreign(PrivacyRequest.id), "
+        "ManualTaskInstance.entity_type=='privacy_request')",
+        uselist=True,
+    )
     property_id = Column(String, nullable=True)
 
     cancel_reason = Column(String(200))
@@ -1175,6 +1191,68 @@ class PrivacyRequest(
             db, manual_webhook_id, "erasure_manual_webhook"
         )
 
+    def create_manual_task_instances(
+        self, db: Session, connection_configs_with_manual_tasks: list[ConnectionConfig]
+    ) -> list[ManualTaskInstance]:
+        """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
+        # Early return if no relevant policy rules
+        policy_rules = {
+            ActionType.access: bool(
+                self.policy.get_rules_for_action(action_type=ActionType.access)
+            ),
+            ActionType.erasure: bool(
+                self.policy.get_rules_for_action(action_type=ActionType.erasure)
+            ),
+        }
+
+        if not any(policy_rules.values()):
+            return []
+
+        # Build configuration types using list comprehension
+        config_types = [
+            (
+                ManualTaskConfigurationType.access_privacy_request
+                if action_type == ActionType.access
+                else ManualTaskConfigurationType.erasure_privacy_request
+            )
+            for action_type, has_rules in policy_rules.items()
+            if has_rules
+        ]
+
+        # Get all relevant manual tasks and configs in one query
+        connection_config_ids = [cc.id for cc in connection_configs_with_manual_tasks]
+        manual_tasks_with_configs = (
+            db.query(ManualTask, ManualTaskConfig)
+            .join(ManualTaskConfig, ManualTask.id == ManualTaskConfig.task_id)
+            .filter(
+                ManualTask.parent_entity_id.in_(connection_config_ids),
+                ManualTask.parent_entity_type == "connection_config",
+                ManualTaskConfig.is_current.is_(True),
+                ManualTaskConfig.config_type.in_(config_types),
+            )
+            .all()
+        )
+
+        # Get existing config IDs to avoid duplicates
+        existing_config_ids = {
+            instance.config_id for instance in self.manual_task_instances
+        }
+
+        # Create instances using list comprehension and filter out existing ones
+        return [
+            ManualTaskInstance.create(
+                db=db,
+                data={
+                    "entity_id": self.id,
+                    "entity_type": ManualTaskEntityType.privacy_request,
+                    "task_id": manual_task.id,
+                    "config_id": config.id,
+                },
+            )
+            for manual_task, config in manual_tasks_with_configs
+            if config.id not in existing_config_ids
+        ]
+
     def get_existing_request_task(
         self,
         db: Session,

fides/api/service/privacy_request/dsr_package/dsr_report_builder.py

@@ -13,7 +13,7 @@ from loguru import logger
 
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
-from fides.api.util.storage_util import StorageJSONEncoder
+from fides.api.util.storage_util import StorageJSONEncoder, format_size
 
 DSR_DIRECTORY = Path(__file__).parent.resolve()
 
@@ -204,7 +204,7 @@ class DsrReportBuilder:
 
             file_size = attachment.get("file_size")
             if isinstance(file_size, (int, float)):
-                file_size = self._format_size(float(file_size))
+                file_size = format_size(float(file_size))
             else:
                 file_size = "Unknown"
 
@@ -321,22 +321,6 @@ class DsrReportBuilder:
 
         return datasets
 
-    def _format_size(self, size_bytes: float) -> str:
-        """
-        Format size in bytes to human readable format.
-
-        Args:
-            size_bytes: Size in bytes
-
-        Returns:
-            Formatted string with appropriate unit (B, KB, MB, GB)
-        """
-        for unit in ["B", "KB", "MB", "GB"]:
-            if size_bytes < 1024.0:
-                return f"{size_bytes:.1f} {unit}"
-            size_bytes /= 1024.0
-        return f"{size_bytes:.1f} TB"
-
     def generate(self) -> BytesIO:
         """
         Processes the request and DSR data to build zip file containing the DSR report.
@@ -395,7 +379,7 @@ class DsrReportBuilder:
 
         # Calculate time taken and file size
         time_taken = time_module.time() - start_time
-        file_size = self._format_size(float(len(self.baos.getvalue())))
+        file_size = format_size(float(len(self.baos.getvalue())))
 
         logger.bind(time_to_generate=time_taken, dsr_package_size=file_size).info(
             "DSR report generation complete."

fides/api/task/create_request_tasks.py

@@ -35,7 +35,7 @@ from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    create_manual_task_instances_for_privacy_request,
+    get_connection_configs_with_manual_tasks,
 )
 from fides.api.util.logger_context_utils import log_context
 
@@ -92,7 +92,7 @@ def build_access_networkx_digraph(
     manual_nodes = [
         addr
         for addr in traversal_nodes.keys()
-        if addr.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
+        if ManualTaskAddress.is_manual_task_address(addr)
     ]
     for manual_node in manual_nodes:
         networkx_graph.add_edge(ROOT_COLLECTION_ADDRESS, manual_node)
@@ -472,7 +472,9 @@ def run_access_request(
             )
 
             # Snapshot manual task field instances for this privacy request
-            create_manual_task_instances_for_privacy_request(session, privacy_request)
+            privacy_request.create_manual_task_instances(
+                session, get_connection_configs_with_manual_tasks(session)
+            )
 
             # Save Access Request Tasks to the database
             ready_tasks = persist_new_access_request_tasks(

fides/api/task/filter_results.py

@@ -39,7 +39,7 @@ def filter_data_categories(
             continue
 
         # Skip manual task data - it doesn't need filtering since it's controlled by field definitions
-        if f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}" in node_address:
+        if ManualTaskAddress.is_manual_task_address(node_address):
             filtered_access_results[node_address].extend(results)
             continue
 

fides/api/task/manual/manual_task_address.py

@@ -1,3 +1,5 @@
+from typing import Union
+
 from fides.api.graph.config import CollectionAddress
 
 
@@ -20,7 +22,7 @@ class ManualTaskAddress:
         return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
 
     @staticmethod
-    def is_manual_task_address(address: CollectionAddress) -> bool:
+    def is_manual_task_address(address: Union[str, CollectionAddress]) -> bool:
         """Check if address represents manual task data"""
         if isinstance(address, str):
             # Handle string format "connection_key:collection_name"
@@ -33,7 +35,7 @@ class ManualTaskAddress:
         return ManualTaskAddress._is_manual_data_collection(address.collection)
 
     @staticmethod
-    def get_connection_key(address: CollectionAddress) -> str:
+    def get_connection_key(address: Union[str, CollectionAddress]) -> str:
         """Extract connection config key from manual task address"""
         if not ManualTaskAddress.is_manual_task_address(address):
             raise ValueError(f"Not a manual task address: {address}")

fides/api/task/manual/manual_task_graph_task.py

@@ -7,11 +7,11 @@ from fides.api.common_exceptions import AwaitingAsyncTaskCallback
 from fides.api.models.attachment import AttachmentType
 from fides.api.models.manual_task import (
     ManualTask,
-    ManualTaskConfig,
     ManualTaskConfigurationType,
     ManualTaskEntityType,
     ManualTaskFieldType,
     ManualTaskInstance,
+    ManualTaskSubmission,
     StatusType,
 )
 from fides.api.models.privacy_request import PrivacyRequest
@@ -23,6 +23,7 @@ from fides.api.task.manual.manual_task_utils import (
     get_manual_task_for_connection_config,
 )
 from fides.api.util.collection_util import Row
+from fides.api.util.storage_util import format_size
 
 
 class ManualTaskGraphTask(GraphTask):
@@ -122,29 +123,36 @@ class ManualTaskGraphTask(GraphTask):
         # request has started, while allowing different config types (access vs erasure)
         # to have separate instances.
         # ------------------------------------------------------------------
-        existing_task_instance = (
-            db.query(ManualTaskInstance)
-            .join(ManualTaskInstance.config)  # Join to access config information
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
-                # Only check for instances of the same config type
-                ManualTaskConfig.config_type == allowed_config_type,
-            )
-            .first()
+        existing_task_instance = next(
+            (
+                instance
+                for instance in privacy_request.manual_task_instances
+                if instance.task_id == manual_task.id
+                and instance.config.config_type == allowed_config_type
+            ),
+            None,
         )
         if existing_task_instance:
             # An instance already exists for this privacy request and config type – no need
             # to create another one tied to a newer config version.
             return
 
-        # Check each active config for instances (now we know none exist yet for this config type)
-        for config in manual_task.configs:
-            if not config.is_current or config.config_type != allowed_config_type:
-                # Skip configs that are not current or not relevant for this request type
-                continue
+        # If no existing instances, create a new one for the current config
+        # There will only be one config of each type per manual task
+        config = next(
+            (
+                config
+                for config in sorted(
+                    manual_task.configs,
+                    key=lambda c: c.version if hasattr(c, "version") else 0,
+                    reverse=True,
+                )
+                if config.is_current and config.config_type == allowed_config_type
+            ),
+            None,
+        )
 
+        if config:
             ManualTaskInstance.create(
                 db=db,
                 data={
@@ -156,7 +164,6 @@ class ManualTaskGraphTask(GraphTask):
                 },
             )
 
-    # pylint: disable=too-many-branches,too-many-nested-blocks
     def _get_submitted_data(
         self,
         db: Session,
@@ -168,93 +175,90 @@ class ManualTaskGraphTask(GraphTask):
         Check if all manual task instances have submissions for ALL fields and return aggregated data
         Returns None if any field submissions are missing (all fields must be completed or skipped)
         """
-        aggregated_data: dict[str, Any] = {}
-
-        def _format_size(size_bytes: int) -> str:
-            units = ["B", "KB", "MB", "GB", "TB"]
-            size = float(size_bytes)
-            for unit in units:
-                if size < 1024.0:
-                    return f"{size:.1f} {unit}"
-                size /= 1024.0
-            return f"{size:.1f} PB"
-
-        candidate_instances: list[ManualTaskInstance] = (
-            db.query(ManualTaskInstance)
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
-            )
-            .all()
-        )
+        candidate_instances: list[ManualTaskInstance] = [
+            instance
+            for instance in privacy_request.manual_task_instances
+            if instance.task_id == manual_task.id
+            and instance.config.config_type == allowed_config_type
+        ]
 
         if not candidate_instances:
             return None  # No instance yet for this manual task
 
+        # Check for incomplete fields and update status in single pass
         for inst in candidate_instances:
-            # Skip instances tied to other request types
-            if not inst.config or inst.config.config_type != allowed_config_type:
-                continue
-
-            all_fields = inst.config.field_definitions or []
-
-            # Every field must have a submission
-            if not all(inst.get_submission_for_field(f.id) for f in all_fields):
+            if inst.incomplete_fields:
                 return None  # At least one instance still incomplete
 
-            # Ensure status set
+            # Update status if needed
             if inst.status != StatusType.completed:
                 inst.status = StatusType.completed
                 inst.save(db)
 
-            # Aggregate submission data from this instance
-            for submission in inst.submissions:
-                if not submission.field or not submission.field.field_key:
-                    continue
+        # Aggregate submission data from all instances
+        aggregated_data = self._aggregate_submission_data(candidate_instances)
+        return aggregated_data or None
+
+    def _aggregate_submission_data(
+        self, instances: list[ManualTaskInstance]
+    ) -> dict[str, Any]:
+        """Aggregate submission data from all instances into a single dictionary."""
+        aggregated_data: dict[str, Any] = {}
+
+        for inst in instances:
+            # Filter valid submissions and process them
+            valid_submissions = (
+                submission
+                for submission in inst.submissions
+                if (
+                    submission.field
+                    and submission.field.field_key
+                    and isinstance(submission.data, dict)
+                )
+            )
 
+            for submission in valid_submissions:
                 field_key = submission.field.field_key
+                # We already checked isinstance(submission.data, dict) in valid_submissions
+                data_dict: dict[str, Any] = submission.data  # type: ignore[assignment]
+                field_type = data_dict.get("field_type")
 
-                if not isinstance(submission.data, dict):
-                    continue
+                # Process field data based on type
+                aggregated_data[field_key] = (
+                    self._process_attachment_field(submission)
+                    if field_type == ManualTaskFieldType.attachment.value
+                    else data_dict.get("value")
+                )
 
-                data_dict: dict[str, Any] = submission.data
+        return aggregated_data
 
-                field_type = data_dict.get("field_type")
+    def _process_attachment_field(
+        self, submission: ManualTaskSubmission
+    ) -> Optional[dict[str, dict[str, Any]]]:
+        """Process attachment field and return attachment map or None."""
+        attachment_map: dict[str, dict[str, Any]] = {}
 
-                if field_type == ManualTaskFieldType.attachment.value:
-                    attachment_map: dict[str, dict[str, Any]] = {}
-                    for attachment in submission.attachments or []:
-                        if (
-                            attachment.attachment_type
-                            == AttachmentType.include_with_access_package
-                        ):
-                            try:
-                                size, url = attachment.retrieve_attachment()
-                                attachment_map[attachment.file_name] = {
-                                    "url": str(url) if url else None,
-                                    "size": (_format_size(size) if size else "Unknown"),
-                                }
-                            except (
-                                Exception
-                            ) as exc:  # pylint: disable=broad-exception-caught
-                                logger.warning(
-                                    "Error retrieving attachment {}: {}",
-                                    attachment.file_name,
-                                    str(exc),
-                                )
-
-                    aggregated_data[field_key] = attachment_map or None
-                else:
-                    aggregated_data[field_key] = data_dict.get("value")
-
-        return aggregated_data if aggregated_data else None
+        for attachment in filter(
+            lambda a: a.attachment_type == AttachmentType.include_with_access_package,
+            submission.attachments,
+        ):
+            try:
+                size, url = attachment.retrieve_attachment()
+                attachment_map[attachment.file_name] = {
+                    "url": str(url) if url else None,
+                    "size": (format_size(size) if size else "Unknown"),
+                }
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                logger.warning(
+                    f"Error retrieving attachment {attachment.file_name}: {str(exc)}"
+                )
+        return attachment_map or None
 
     def dry_run_task(self) -> int:
         """Return estimated row count for dry run - manual tasks don't have predictable counts"""
         return 1  # Placeholder - manual tasks generate variable data
 
-    # NEW METHOD: Provide erasure support for manual tasks
+    # Provide erasure support for manual tasks
     @retry(action_type=ActionType.erasure, default_return=0)
     def erasure_request(
         self,