ethyca-fides 2.67.1b1__py2.py3-none-any.whl → 2.67.1rc1__py2.py3-none-any.whl

fides/api/task/manual/manual_task_address.py

@@ -1,5 +1,3 @@
-from typing import Union
-
 from fides.api.graph.config import CollectionAddress
 
 
@@ -22,7 +20,7 @@ class ManualTaskAddress:
         return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
 
     @staticmethod
-    def is_manual_task_address(address: Union[str, CollectionAddress]) -> bool:
+    def is_manual_task_address(address: CollectionAddress) -> bool:
         """Check if address represents manual task data"""
         if isinstance(address, str):
             # Handle string format "connection_key:collection_name"
@@ -35,7 +33,7 @@ class ManualTaskAddress:
         return ManualTaskAddress._is_manual_data_collection(address.collection)
 
     @staticmethod
-    def get_connection_key(address: Union[str, CollectionAddress]) -> str:
+    def get_connection_key(address: CollectionAddress) -> str:
         """Extract connection config key from manual task address"""
         if not ManualTaskAddress.is_manual_task_address(address):
             raise ValueError(f"Not a manual task address: {address}")

fides/api/task/manual/manual_task_graph_task.py

@@ -7,11 +7,11 @@ from fides.api.common_exceptions import AwaitingAsyncTaskCallback
 from fides.api.models.attachment import AttachmentType
 from fides.api.models.manual_task import (
     ManualTask,
+    ManualTaskConfig,
     ManualTaskConfigurationType,
     ManualTaskEntityType,
     ManualTaskFieldType,
     ManualTaskInstance,
-    ManualTaskSubmission,
     StatusType,
 )
 from fides.api.models.privacy_request import PrivacyRequest
@@ -23,7 +23,6 @@ from fides.api.task.manual.manual_task_utils import (
     get_manual_task_for_connection_config,
 )
 from fides.api.util.collection_util import Row
-from fides.api.util.storage_util import format_size
 
 
 class ManualTaskGraphTask(GraphTask):
@@ -123,36 +122,29 @@ class ManualTaskGraphTask(GraphTask):
         # request has started, while allowing different config types (access vs erasure)
         # to have separate instances.
         # ------------------------------------------------------------------
-        existing_task_instance = next(
-            (
-                instance
-                for instance in privacy_request.manual_task_instances
-                if instance.task_id == manual_task.id
-                and instance.config.config_type == allowed_config_type
-            ),
-            None,
+        existing_task_instance = (
+            db.query(ManualTaskInstance)
+            .join(ManualTaskInstance.config)  # Join to access config information
+            .filter(
+                ManualTaskInstance.task_id == manual_task.id,
+                ManualTaskInstance.entity_id == privacy_request.id,
+                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
+                # Only check for instances of the same config type
+                ManualTaskConfig.config_type == allowed_config_type,
+            )
+            .first()
         )
         if existing_task_instance:
             # An instance already exists for this privacy request and config type – no need
             # to create another one tied to a newer config version.
             return
 
-        # If no existing instances, create a new one for the current config
-        # There will only be one config of each type per manual task
-        config = next(
-            (
-                config
-                for config in sorted(
-                    manual_task.configs,
-                    key=lambda c: c.version if hasattr(c, "version") else 0,
-                    reverse=True,
-                )
-                if config.is_current and config.config_type == allowed_config_type
-            ),
-            None,
-        )
+        # Check each active config for instances (now we know none exist yet for this config type)
+        for config in manual_task.configs:
+            if not config.is_current or config.config_type != allowed_config_type:
+                # Skip configs that are not current or not relevant for this request type
+                continue
 
-        if config:
             ManualTaskInstance.create(
                 db=db,
                 data={
@@ -164,6 +156,7 @@ class ManualTaskGraphTask(GraphTask):
                 },
             )
 
+    # pylint: disable=too-many-branches,too-many-nested-blocks
     def _get_submitted_data(
         self,
         db: Session,
@@ -175,90 +168,93 @@ class ManualTaskGraphTask(GraphTask):
         Check if all manual task instances have submissions for ALL fields and return aggregated data
         Returns None if any field submissions are missing (all fields must be completed or skipped)
         """
-        candidate_instances: list[ManualTaskInstance] = [
-            instance
-            for instance in privacy_request.manual_task_instances
-            if instance.task_id == manual_task.id
-            and instance.config.config_type == allowed_config_type
-        ]
+        aggregated_data: dict[str, Any] = {}
+
+        def _format_size(size_bytes: int) -> str:
+            units = ["B", "KB", "MB", "GB", "TB"]
+            size = float(size_bytes)
+            for unit in units:
+                if size < 1024.0:
+                    return f"{size:.1f} {unit}"
+                size /= 1024.0
+            return f"{size:.1f} PB"
+
+        candidate_instances: list[ManualTaskInstance] = (
+            db.query(ManualTaskInstance)
+            .filter(
+                ManualTaskInstance.task_id == manual_task.id,
+                ManualTaskInstance.entity_id == privacy_request.id,
+                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
+            )
+            .all()
+        )
 
         if not candidate_instances:
             return None  # No instance yet for this manual task
 
-        # Check for incomplete fields and update status in single pass
         for inst in candidate_instances:
-            if inst.incomplete_fields:
+            # Skip instances tied to other request types
+            if not inst.config or inst.config.config_type != allowed_config_type:
+                continue
+
+            all_fields = inst.config.field_definitions or []
+
+            # Every field must have a submission
+            if not all(inst.get_submission_for_field(f.id) for f in all_fields):
                 return None  # At least one instance still incomplete
 
-            # Update status if needed
+            # Ensure status set
             if inst.status != StatusType.completed:
                 inst.status = StatusType.completed
                 inst.save(db)
 
-        # Aggregate submission data from all instances
-        aggregated_data = self._aggregate_submission_data(candidate_instances)
-        return aggregated_data or None
-
-    def _aggregate_submission_data(
-        self, instances: list[ManualTaskInstance]
-    ) -> dict[str, Any]:
-        """Aggregate submission data from all instances into a single dictionary."""
-        aggregated_data: dict[str, Any] = {}
-
-        for inst in instances:
-            # Filter valid submissions and process them
-            valid_submissions = (
-                submission
-                for submission in inst.submissions
-                if (
-                    submission.field
-                    and submission.field.field_key
-                    and isinstance(submission.data, dict)
-                )
-            )
+            # Aggregate submission data from this instance
+            for submission in inst.submissions:
+                if not submission.field or not submission.field.field_key:
+                    continue
 
-            for submission in valid_submissions:
                 field_key = submission.field.field_key
-                # We already checked isinstance(submission.data, dict) in valid_submissions
-                data_dict: dict[str, Any] = submission.data  # type: ignore[assignment]
-                field_type = data_dict.get("field_type")
 
-                # Process field data based on type
-                aggregated_data[field_key] = (
-                    self._process_attachment_field(submission)
-                    if field_type == ManualTaskFieldType.attachment.value
-                    else data_dict.get("value")
-                )
+                if not isinstance(submission.data, dict):
+                    continue
 
-        return aggregated_data
+                data_dict: dict[str, Any] = submission.data
 
-    def _process_attachment_field(
-        self, submission: ManualTaskSubmission
-    ) -> Optional[dict[str, dict[str, Any]]]:
-        """Process attachment field and return attachment map or None."""
-        attachment_map: dict[str, dict[str, Any]] = {}
+                field_type = data_dict.get("field_type")
 
-        for attachment in filter(
-            lambda a: a.attachment_type == AttachmentType.include_with_access_package,
-            submission.attachments,
-        ):
-            try:
-                size, url = attachment.retrieve_attachment()
-                attachment_map[attachment.file_name] = {
-                    "url": str(url) if url else None,
-                    "size": (format_size(size) if size else "Unknown"),
-                }
-            except Exception as exc:  # pylint: disable=broad-exception-caught
-                logger.warning(
-                    f"Error retrieving attachment {attachment.file_name}: {str(exc)}"
-                )
-        return attachment_map or None
+                if field_type == ManualTaskFieldType.attachment.value:
+                    attachment_map: dict[str, dict[str, Any]] = {}
+                    for attachment in submission.attachments or []:
+                        if (
+                            attachment.attachment_type
+                            == AttachmentType.include_with_access_package
+                        ):
+                            try:
+                                size, url = attachment.retrieve_attachment()
+                                attachment_map[attachment.file_name] = {
+                                    "url": str(url) if url else None,
+                                    "size": (_format_size(size) if size else "Unknown"),
+                                }
+                            except (
+                                Exception
+                            ) as exc:  # pylint: disable=broad-exception-caught
+                                logger.warning(
+                                    "Error retrieving attachment {}: {}",
+                                    attachment.file_name,
+                                    str(exc),
+                                )
+
+                    aggregated_data[field_key] = attachment_map or None
+                else:
+                    aggregated_data[field_key] = data_dict.get("value")
+
+        return aggregated_data if aggregated_data else None
 
     def dry_run_task(self) -> int:
         """Return estimated row count for dry run - manual tasks don't have predictable counts"""
         return 1  # Placeholder - manual tasks generate variable data
 
-    # Provide erasure support for manual tasks
+    # NEW METHOD: Provide erasure support for manual tasks
     @retry(action_type=ActionType.erasure, default_return=0)
     def erasure_request(
         self,

fides/api/task/manual/manual_task_utils.py

@@ -1,4 +1,3 @@
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.graph.config import (
@@ -13,7 +12,15 @@ from fides.api.graph.traversal import TraversalNode
 from fides.api.models.connectionconfig import ConnectionConfig
 
 # Import application models
-from fides.api.models.manual_task import ManualTask, ManualTaskConfigurationType
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskConfig,
+    ManualTaskConfigurationType,
+    ManualTaskEntityType,
+    ManualTaskInstance,
+)
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import ActionType
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 
 
@@ -21,18 +28,13 @@ def get_connection_configs_with_manual_tasks(db: Session) -> list[ConnectionConf
     """
     Get all connection configs that have manual tasks.
     """
-    logger.info("Querying for connection configs with manual tasks")
-    connection_configs = (
+    return (
         db.query(ConnectionConfig)
         .join(ManualTask, ConnectionConfig.id == ManualTask.parent_entity_id)
         .filter(ManualTask.parent_entity_type == "connection_config")
         .filter(ConnectionConfig.disabled.is_(False))
         .all()
     )
-    logger.info(
-        f"Found {len(connection_configs)} connection configs with manual tasks: {[cc.key for cc in connection_configs]}"
-    )
-    return connection_configs
 
 
 def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
@@ -45,19 +47,12 @@ def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
     """
     # Get all connection configs that have manual tasks (excluding disabled ones)
     connection_configs_with_manual_tasks = get_connection_configs_with_manual_tasks(db)
-    logger.debug(
-        f"Found {len(connection_configs_with_manual_tasks)} connection configs with manual tasks"
-    )
 
     # Create addresses for all connections that have manual tasks
     manual_task_addresses = []
     for config in connection_configs_with_manual_tasks:
-        logger.info(f"Creating manual task address for connection config: {config.key}")
         manual_task_addresses.append(ManualTaskAddress.create(config.key))
 
-    logger.info(
-        f"Created {len(manual_task_addresses)} manual task addresses: {manual_task_addresses}"
-    )
     return manual_task_addresses
 
 
@@ -67,11 +62,7 @@ def get_manual_task_for_connection_config(
     """Get the ManualTask for a specific connection config,
     the manual task/connection config relationship is 1:1.
     """
-    logger.info(
-        f"Looking for manual task for connection config: {connection_config_key}"
-    )
-
-    manual_task = (
+    return (
         db.query(ManualTask)
         .join(ConnectionConfig, ManualTask.parent_entity_id == ConnectionConfig.id)
         .filter(
@@ -81,17 +72,6 @@ def get_manual_task_for_connection_config(
         .one_or_none()
     )
 
-    if manual_task:
-        logger.info(
-            f"Found manual task {manual_task.id} for connection {connection_config_key}"
-        )
-    else:
-        logger.warning(
-            f"No manual task found for connection config: {connection_config_key}"
-        )
-
-    return manual_task
-
 
 def create_manual_data_traversal_node(
     db: Session, address: CollectionAddress
@@ -142,6 +122,116 @@ def create_manual_data_traversal_node(
     return traversal_node
 
 
+def create_manual_task_instances_for_privacy_request(
+    db: Session, privacy_request: PrivacyRequest
+) -> list[ManualTaskInstance]:
+    """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
+    instances = []
+
+    # Get all connection configs that have manual tasks (excluding disabled ones)
+    connection_configs_with_manual_tasks = get_connection_configs_with_manual_tasks(db)
+
+    # Determine the privacy request type based on policy rules
+    has_access_rules = bool(
+        privacy_request.policy.get_rules_for_action(action_type=ActionType.access)
+    )
+    has_erasure_rules = bool(
+        privacy_request.policy.get_rules_for_action(action_type=ActionType.erasure)
+    )
+
+    for connection_config in connection_configs_with_manual_tasks:
+        manual_tasks = (
+            db.query(ManualTask)
+            .filter(
+                ManualTask.parent_entity_id == connection_config.id,
+                ManualTask.parent_entity_type == "connection_config",
+            )
+            .all()
+        )
+
+        for manual_task in manual_tasks:
+            # Get the active config for this manual task, filtered by request type
+            active_config_query = db.query(ManualTaskConfig).filter(
+                ManualTaskConfig.task_id == manual_task.id,
+                ManualTaskConfig.is_current.is_(True),
+            )
+
+            # Filter by configuration type based on privacy request type
+            if has_access_rules and has_erasure_rules:
+                # If both access and erasure rules exist, include both types
+                active_config_query = active_config_query.filter(
+                    ManualTaskConfig.config_type.in_(
+                        [
+                            ManualTaskConfigurationType.access_privacy_request,
+                            ManualTaskConfigurationType.erasure_privacy_request,
+                        ]
+                    )
+                )
+            elif has_access_rules:
+                # Only access rules - only include access configurations
+                active_config_query = active_config_query.filter(
+                    ManualTaskConfig.config_type
+                    == ManualTaskConfigurationType.access_privacy_request
+                )
+            elif has_erasure_rules:
+                # Only erasure rules - only include erasure configurations
+                active_config_query = active_config_query.filter(
+                    ManualTaskConfig.config_type
+                    == ManualTaskConfigurationType.erasure_privacy_request
+                )
+            else:
+                # No relevant rules - skip this manual task
+                continue
+
+            active_configs = active_config_query.all()
+
+            if not active_configs:
+                continue  # Skip if no active configs
+
+            # Create instances for each active config
+            for active_config in active_configs:
+                # Check if instance already exists for this config
+                existing_instance = (
+                    db.query(ManualTaskInstance)
+                    .filter(
+                        ManualTaskInstance.entity_id == privacy_request.id,
+                        ManualTaskInstance.entity_type == "privacy_request",
+                        ManualTaskInstance.task_id == manual_task.id,
+                        ManualTaskInstance.config_id == active_config.id,
+                    )
+                    .first()
+                )
+
+                if not existing_instance:
+                    instance = ManualTaskInstance(
+                        entity_id=privacy_request.id,
+                        entity_type=ManualTaskEntityType.privacy_request,
+                        task_id=manual_task.id,
+                        config_id=active_config.id,
+                    )
+                    db.add(instance)
+                    instances.append(instance)
+
+    if instances:
+        db.commit()
+
+    return instances
+
+
+def get_manual_task_instances_for_privacy_request(
+    db: Session, privacy_request: PrivacyRequest
+) -> list[ManualTaskInstance]:
+    """Get all manual task instances for a privacy request."""
+    return (
+        db.query(ManualTaskInstance)
+        .filter(
+            ManualTaskInstance.entity_id == privacy_request.id,
+            ManualTaskInstance.entity_type == "privacy_request",
+        )
+        .all()
+    )
+
+
 def create_manual_task_artificial_graphs(
     db: Session,
 ) -> list:
@@ -164,18 +254,11 @@ def create_manual_task_artificial_graphs(
         List of GraphDataset objects representing manual tasks as root nodes
     """
 
-    logger.debug("Creating manual task artificial graphs")
     manual_task_graphs = []
     manual_addresses = get_manual_task_addresses(db)
-    logger.debug(
-        f"Found {len(manual_addresses)} manual task addresses: {manual_addresses}"
-    )
 
     for address in manual_addresses:
         connection_key = address.dataset
-        logger.debug(
-            f"Processing manual task address: {address} for connection: {connection_key}"
-        )
 
         # Get manual tasks for this connection to determine fields
         manual_task = get_manual_task_for_connection_config(db, connection_key)
@@ -185,47 +268,28 @@ def create_manual_task_artificial_graphs(
 
         # Manual task collections act as root nodes - they don't need identity dependencies
         # since they provide manually-entered data rather than consuming identity data.
-        if manual_task:
-            logger.debug(
-                f"Processing manual task {manual_task.id} with {len(manual_task.configs)} configs"
-            )
-            current_configs = [
-                config
-                for config in manual_task.configs
-                if config.is_current
-                and config.config_type
-                in [
-                    ManualTaskConfigurationType.access_privacy_request,
-                    ManualTaskConfigurationType.erasure_privacy_request,
-                ]
-            ]
-            logger.debug(
-                f"Found {len(current_configs)} current configs for manual task {manual_task.id}"
-            )
-
-            for config in current_configs:
-                logger.debug(
-                    f"Processing config {config.id} with {len(config.field_definitions)} fields"
+        current_configs = [
+            config for config in manual_task.configs if config.is_current
+        ]
+        for config in current_configs:
+            if config.config_type not in [
+                ManualTaskConfigurationType.access_privacy_request,
+                ManualTaskConfigurationType.erasure_privacy_request,
+            ]:
+                continue
+
+            for field in config.field_definitions:
+                # Create a scalar field for each manual task field
+                field_metadata = field.field_metadata or {}
+                data_categories = field_metadata.get("data_categories", [])
+
+                scalar_field = ScalarField(
+                    name=field.field_key,
+                    data_categories=data_categories,
                 )
-                for field in config.field_definitions:
-                    # Create a scalar field for each manual task field
-                    field_metadata = field.field_metadata or {}
-                    data_categories = field_metadata.get("data_categories", [])
-
-                    scalar_field = ScalarField(
-                        name=field.field_key,
-                        data_categories=data_categories,
-                    )
-                    fields.append(scalar_field)
-        else:
-            logger.warning(
-                f"No manual task found for connection {connection_key}, skipping"
-            )
+                fields.append(scalar_field)
 
         if fields:  # Only create graph if there are fields
-            logger.debug(
-                f"Creating graph for connection {connection_key} with {len(fields)} fields"
-            )
             # Create a synthetic Collection
             collection = Collection(
                 name=ManualTaskAddress.MANUAL_DATA_COLLECTION,
@@ -243,13 +307,5 @@ def create_manual_task_artificial_graphs(
             )
 
             manual_task_graphs.append(graph_dataset)
-            logger.debug(
-                f"Successfully created manual task graph for connection {connection_key}"
-            )
-        else:
-            logger.warning(
-                f"No fields found for connection {connection_key}, skipping graph creation"
-            )
 
-    logger.debug(f"Created {len(manual_task_graphs)} manual task graphs")
     return manual_task_graphs

fides/api/util/storage_util.py

@@ -14,25 +14,6 @@ from fides.api.schemas.storage.storage_secrets_docs_only import possible_storage
 from fides.api.util.custom_json_encoder import CustomJSONEncoder
 
 
-def format_size(size_bytes: float) -> str:
-    """
-    Format size in bytes to human readable format.
-
-    Args:
-        size_bytes: Size in bytes
-
-    Returns:
-        Formatted string with appropriate unit (B, KB, MB, GB, TB, PB)
-    """
-    units = ["B", "KB", "MB", "GB", "TB"]
-    size = float(size_bytes)
-    for unit in units:
-        if size < 1024.0:
-            return f"{size:.1f} {unit}"
-        size /= 1024.0
-    return f"{size:.1f} PB"
-
-
 def get_schema_for_secrets(
     storage_type: Union[StorageType, str],
     secrets: possible_storage_secrets,

fides/ui-build/static/admin/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/d9924caa849931b3.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d9924caa849931b3.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-90e8ec1fc5c6455b.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-750d6bd16c971bb9.js" defer=""></script><script src="/_next/static/chunks/pages/404-2d803dab6a00f353.js" defer=""></script><script src="/_next/static/sFDtXdzSc8wR5M8bdSVpg/_buildManifest.js" defer=""></script><script src="/_next/static/sFDtXdzSc8wR5M8bdSVpg/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"sFDtXdzSc8wR5M8bdSVpg","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/d9924caa849931b3.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d9924caa849931b3.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-90e8ec1fc5c6455b.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-750d6bd16c971bb9.js" defer=""></script><script src="/_next/static/chunks/pages/404-2d803dab6a00f353.js" defer=""></script><script src="/_next/static/-pDvglWsgqfWeQixOy5zJ/_buildManifest.js" defer=""></script><script src="/_next/static/-pDvglWsgqfWeQixOy5zJ/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"-pDvglWsgqfWeQixOy5zJ","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>