ethyca-fides 2.67.2rc0__py2.py3-none-any.whl → 2.67.3b0__py2.py3-none-any.whl

fides/api/task/manual/manual_task_graph_task.py

@@ -7,11 +7,11 @@ from fides.api.common_exceptions import AwaitingAsyncTaskCallback
 from fides.api.models.attachment import AttachmentType
 from fides.api.models.manual_task import (
     ManualTask,
-    ManualTaskConfig,
     ManualTaskConfigurationType,
     ManualTaskEntityType,
     ManualTaskFieldType,
     ManualTaskInstance,
+    ManualTaskSubmission,
     StatusType,
 )
 from fides.api.models.privacy_request import PrivacyRequest
@@ -23,6 +23,7 @@ from fides.api.task.manual.manual_task_utils import (
     get_manual_task_for_connection_config,
 )
 from fides.api.util.collection_util import Row
+from fides.api.util.storage_util import format_size
 
 
 class ManualTaskGraphTask(GraphTask):
@@ -122,29 +123,36 @@ class ManualTaskGraphTask(GraphTask):
         # request has started, while allowing different config types (access vs erasure)
         # to have separate instances.
         # ------------------------------------------------------------------
-        existing_task_instance = (
-            db.query(ManualTaskInstance)
-            .join(ManualTaskInstance.config)  # Join to access config information
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
-                # Only check for instances of the same config type
-                ManualTaskConfig.config_type == allowed_config_type,
-            )
-            .first()
+        existing_task_instance = next(
+            (
+                instance
+                for instance in privacy_request.manual_task_instances
+                if instance.task_id == manual_task.id
+                and instance.config.config_type == allowed_config_type
+            ),
+            None,
         )
         if existing_task_instance:
             # An instance already exists for this privacy request and config type – no need
             # to create another one tied to a newer config version.
             return
 
-        # Check each active config for instances (now we know none exist yet for this config type)
-        for config in manual_task.configs:
-            if not config.is_current or config.config_type != allowed_config_type:
-                # Skip configs that are not current or not relevant for this request type
-                continue
+        # If no existing instances, create a new one for the current config
+        # There will only be one config of each type per manual task
+        config = next(
+            (
+                config
+                for config in sorted(
+                    manual_task.configs,
+                    key=lambda c: c.version if hasattr(c, "version") else 0,
+                    reverse=True,
+                )
+                if config.is_current and config.config_type == allowed_config_type
+            ),
+            None,
+        )
 
+        if config:
             ManualTaskInstance.create(
                 db=db,
                 data={
@@ -156,7 +164,6 @@ class ManualTaskGraphTask(GraphTask):
                 },
             )
 
-    # pylint: disable=too-many-branches,too-many-nested-blocks
     def _get_submitted_data(
         self,
         db: Session,
@@ -168,93 +175,90 @@ class ManualTaskGraphTask(GraphTask):
         Check if all manual task instances have submissions for ALL fields and return aggregated data
         Returns None if any field submissions are missing (all fields must be completed or skipped)
         """
-        aggregated_data: dict[str, Any] = {}
-
-        def _format_size(size_bytes: int) -> str:
-            units = ["B", "KB", "MB", "GB", "TB"]
-            size = float(size_bytes)
-            for unit in units:
-                if size < 1024.0:
-                    return f"{size:.1f} {unit}"
-                size /= 1024.0
-            return f"{size:.1f} PB"
-
-        candidate_instances: list[ManualTaskInstance] = (
-            db.query(ManualTaskInstance)
-            .filter(
-                ManualTaskInstance.task_id == manual_task.id,
-                ManualTaskInstance.entity_id == privacy_request.id,
-                ManualTaskInstance.entity_type == ManualTaskEntityType.privacy_request,
-            )
-            .all()
-        )
+        candidate_instances: list[ManualTaskInstance] = [
+            instance
+            for instance in privacy_request.manual_task_instances
+            if instance.task_id == manual_task.id
+            and instance.config.config_type == allowed_config_type
+        ]
 
         if not candidate_instances:
             return None  # No instance yet for this manual task
 
+        # Check for incomplete fields and update status in single pass
         for inst in candidate_instances:
-            # Skip instances tied to other request types
-            if not inst.config or inst.config.config_type != allowed_config_type:
-                continue
-
-            all_fields = inst.config.field_definitions or []
-
-            # Every field must have a submission
-            if not all(inst.get_submission_for_field(f.id) for f in all_fields):
+            if inst.incomplete_fields:
                 return None  # At least one instance still incomplete
 
-            # Ensure status set
+            # Update status if needed
             if inst.status != StatusType.completed:
                 inst.status = StatusType.completed
                 inst.save(db)
 
-            # Aggregate submission data from this instance
-            for submission in inst.submissions:
-                if not submission.field or not submission.field.field_key:
-                    continue
+        # Aggregate submission data from all instances
+        aggregated_data = self._aggregate_submission_data(candidate_instances)
+        return aggregated_data or None
+
+    def _aggregate_submission_data(
+        self, instances: list[ManualTaskInstance]
+    ) -> dict[str, Any]:
+        """Aggregate submission data from all instances into a single dictionary."""
+        aggregated_data: dict[str, Any] = {}
+
+        for inst in instances:
+            # Filter valid submissions and process them
+            valid_submissions = (
+                submission
+                for submission in inst.submissions
+                if (
+                    submission.field
+                    and submission.field.field_key
+                    and isinstance(submission.data, dict)
+                )
+            )
 
+            for submission in valid_submissions:
                 field_key = submission.field.field_key
+                # We already checked isinstance(submission.data, dict) in valid_submissions
+                data_dict: dict[str, Any] = submission.data  # type: ignore[assignment]
+                field_type = data_dict.get("field_type")
 
-                if not isinstance(submission.data, dict):
-                    continue
+                # Process field data based on type
+                aggregated_data[field_key] = (
+                    self._process_attachment_field(submission)
+                    if field_type == ManualTaskFieldType.attachment.value
+                    else data_dict.get("value")
+                )
 
-                data_dict: dict[str, Any] = submission.data
+        return aggregated_data
 
-                field_type = data_dict.get("field_type")
+    def _process_attachment_field(
+        self, submission: ManualTaskSubmission
+    ) -> Optional[dict[str, dict[str, Any]]]:
+        """Process attachment field and return attachment map or None."""
+        attachment_map: dict[str, dict[str, Any]] = {}
 
-                if field_type == ManualTaskFieldType.attachment.value:
-                    attachment_map: dict[str, dict[str, Any]] = {}
-                    for attachment in submission.attachments or []:
-                        if (
-                            attachment.attachment_type
-                            == AttachmentType.include_with_access_package
-                        ):
-                            try:
-                                size, url = attachment.retrieve_attachment()
-                                attachment_map[attachment.file_name] = {
-                                    "url": str(url) if url else None,
-                                    "size": (_format_size(size) if size else "Unknown"),
-                                }
-                            except (
-                                Exception
-                            ) as exc:  # pylint: disable=broad-exception-caught
-                                logger.warning(
-                                    "Error retrieving attachment {}: {}",
-                                    attachment.file_name,
-                                    str(exc),
-                                )
-
-                    aggregated_data[field_key] = attachment_map or None
-                else:
-                    aggregated_data[field_key] = data_dict.get("value")
-
-        return aggregated_data if aggregated_data else None
+        for attachment in filter(
+            lambda a: a.attachment_type == AttachmentType.include_with_access_package,
+            submission.attachments,
+        ):
+            try:
+                size, url = attachment.retrieve_attachment()
+                attachment_map[attachment.file_name] = {
+                    "url": str(url) if url else None,
+                    "size": (format_size(size) if size else "Unknown"),
+                }
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                logger.warning(
+                    f"Error retrieving attachment {attachment.file_name}: {str(exc)}"
+                )
+        return attachment_map or None
 
     def dry_run_task(self) -> int:
         """Return estimated row count for dry run - manual tasks don't have predictable counts"""
         return 1  # Placeholder - manual tasks generate variable data
 
-    # NEW METHOD: Provide erasure support for manual tasks
+    # Provide erasure support for manual tasks
     @retry(action_type=ActionType.erasure, default_return=0)
     def erasure_request(
         self,

fides/api/task/manual/manual_task_utils.py

@@ -1,40 +1,43 @@
+from typing import Optional
+
+from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.graph.config import (
     Collection,
     CollectionAddress,
-    Field,
+    FieldAddress,
     GraphDataset,
     ScalarField,
 )
-from fides.api.graph.graph import Node
-from fides.api.graph.traversal import TraversalNode
 from fides.api.models.connectionconfig import ConnectionConfig
 
 # Import application models
 from fides.api.models.manual_task import (
     ManualTask,
-    ManualTaskConfig,
+    ManualTaskConditionalDependencyType,
     ManualTaskConfigurationType,
-    ManualTaskEntityType,
-    ManualTaskInstance,
 )
-from fides.api.models.privacy_request import PrivacyRequest
-from fides.api.schemas.policy import ActionType
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 
+PRIVACY_REQUEST_CONFIG_TYPES = {
+    ManualTaskConfigurationType.access_privacy_request,
+    ManualTaskConfigurationType.erasure_privacy_request,
+}
+
 
 def get_connection_configs_with_manual_tasks(db: Session) -> list[ConnectionConfig]:
     """
     Get all connection configs that have manual tasks.
     """
-    return (
+    connection_configs = (
         db.query(ConnectionConfig)
         .join(ManualTask, ConnectionConfig.id == ManualTask.parent_entity_id)
         .filter(ManualTask.parent_entity_type == "connection_config")
         .filter(ConnectionConfig.disabled.is_(False))
         .all()
     )
+    return connection_configs
 
 
 def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
@@ -48,12 +51,11 @@ def get_manual_task_addresses(db: Session) -> list[CollectionAddress]:
     # Get all connection configs that have manual tasks (excluding disabled ones)
     connection_configs_with_manual_tasks = get_connection_configs_with_manual_tasks(db)
 
-    # Create addresses for all connections that have manual tasks
-    manual_task_addresses = []
-    for config in connection_configs_with_manual_tasks:
-        manual_task_addresses.append(ManualTaskAddress.create(config.key))
-
-    return manual_task_addresses
+    # Return addresses for all connections that have manual tasks
+    return [
+        ManualTaskAddress.create(config.key)
+        for config in connection_configs_with_manual_tasks
+    ]
 
 
 def get_manual_task_for_connection_config(
@@ -73,20 +75,18 @@ def get_manual_task_for_connection_config(
     )
 
 
-def create_manual_data_traversal_node(
-    db: Session, address: CollectionAddress
-) -> "TraversalNode":
+def create_data_category_scalar_fields(manual_task: ManualTask) -> list[ScalarField]:
     """
-    Create a TraversalNode for a manual_data collection
+    Create scalar fields for each field in the given manual task configs.
     """
-    connection_key = address.dataset
-
-    # Get manual tasks for this connection to determine fields
-    manual_task = get_manual_task_for_connection_config(db, connection_key)
-
-    # Create fields based on ManualTaskConfigFields
-    fields: list[Field] = []
-    for config in manual_task.configs:
+    fields = []
+    # Get current privacy request configs for this manual task
+    current_configs = [
+        config
+        for config in manual_task.configs
+        if config.is_current and config.config_type in PRIVACY_REQUEST_CONFIG_TYPES
+    ]
+    for config in current_configs:
         for field in config.field_definitions:
             # Create a scalar field for each manual task field
             # Extract data categories from field metadata if available
@@ -99,211 +99,94 @@ def create_manual_data_traversal_node(
                 # Manual task fields don't have complex relationships
             )
             fields.append(scalar_field)
+    return fields
+
+
+def create_conditional_dependency_scalar_fields(
+    field_addresses: set[str],
+) -> list[ScalarField]:
+    fields: list[ScalarField] = []
+    for field_address in field_addresses:
+        # Use the full field address as the field name to preserve collection context
+        # This allows the manual task to receive data from specific collections
+        # e.g., "user.name" or "customer.profile.email" instead of just "name" or "email"
+        logger.info(
+            f"Creating conditional dependency scalar field for field address: {field_address}"
+        )
+        field_address_obj = FieldAddress.from_string(field_address)
 
-    # Create a synthetic Collection
-    collection = Collection(
-        name=ManualTaskAddress.MANUAL_DATA_COLLECTION,
-        fields=fields,
-        # Manual tasks don't have complex dependencies
-        after=set(),
-    )
-
-    # Create a synthetic GraphDataset
-    dataset = GraphDataset(
-        name=connection_key,
-        collections=[collection],
-        connection_key=connection_key,
-        after=set(),
-    )
-
-    node = Node(dataset, collection)
-    traversal_node = TraversalNode(node)
-
-    return traversal_node
+        scalar_field = ScalarField(
+            name=field_address_obj.value,
+            # Conditional dependency fields don't have predefined data categories
+            data_categories=[],
+            references=[(field_address_obj, "from")],
+        )
+        fields.append(scalar_field)
 
+    return fields
 
-def create_manual_task_instances_for_privacy_request(
-    db: Session, privacy_request: PrivacyRequest
-) -> list[ManualTaskInstance]:
-    """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
-    instances = []
 
-    # Get all connection configs that have manual tasks (excluding disabled ones)
-    connection_configs_with_manual_tasks = get_connection_configs_with_manual_tasks(db)
+def create_collection_for_connection_key(
+    db: Session, connection_key: str
+) -> Optional[Collection]:
+    # Get the manual task for this connection config
+    manual_task = get_manual_task_for_connection_config(db, connection_key)
 
-    # Determine the privacy request type based on policy rules
-    has_access_rules = bool(
-        privacy_request.policy.get_rules_for_action(action_type=ActionType.access)
-    )
-    has_erasure_rules = bool(
-        privacy_request.policy.get_rules_for_action(action_type=ActionType.erasure)
+    if not manual_task:
+        return None
+
+    # Get conditional dependency field addresses - raw field data
+    conditional_field_addresses: set[str] = {
+        dependency.field_address
+        for dependency in manual_task.conditional_dependencies
+        if dependency.condition_type == ManualTaskConditionalDependencyType.leaf
+        and dependency.field_address is not None
+    }
+
+    # Create scalar fields for data category fields and conditional dependency field addresses
+    fields: list[ScalarField] = []
+    fields.extend(create_data_category_scalar_fields(manual_task))
+    fields.extend(
+        create_conditional_dependency_scalar_fields(conditional_field_addresses)
     )
 
-    for connection_config in connection_configs_with_manual_tasks:
-        manual_tasks = (
-            db.query(ManualTask)
-            .filter(
-                ManualTask.parent_entity_id == connection_config.id,
-                ManualTask.parent_entity_type == "connection_config",
-            )
-            .all()
-        )
+    # Only create collection if there are fields
+    if not fields:
+        return None
 
-        for manual_task in manual_tasks:
-            # Get the active config for this manual task, filtered by request type
-            active_config_query = db.query(ManualTaskConfig).filter(
-                ManualTaskConfig.task_id == manual_task.id,
-                ManualTaskConfig.is_current.is_(True),
-            )
-
-            # Filter by configuration type based on privacy request type
-            if has_access_rules and has_erasure_rules:
-                # If both access and erasure rules exist, include both types
-                active_config_query = active_config_query.filter(
-                    ManualTaskConfig.config_type.in_(
-                        [
-                            ManualTaskConfigurationType.access_privacy_request,
-                            ManualTaskConfigurationType.erasure_privacy_request,
-                        ]
-                    )
-                )
-            elif has_access_rules:
-                # Only access rules - only include access configurations
-                active_config_query = active_config_query.filter(
-                    ManualTaskConfig.config_type
-                    == ManualTaskConfigurationType.access_privacy_request
-                )
-            elif has_erasure_rules:
-                # Only erasure rules - only include erasure configurations
-                active_config_query = active_config_query.filter(
-                    ManualTaskConfig.config_type
-                    == ManualTaskConfigurationType.erasure_privacy_request
-                )
-            else:
-                # No relevant rules - skip this manual task
-                continue
-
-            active_configs = active_config_query.all()
-
-            if not active_configs:
-                continue  # Skip if no active configs
-
-            # Create instances for each active config
-            for active_config in active_configs:
-                # Check if instance already exists for this config
-                existing_instance = (
-                    db.query(ManualTaskInstance)
-                    .filter(
-                        ManualTaskInstance.entity_id == privacy_request.id,
-                        ManualTaskInstance.entity_type == "privacy_request",
-                        ManualTaskInstance.task_id == manual_task.id,
-                        ManualTaskInstance.config_id == active_config.id,
-                    )
-                    .first()
-                )
-
-                if not existing_instance:
-                    instance = ManualTaskInstance(
-                        entity_id=privacy_request.id,
-                        entity_type=ManualTaskEntityType.privacy_request,
-                        task_id=manual_task.id,
-                        config_id=active_config.id,
-                    )
-                    db.add(instance)
-                    instances.append(instance)
-
-    if instances:
-        db.commit()
-
-    return instances
-
-
-def get_manual_task_instances_for_privacy_request(
-    db: Session, privacy_request: PrivacyRequest
-) -> list[ManualTaskInstance]:
-    """Get all manual task instances for a privacy request."""
-    return (
-        db.query(ManualTaskInstance)
-        .filter(
-            ManualTaskInstance.entity_id == privacy_request.id,
-            ManualTaskInstance.entity_type == "privacy_request",
-        )
-        .all()
-    )
+    return Collection(name=ManualTaskAddress.MANUAL_DATA_COLLECTION, fields=fields)
 
 
-def create_manual_task_artificial_graphs(
-    db: Session,
-) -> list:
+def create_manual_task_artificial_graphs(db: Session) -> list[GraphDataset]:
     """
     Create artificial GraphDataset objects for manual tasks that can be included
     in the main dataset graph during the dataset configuration phase.
 
-    Manual tasks should be treated as data sources/datasets rather than being
-    appended to the traversal graph later.
-
-    Manual task collections are designed as root nodes that execute immediately when
-    the privacy request starts, in parallel with identity processing. They don't depend
-    on identity data since they provide manually-entered data rather than consuming it.
+    Each manual task gets its own collection with its own dependencies based on
+    its specific conditional dependencies. This allows individual manual tasks
+    to receive only the data they need from regular tasks.
 
     Args:
         db: Database session
-        policy: The policy being executed (optional, for filtering manual task configs)
 
     Returns:
-        List of GraphDataset objects representing manual tasks as root nodes
+        List of GraphDataset objects representing manual tasks as individual collections
     """
-
     manual_task_graphs = []
     manual_addresses = get_manual_task_addresses(db)
 
     for address in manual_addresses:
         connection_key = address.dataset
 
-        # Get manual tasks for this connection to determine fields
-        manual_task = get_manual_task_for_connection_config(db, connection_key)
-
-        # Create fields based only on ManualTaskConfigFields
-        fields: list = []
-
-        # Manual task collections act as root nodes - they don't need identity dependencies
-        # since they provide manually-entered data rather than consuming identity data.
-        current_configs = [
-            config for config in manual_task.configs if config.is_current
-        ]
-        for config in current_configs:
-            if config.config_type not in [
-                ManualTaskConfigurationType.access_privacy_request,
-                ManualTaskConfigurationType.erasure_privacy_request,
-            ]:
-                continue
-
-            for field in config.field_definitions:
-                # Create a scalar field for each manual task field
-                field_metadata = field.field_metadata or {}
-                data_categories = field_metadata.get("data_categories", [])
-
-                scalar_field = ScalarField(
-                    name=field.field_key,
-                    data_categories=data_categories,
-                )
-                fields.append(scalar_field)
-
-        if fields:  # Only create graph if there are fields
-            # Create a synthetic Collection
-            collection = Collection(
-                name=ManualTaskAddress.MANUAL_DATA_COLLECTION,
-                fields=fields,
-                # Manual tasks have no dependencies - they're root nodes
-                after=set(),
-            )
+        # Get the collection for this connection config using the reusable function
+        collection = create_collection_for_connection_key(db, connection_key)
 
-            # Create a synthetic GraphDataset
+        if collection:  # Only create graph if there are collections
+            # Create a synthetic GraphDataset with all manual task collections
             graph_dataset = GraphDataset(
                 name=connection_key,
                 collections=[collection],
                 connection_key=connection_key,
-                after=set(),
             )
 
             manual_task_graphs.append(graph_dataset)

fides/api/util/storage_util.py

@@ -14,6 +14,25 @@ from fides.api.schemas.storage.storage_secrets_docs_only import possible_storage
 from fides.api.util.custom_json_encoder import CustomJSONEncoder
 
 
+def format_size(size_bytes: float) -> str:
+    """
+    Format size in bytes to human readable format.
+
+    Args:
+        size_bytes: Size in bytes
+
+    Returns:
+        Formatted string with appropriate unit (B, KB, MB, GB, TB, PB)
+    """
+    units = ["B", "KB", "MB", "GB", "TB"]
+    size = float(size_bytes)
+    for unit in units:
+        if size < 1024.0:
+            return f"{size:.1f} {unit}"
+        size /= 1024.0
+    return f"{size:.1f} PB"
+
+
 def get_schema_for_secrets(
     storage_type: Union[StorageType, str],
     secrets: possible_storage_secrets,

fides/config/__init__.py

@@ -29,6 +29,7 @@ from .fides_settings import FidesSettings
 from .helpers import handle_deprecated_env_variables, handle_deprecated_fields
 from .logging_settings import LoggingSettings
 from .notification_settings import NotificationSettings
+from .privacy_center_settings import PrivacyCenterSettings
 from .redis_settings import RedisSettings
 from .security_settings import SecuritySettings
 from .user_settings import UserSettings
@@ -82,6 +83,7 @@ class FidesConfig(FidesSettings):
     logging: LoggingSettings
     notifications: NotificationSettings
     redis: RedisSettings
+    privacy_center: PrivacyCenterSettings
     security: SecuritySettings
     user: UserSettings
 
@@ -111,6 +113,7 @@ class FidesConfig(FidesSettings):
             self.security,
             self.execution,
             self.admin_ui,
+            self.privacy_center,
         ]:
             for key, value in settings.model_dump(mode="json").items():  # type: ignore
                 log.debug(
@@ -161,6 +164,7 @@ def build_config(config_dict: Dict[str, Any]) -> FidesConfig:
         "execution": ExecutionSettings,
         "logging": LoggingSettings,
         "notifications": NotificationSettings,
+        "privacy_center": PrivacyCenterSettings,
         "redis": RedisSettings,
         "security": SecuritySettings,
         "user": UserSettings,