ethyca-fides 2.64.5rc4__py2.py3-none-any.whl → 2.64.6b1__py2.py3-none-any.whl

fides/api/task/manual/manual_task_graph_task.py

@@ -0,0 +1,300 @@
+from typing import Any, Dict, List, Optional
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.common_exceptions import AwaitingAsyncTaskCallback
+from fides.api.models.attachment import AttachmentType
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskConfigurationType,
+    ManualTaskEntityType,
+    ManualTaskFieldType,
+    ManualTaskInstance,
+    StatusType,
+)
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import ActionType
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
+from fides.api.task.graph_task import GraphTask, retry
+from fides.api.task.manual.manual_task_utils import (
+    ManualTaskAddress,
+    get_manual_tasks_for_connection_config,
+)
+from fides.api.util.collection_util import Row
+
+
+class ManualTaskGraphTask(GraphTask):
+    """GraphTask implementation for ManualTask execution"""
+
+    @retry(action_type=ActionType.access, default_return=[])
+    def access_request(self, *inputs: List[Row]) -> List[Row]:
+        """
+        Execute manual task logic following the standard GraphTask pattern:
+        1. Create ManualTaskInstances if they don't exist
+        2. Check for submissions
+        3. Return data if submitted, raise AwaitingAsyncTaskCallback if not
+        """
+        db = self.resources.session
+        collection_address = self.execution_node.address
+
+        # Verify this is a manual task address
+        if not ManualTaskAddress.is_manual_task_address(collection_address):
+            raise ValueError(f"Invalid manual task address: {collection_address}")
+
+        connection_key = ManualTaskAddress.get_connection_key(collection_address)
+
+        # Get manual tasks for this connection
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+
+        if not manual_tasks:
+            return []
+
+        # Check/create manual task instances for ACCESS configs only
+        self._ensure_manual_task_instances(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.access_privacy_request,
+        )
+
+        # Check if all manual task instances have submissions for ACCESS configs only
+        submitted_data = self._get_submitted_data(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.access_privacy_request,
+        )
+
+        if submitted_data is not None:
+            result: List[Row] = [submitted_data] if submitted_data else []
+            self.request_task.access_data = result
+
+            # Mark request task as complete and write execution log
+            self.log_end(ActionType.access)
+            return result
+
+        # Set privacy request status to requires_input if not already set
+        if self.resources.request.status != PrivacyRequestStatus.requires_input:
+            self.resources.request.status = PrivacyRequestStatus.requires_input
+            self.resources.request.save(db)
+
+        # This should trigger log_awaiting_processing via the @retry decorator
+        raise AwaitingAsyncTaskCallback(
+            f"Manual task for {connection_key} requires user input"
+        )
+
+    def _ensure_manual_task_instances(
+        self,
+        db: Session,
+        manual_tasks: List[ManualTask],
+        privacy_request: PrivacyRequest,
+        allowed_config_type: "ManualTaskConfigurationType",
+    ) -> None:
+        """Create ManualTaskInstances for configs matching `allowed_config_type` if they don't exist."""
+
+        for manual_task in manual_tasks:
+            # ------------------------------------------------------------------
+            # Short-circuit: if instances already exist for this task & entity
+            # (no matter what config version they were created for) we should reuse
+            # them instead of creating a brand-new one that would result in
+            # duplicates when configurations are versioned after the privacy
+            # request has started.
+            # ------------------------------------------------------------------
+            existing_task_instance = (
+                db.query(ManualTaskInstance)
+                .filter(
+                    ManualTaskInstance.task_id == manual_task.id,
+                    ManualTaskInstance.entity_id == privacy_request.id,
+                    ManualTaskInstance.entity_type
+                    == ManualTaskEntityType.privacy_request,
+                )
+                .first()
+            )
+            if existing_task_instance:
+                # An instance already exists for this privacy request – no need
+                # to create another one tied to a newer config version.
+                continue
+
+            # Check each active config for instances (now we know none exist yet)
+            for config in manual_task.configs:
+                if not config.is_current or config.config_type != allowed_config_type:
+                    # Skip configs that are not current or not relevant for this request type
+                    continue
+
+                ManualTaskInstance.create(
+                    db=db,
+                    data={
+                        "task_id": manual_task.id,
+                        "config_id": config.id,
+                        "entity_id": privacy_request.id,
+                        "entity_type": ManualTaskEntityType.privacy_request.value,
+                        "status": StatusType.pending.value,
+                    },
+                )
+
+    # pylint: disable=too-many-branches,too-many-nested-blocks
+    def _get_submitted_data(
+        self,
+        db: Session,
+        manual_tasks: List[ManualTask],
+        privacy_request: PrivacyRequest,
+        allowed_config_type: "ManualTaskConfigurationType",
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Check if all manual task instances have submissions for ALL fields and return aggregated data
+        Returns None if any field submissions are missing (all fields must be completed or skipped)
+        """
+        aggregated_data: Dict[str, Any] = {}
+
+        def _format_size(size_bytes: int) -> str:
+            units = ["B", "KB", "MB", "GB", "TB"]
+            size = float(size_bytes)
+            for unit in units:
+                if size < 1024.0:
+                    return f"{size:.1f} {unit}"
+                size /= 1024.0
+            return f"{size:.1f} PB"
+
+        for manual_task in manual_tasks:
+
+            candidate_instances: list[ManualTaskInstance] = (
+                db.query(ManualTaskInstance)
+                .filter(
+                    ManualTaskInstance.task_id == manual_task.id,
+                    ManualTaskInstance.entity_id == privacy_request.id,
+                    ManualTaskInstance.entity_type
+                    == ManualTaskEntityType.privacy_request,
+                )
+                .all()
+            )
+
+            if not candidate_instances:
+                return None  # No instance yet for this manual task
+
+            for inst in candidate_instances:
+                # Skip instances tied to other request types
+                if not inst.config or inst.config.config_type != allowed_config_type:
+                    continue
+
+                all_fields = inst.config.field_definitions or []
+
+                # Every field must have a submission
+                if not all(inst.get_submission_for_field(f.id) for f in all_fields):
+                    return None  # At least one instance still incomplete
+
+                # Ensure status set
+                if inst.status != StatusType.completed:
+                    inst.status = StatusType.completed
+                    inst.save(db)
+
+                # Aggregate submission data from this instance
+                for submission in inst.submissions:
+                    if not submission.field or not submission.field.field_key:
+                        continue
+
+                    field_key = submission.field.field_key
+
+                    if not isinstance(submission.data, dict):
+                        continue
+
+                    data_dict: Dict[str, Any] = submission.data
+
+                    field_type = data_dict.get("field_type")
+
+                    if field_type == ManualTaskFieldType.attachment.value:
+                        attachment_map: Dict[str, Dict[str, Any]] = {}
+                        for attachment in submission.attachments or []:
+                            if (
+                                attachment.attachment_type
+                                == AttachmentType.include_with_access_package
+                            ):
+                                try:
+                                    size, url = attachment.retrieve_attachment()
+                                    attachment_map[attachment.file_name] = {
+                                        "url": str(url) if url else None,
+                                        "size": (
+                                            _format_size(size) if size else "Unknown"
+                                        ),
+                                    }
+                                except (
+                                    Exception
+                                ) as exc:  # pylint: disable=broad-exception-caught
+                                    logger.warning(
+                                        "Error retrieving attachment {}: {}",
+                                        attachment.file_name,
+                                        str(exc),
+                                    )
+
+                        aggregated_data[field_key] = attachment_map or None
+                    else:
+                        aggregated_data[field_key] = data_dict.get("value")
+
+        return aggregated_data if aggregated_data else None
+
+    def dry_run_task(self) -> int:
+        """Return estimated row count for dry run - manual tasks don't have predictable counts"""
+        return 1  # Placeholder - manual tasks generate variable data
+
+    # NEW METHOD: Provide erasure support for manual tasks
+    @retry(action_type=ActionType.erasure, default_return=0)
+    def erasure_request(
+        self,
+        retrieved_data: List[Row],
+        *erasure_prereqs: int,  # noqa: D401, pylint: disable=unused-argument
+    ) -> int:
+        """Execute manual-task-driven erasure logic.
+
+        Mirrors access_request behaviour but returns the number of rows masked (always 0)
+        once all required manual task submissions are present. If submissions are
+        incomplete the privacy request is paused awaiting user input.
+        """
+        db = self.resources.session
+        collection_address = self.execution_node.address
+
+        # Validate manual task address
+        if not ManualTaskAddress.is_manual_task_address(collection_address):
+            raise ValueError(f"Invalid manual task address: {collection_address}")
+
+        connection_key = ManualTaskAddress.get_connection_key(collection_address)
+
+        # Fetch relevant manual tasks for this connection
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+        if not manual_tasks:
+            # No manual tasks defined – nothing to erase
+            self.log_end(ActionType.erasure)
+            return 0
+
+        # Create ManualTaskInstances for ERASURE configs only
+        self._ensure_manual_task_instances(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.erasure_privacy_request,
+        )
+
+        # Check for full submissions – reuse helper used by access flow, filtering ERASURE configs
+        submissions_complete = self._get_submitted_data(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.erasure_privacy_request,
+        )
+
+        # If any field submissions are missing, pause processing
+        if submissions_complete is None:
+            if self.resources.request.status != PrivacyRequestStatus.requires_input:
+                self.resources.request.status = PrivacyRequestStatus.requires_input
+                self.resources.request.save(db)
+            raise AwaitingAsyncTaskCallback(
+                f"Manual erasure task for {connection_key} requires user input"
+            )
+
+        # Mark rows_masked = 0 (manual tasks do not mask data directly)
+        if self.request_task.id:
+            # Storing result for DSR 3.0; SQLAlchemy column typing triggers mypy warning
+            self.request_task.rows_masked = 0  # type: ignore[assignment]
+
+        # Mark successful completion
+        self.log_end(ActionType.erasure)
+        return 0

fides/api/task/manual/manual_task_utils.py

@@ -0,0 +1,322 @@
+from typing import TYPE_CHECKING, List
+
+from sqlalchemy.orm import Session
+
+from fides.api.graph.config import (
+    Collection,
+    CollectionAddress,
+    Field,
+    GraphDataset,
+    ScalarField,
+)
+from fides.api.graph.graph import Node
+from fides.api.models.connectionconfig import ConnectionConfig
+
+# Import application models
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskConfig,
+    ManualTaskConfigurationType,
+    ManualTaskEntityType,
+    ManualTaskInstance,
+)
+from fides.api.models.privacy_request import PrivacyRequest
+
+# TYPE_CHECKING import placed after all runtime imports to avoid lint issues
+if TYPE_CHECKING:  # pragma: no cover
+    from fides.api.graph.traversal import TraversalNode  # noqa: F401
+
+
+class ManualTaskAddress:
+    """Utility class for creating and parsing manual task addresses"""
+
+    MANUAL_DATA_COLLECTION = "manual_data"
+
+    @staticmethod
+    def create(connection_config_key: str) -> CollectionAddress:
+        """Create a CollectionAddress for manual data: {connection_key}:manual_data"""
+        return CollectionAddress(
+            dataset=connection_config_key,
+            collection=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+        )
+
+    @staticmethod
+    def is_manual_task_address(address: CollectionAddress) -> bool:
+        """Check if address represents manual task data"""
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.endswith(f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}")
+
+        # Handle CollectionAddress object
+        return address.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
+
+    @staticmethod
+    def get_connection_key(address: CollectionAddress) -> str:
+        """Extract connection config key from manual task address"""
+        if not ManualTaskAddress.is_manual_task_address(address):
+            raise ValueError(f"Not a manual task address: {address}")
+
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.split(":")[0]
+
+        # Handle CollectionAddress object
+        return address.dataset
+
+
+def get_manual_task_addresses(db: Session) -> List[CollectionAddress]:
+    """
+    Get manual task addresses for all connection configs that have manual tasks.
+
+    Note: Manual tasks should be included in the graph if they exist for any connection config
+    that's part of the dataset graph, regardless of specific policy targets. This allows
+    manual tasks to collect additional data that may be needed for the privacy request.
+    """
+    # Get all connection configs that have manual tasks
+    connection_configs_with_manual_tasks = (
+        db.query(ConnectionConfig)
+        .join(ManualTask, ConnectionConfig.id == ManualTask.parent_entity_id)
+        .filter(ManualTask.parent_entity_type == "connection_config")
+        .all()
+    )
+
+    # Create addresses for all connections that have manual tasks
+    manual_task_addresses = []
+    for config in connection_configs_with_manual_tasks:
+        manual_task_addresses.append(ManualTaskAddress.create(config.key))
+
+    return manual_task_addresses
+
+
+def get_manual_tasks_for_connection_config(
+    db: Session, connection_config_key: str
+) -> List[ManualTask]:
+    """Get all ManualTasks for a specific connection config"""
+    connection_config = (
+        db.query(ConnectionConfig)
+        .filter(ConnectionConfig.key == connection_config_key)
+        .first()
+    )
+
+    if not connection_config:
+        return []
+
+    return (
+        db.query(ManualTask)
+        .filter(
+            ManualTask.parent_entity_id == connection_config.id,
+            ManualTask.parent_entity_type == "connection_config",
+        )
+        .all()
+    )
+
+
+def create_manual_data_traversal_node(
+    db: Session, address: CollectionAddress
+) -> "TraversalNode":
+    """
+    Create a TraversalNode for a manual_data collection
+    """
+    connection_key = address.dataset
+
+    # Get manual tasks for this connection to determine fields
+    manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+
+    # Create fields based on ManualTaskConfigFields
+    fields: List[Field] = []
+    for manual_task in manual_tasks:
+        for config in manual_task.configs:
+            for field in config.field_definitions:
+                # Create a scalar field for each manual task field
+                # Extract data categories from field metadata if available
+                field_metadata = field.field_metadata or {}
+                data_categories = field_metadata.get("data_categories", [])
+
+                scalar_field = ScalarField(
+                    name=field.field_key,
+                    data_categories=data_categories,
+                    # Manual task fields don't have complex relationships
+                )
+                fields.append(scalar_field)
+
+    # Create a synthetic Collection
+    collection = Collection(
+        name=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+        fields=fields,
+        # Manual tasks don't have complex dependencies
+        after=set(),
+    )
+
+    # Create a synthetic GraphDataset
+    dataset = GraphDataset(
+        name=connection_key,
+        collections=[collection],
+        connection_key=connection_key,
+        after=set(),
+    )
+
+    # Create Node and TraversalNode (import locally to avoid cyclic import)
+    from fides.api.graph.traversal import TraversalNode  # local import
+
+    node = Node(dataset, collection)
+    traversal_node = TraversalNode(node)
+
+    return traversal_node
+
+
+def create_manual_task_instances_for_privacy_request(
+    db: Session, privacy_request: PrivacyRequest
+) -> List[ManualTaskInstance]:
+    """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
+    instances = []
+
+    # Get all connection configs that have manual tasks
+    connection_configs_with_manual_tasks = (
+        db.query(ConnectionConfig)
+        .join(ManualTask, ConnectionConfig.id == ManualTask.parent_entity_id)
+        .filter(ManualTask.parent_entity_type == "connection_config")
+        .all()
+    )
+
+    for connection_config in connection_configs_with_manual_tasks:
+        manual_tasks = (
+            db.query(ManualTask)
+            .filter(
+                ManualTask.parent_entity_id == connection_config.id,
+                ManualTask.parent_entity_type == "connection_config",
+            )
+            .all()
+        )
+
+        for manual_task in manual_tasks:
+            # Get the active config for this manual task
+            active_config = (
+                db.query(ManualTaskConfig)
+                .filter(
+                    ManualTaskConfig.task_id == manual_task.id,
+                    ManualTaskConfig.is_current.is_(True),
+                )
+                .first()
+            )
+
+            if not active_config:
+                continue  # Skip if no active config
+
+            # Check if instance already exists
+            existing_instance = (
+                db.query(ManualTaskInstance)
+                .filter(
+                    ManualTaskInstance.entity_id == privacy_request.id,
+                    ManualTaskInstance.entity_type == "privacy_request",
+                    ManualTaskInstance.task_id == manual_task.id,
+                    ManualTaskInstance.config_id == active_config.id,
+                )
+                .first()
+            )
+
+            if not existing_instance:
+                instance = ManualTaskInstance(
+                    entity_id=privacy_request.id,
+                    entity_type=ManualTaskEntityType.privacy_request,
+                    task_id=manual_task.id,
+                    config_id=active_config.id,
+                )
+                db.add(instance)
+                instances.append(instance)
+
+    if instances:
+        db.commit()
+
+    return instances
+
+
+def get_manual_task_instances_for_privacy_request(
+    db: Session, privacy_request: PrivacyRequest
+) -> List[ManualTaskInstance]:
+    """Get all manual task instances for a privacy request."""
+    return (
+        db.query(ManualTaskInstance)
+        .filter(
+            ManualTaskInstance.entity_id == privacy_request.id,
+            ManualTaskInstance.entity_type == "privacy_request",
+        )
+        .all()
+    )
+
+
+def create_manual_task_artificial_graphs(
+    db: Session,
+) -> List:
+    """
+    Create artificial GraphDataset objects for manual tasks that can be included
+    in the main dataset graph during the dataset configuration phase.
+
+    Manual tasks should be treated as data sources/datasets rather than being
+    appended to the traversal graph later.
+
+    Manual task collections are designed as root nodes that execute immediately when
+    the privacy request starts, in parallel with identity processing. They don't depend
+    on identity data since they provide manually-entered data rather than consuming it.
+
+    Args:
+        db: Database session
+        policy: The policy being executed
+
+    Returns:
+        List of GraphDataset objects representing manual tasks as root nodes
+    """
+
+    manual_task_graphs = []
+    manual_addresses = get_manual_task_addresses(db)
+
+    for address in manual_addresses:
+        connection_key = address.dataset
+
+        # Get manual tasks for this connection to determine fields
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+
+        # Create fields based only on ManualTaskConfigFields
+        fields: List = []
+
+        # Manual task collections act as root nodes - they don't need identity dependencies
+        # since they provide manually-entered data rather than consuming identity data.
+        for manual_task in manual_tasks:
+            for config in manual_task.configs:
+                if config.config_type not in [
+                    ManualTaskConfigurationType.access_privacy_request,
+                    ManualTaskConfigurationType.erasure_privacy_request,
+                ]:
+                    continue
+                if not config.is_current:
+                    continue
+                for field in config.field_definitions:
+                    # Create a scalar field for each manual task field
+                    field_metadata = field.field_metadata or {}
+                    data_categories = field_metadata.get("data_categories", [])
+
+                    scalar_field = ScalarField(
+                        name=field.field_key,
+                        data_categories=data_categories,
+                    )
+                    fields.append(scalar_field)
+
+        if fields:  # Only create graph if there are fields
+            # Create a synthetic Collection
+            collection = Collection(
+                name=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+                fields=fields,
+                # Manual tasks have no dependencies - they're root nodes
+                after=set(),
+            )
+
+            # Create a synthetic GraphDataset
+            graph_dataset = GraphDataset(
+                name=connection_key,
+                collections=[collection],
+                connection_key=connection_key,
+                after=set(),
+            )
+
+            manual_task_graphs.append(graph_dataset)
+
+    return manual_task_graphs

fides/api/task/task_resources.py

@@ -15,6 +15,7 @@ from fides.api.service.connectors import (
     FidesConnector,
     GoogleCloudSQLMySQLConnector,
     GoogleCloudSQLPostgresConnector,
+    ManualTaskConnector,
     MariaDBConnector,
     MicrosoftSQLServerConnector,
     MongoDBConnector,
@@ -92,6 +93,8 @@ class Connections:
             return S3Connector(connection_config)
         if connection_config.connection_type == ConnectionType.scylla:
             return ScyllaConnector(connection_config)
+        if connection_config.connection_type == ConnectionType.manual_task:
+            return ManualTaskConnector(connection_config)
         raise NotImplementedError(
             f"No connector available for {connection_config.connection_type}"
         )

fides/api/util/connection_util.py

@@ -25,6 +25,11 @@ from fides.api.models.connectionconfig import (
     ConnectionType,
 )
 from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskParentEntityType,
+    ManualTaskType,
+)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.models.sql_models import Dataset as CtlDataset  # type: ignore
@@ -243,7 +248,9 @@ def patch_connection_configs(
                     ).model_dump(mode="json")
                     connection_config.save(db=db)
                     created_or_updated.append(
-                        ConnectionConfigurationResponse(**connection_config.__dict__)
+                        ConnectionConfigurationResponse.model_validate(
+                            connection_config
+                        )
                     )
                     continue
 
@@ -268,8 +275,24 @@ def patch_connection_configs(
             connection_config = ConnectionConfig.create_or_update(
                 db, data=config_dict, check_name=False
             )
+
+            # Automatically create a ManualTask if this is a connection config of type manual_task
+            # and it doesn't already have one
+            if (
+                connection_config.connection_type == ConnectionType.manual_task
+                and not connection_config.manual_task
+            ):
+                ManualTask.create(
+                    db=db,
+                    data={
+                        "task_type": ManualTaskType.privacy_request,
+                        "parent_entity_id": connection_config.id,
+                        "parent_entity_type": ManualTaskParentEntityType.connection_config,
+                    },
+                )
+
             created_or_updated.append(
-                ConnectionConfigurationResponse(**connection_config.__dict__)
+                ConnectionConfigurationResponse.model_validate(connection_config)
             )
         except KeyOrNameAlreadyExists as exc:
             logger.warning(

fides/common/api/v1/urn_registry.py

@@ -155,6 +155,10 @@ AUTHORIZE = "/connection/{connection_key}/authorize"
 ACCESS_MANUAL_WEBHOOKS = "/access_manual_webhook"
 ACCESS_MANUAL_WEBHOOK = CONNECTION_BY_KEY + "/access_manual_webhook"
 
+# Manual Tasks
+MANUAL_TASKS = "/manual-tasks"
+MANUAL_TASK = CONNECTION_BY_KEY + "/manual-task"
+
 # Collection URLs
 DATASETS = "/dataset"
 DATASET_CONFIG = "/datasetconfig"