ethyca-fides 2.64.6b0__py2.py3-none-any.whl → 2.64.6b2__py2.py3-none-any.whl

fides/api/service/connectors/manual_task_connector.py

@@ -0,0 +1,96 @@
+"""
+Manual Task Connector - A minimal connector for manual task operations.
+
+Since manual tasks don't actually connect to external systems, this connector
+provides no-op implementations of the BaseConnector interface.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from fides.api.graph.execution import ExecutionNode
+from fides.api.models.connectionconfig import ConnectionTestStatus
+from fides.api.models.policy import Policy
+from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.service.connectors.base_connector import BaseConnector
+from fides.api.service.connectors.query_configs.query_config import QueryConfig
+from fides.api.util.collection_util import Row
+
+
+class ManualTaskQueryConfig(QueryConfig):
+    """Minimal query config for manual tasks - not actually used"""
+
+    def generate_query(
+        self, input_data: Dict[str, List[Any]], policy: Optional[Policy]
+    ) -> str:
+        return "Manual task: no query needed"
+
+    def dry_run_query(self) -> str:
+        return "Manual task: no query needed"
+
+    def query_to_str(self, t: Any, input_data: Dict[str, List[Any]]) -> str:
+        """Convert query to string - not used for manual tasks"""
+        return "Manual task: no query needed"
+
+    def generate_update_stmt(
+        self, row: Row, policy: Policy, request: PrivacyRequest
+    ) -> Any:
+        """Generate update statement - not used for manual tasks"""
+        return None
+
+
+class ManualTaskConnector(BaseConnector):
+    """
+    Minimal connector for manual tasks.
+
+    This connector provides no-op implementations since manual tasks don't
+    actually connect to external systems. The actual manual task logic
+    is handled by ManualTaskGraphTask.access_request()
+    """
+
+    def query_config(self, node: ExecutionNode) -> QueryConfig[Any]:
+        """Return a minimal query config - not actually used for manual tasks"""
+        return ManualTaskQueryConfig(node)
+
+    def test_connection(self) -> Optional[ConnectionTestStatus]:
+        """Manual tasks don't have connections to test"""
+        return ConnectionTestStatus.succeeded
+
+    def create_client(self) -> None:
+        """Manual tasks don't need database clients"""
+        return None
+
+    def retrieve_data(
+        self,
+        node: ExecutionNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        request_task: RequestTask,
+        input_data: Dict[str, List[Any]],
+    ) -> List[Row]:
+        """
+        This method is not used for manual tasks.
+        Manual task data retrieval is handled by ManualTaskGraphTask.access_request()
+        """
+        return []
+
+    def mask_data(
+        self,
+        node: ExecutionNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        request_task: RequestTask,
+        rows: List[Row],
+    ) -> int:
+        """
+        Manual tasks don't support erasure operations.
+        Manual tasks are for data collection, not data modification.
+        """
+        return 0
+
+    def close(self) -> None:
+        """No resources to close for manual tasks"""
+
+    @property
+    def requires_primary_keys(self) -> bool:
+        """Manual tasks don't require primary keys since they don't modify data"""
+        return False

fides/api/service/messaging/message_dispatch_service.py

@@ -26,6 +26,7 @@ from fides.api.schemas.messaging.messaging import (
     EmailForActionType,
     ErasureRequestBodyParams,
     ErrorNotificationBodyParams,
+    ExternalUserWelcomeBodyParams,
     FidesopsMessage,
     MessagingActionType,
     MessagingMethod,
@@ -176,6 +177,7 @@ def dispatch_message(
             ErasureRequestBodyParams,
             UserInviteBodyParams,
             ErrorNotificationBodyParams,
+            ExternalUserWelcomeBodyParams,
         ]
     ] = None,
     subject_override: Optional[str] = None,
@@ -351,7 +353,7 @@ def _render(template_str: str, variables: Optional[Dict] = None) -> str:
     return template_str
 
 
-def _build_email(  # pylint: disable=too-many-return-statements
+def _build_email(  # pylint: disable=too-many-return-statements, too-many-branches
     config_proxy: ConfigProxy,
     action_type: MessagingActionType,
     body_params: Any,
@@ -463,6 +465,36 @@ def _build_email(  # pylint: disable=too-many-return-statements
                 }
             ),
         )
+    if action_type == MessagingActionType.EXTERNAL_USER_WELCOME:
+        base_template = get_email_template(action_type)
+        # Generate display name for personalization
+        display_name = body_params.username
+        if body_params.first_name:
+            display_name = body_params.first_name
+            if body_params.last_name:
+                display_name = f"{body_params.first_name} {body_params.last_name}"
+
+        portal_link = (
+            f"{body_params.privacy_center_url}?access_token={body_params.access_token}"
+        )
+
+        variables = {
+            "username": body_params.username,
+            "display_name": display_name,
+            "first_name": body_params.first_name,
+            "last_name": body_params.last_name,
+            "org_name": body_params.org_name,
+            "portal_link": portal_link,
+            "privacy_center_url": body_params.privacy_center_url,
+            "access_token": body_params.access_token,
+        }
+
+        return EmailForActionType(
+            subject="Welcome to our Privacy Center",
+            body=base_template.render(variables),
+            template_variables=variables,
+        )
+
     logger.error("Message action type {} is not implemented", action_type)
     raise MessageDispatchException(
         f"Message action type {action_type} is not implemented"

fides/api/service/privacy_request/dsr_package/templates/collection_index.html

@@ -26,7 +26,15 @@
                   <div class="table-row">
                      <div class="table-cell">{{ field }}</div>
                      <div class="table-cell">
-                        {% if field == "attachments" and value is mapping and value|length > 0 %}
+                        {% set _is_attachment_block = false %}
+                        {% if value is mapping and value|length > 0 %}
+                           {% set _first_key = (value.keys() | list)[0] %}
+                           {% if value[_first_key] is mapping and ('url' in value[_first_key]) %}
+                               {% set _is_attachment_block = true %}
+                           {% endif %}
+                        {% endif %}
+
+                        {% if _is_attachment_block %}
                            <p class="expiration-notice">Note: All download links will expire in 7 days.</p>
                            <div class="table table-hover">
                               <div class="table-row">

fides/api/service/privacy_request/dsr_package/templates/main.css

@@ -23,6 +23,10 @@ h1 {
     color: var(--text-color);
 }
 
+h2 {
+    margin-bottom: 12px;
+}
+
 .container {
     display: flex;
     flex-direction: column;
@@ -109,8 +113,8 @@ h1 {
     width: 100%;
     border-collapse: separate;
     border-spacing: 0;
-    padding-top: 20px;
-    padding-bottom: 100px;
+    padding-top: 0;
+    padding-bottom: 14px;
     font-size: 14px;
 }
 

fides/api/service/privacy_request/request_runner_service.py

@@ -1,3 +1,4 @@
+# pylint: disable=too-many-lines
 import time
 from copy import deepcopy
 from datetime import datetime, timedelta
@@ -71,6 +72,7 @@ from fides.api.task.graph_task import (
     filter_by_enabled_actions,
     get_cached_data_for_erasures,
 )
+from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
 from fides.api.tasks import DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
 from fides.api.util.collection_util import Row
@@ -450,6 +452,11 @@ def run_privacy_request(
                     for dataset_config in datasets
                     if not dataset_config.connection_config.disabled
                 ]
+
+                # Add manual task artificial graphs to dataset graphs
+                manual_task_graphs = create_manual_task_artificial_graphs(session)
+                dataset_graphs.extend(manual_task_graphs)
+
                 dataset_graph = DatasetGraph(*dataset_graphs)
 
                 # Add success log for dataset configuration

fides/api/task/create_request_tasks.py

@@ -33,6 +33,10 @@ from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
+from fides.api.task.manual.manual_task_utils import (
+    ManualTaskAddress,
+    create_manual_task_instances_for_privacy_request,
+)
 from fides.api.util.logger_context_utils import log_context
 
 
@@ -85,6 +89,14 @@ def build_access_networkx_digraph(
         # Connect the end nodes, those that have no downstream dependencies, to the terminator node
         networkx_graph.add_edge(node, TERMINATOR_ADDRESS)
 
+    manual_nodes = [
+        addr
+        for addr in traversal_nodes.keys()
+        if addr.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
+    ]
+    for manual_node in manual_nodes:
+        networkx_graph.add_edge(ROOT_COLLECTION_ADDRESS, manual_node)
+
     _add_edge_if_no_nodes(traversal_nodes, networkx_graph)
     return networkx_graph
 
@@ -458,6 +470,10 @@ def run_access_request(
             end_nodes: List[CollectionAddress] = traversal.traverse(
                 traversal_nodes, collect_tasks_fn
             )
+
+            # Snapshot manual task field instances for this privacy request
+            create_manual_task_instances_for_privacy_request(session, privacy_request)
+
             # Save Access Request Tasks to the database
             ready_tasks = persist_new_access_request_tasks(
                 session, privacy_request, traversal, traversal_nodes, end_nodes, graph

fides/api/task/execute_request_tasks.py

@@ -29,6 +29,8 @@ from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
 )
+from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
+from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.task.task_resources import TaskResources
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.util.cache import cache_task_tracking_key
@@ -108,7 +110,14 @@ def create_graph_task(
     to begin with - this may be unrecoverable and a new Privacy Request should be created.
     """
     try:
-        graph_task: GraphTask = GraphTask(resources)
+        collection_address = request_task.request_task_address
+
+        # Check if this is a manual task address
+        graph_task: GraphTask
+        if ManualTaskAddress.is_manual_task_address(collection_address):
+            graph_task = ManualTaskGraphTask(resources)
+        else:
+            graph_task = GraphTask(resources)
 
     except Exception as exc:
         logger.debug(

fides/api/task/filter_results.py

@@ -6,6 +6,7 @@ from loguru import logger
 
 from fides.api.graph.config import CollectionAddress, FieldPath
 from fides.api.graph.graph import DatasetGraph
+from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.util.collection_util import Row
 
 
@@ -37,6 +38,11 @@ def filter_data_categories(
         if not results:
             continue
 
+        # Skip manual task data - it doesn't need filtering since it's controlled by field definitions
+        if f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}" in node_address:
+            filtered_access_results[node_address].extend(results)
+            continue
+
         # Results from fides connectors are a special case:
         # they've already been filtered and stored in a dict keyed by rule key.
         # So here, we simply find the results corresponding to our current rule

fides/api/task/graph_task.py

@@ -109,6 +109,7 @@ def retry(
                         method_name,
                         self.execution_node.address,
                     )
+                    # Log the awaiting processing status and exit without retrying.
                     self.log_awaiting_processing(action_type, ex)
                     # Request Task put in "awaiting_processing" status and exited, awaiting Async Callback
                     return None

fides/api/task/manual/manual_task_graph_task.py

@@ -0,0 +1,300 @@
+from typing import Any, Dict, List, Optional
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.common_exceptions import AwaitingAsyncTaskCallback
+from fides.api.models.attachment import AttachmentType
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskConfigurationType,
+    ManualTaskEntityType,
+    ManualTaskFieldType,
+    ManualTaskInstance,
+    StatusType,
+)
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import ActionType
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
+from fides.api.task.graph_task import GraphTask, retry
+from fides.api.task.manual.manual_task_utils import (
+    ManualTaskAddress,
+    get_manual_tasks_for_connection_config,
+)
+from fides.api.util.collection_util import Row
+
+
+class ManualTaskGraphTask(GraphTask):
+    """GraphTask implementation for ManualTask execution"""
+
+    @retry(action_type=ActionType.access, default_return=[])
+    def access_request(self, *inputs: List[Row]) -> List[Row]:
+        """
+        Execute manual task logic following the standard GraphTask pattern:
+        1. Create ManualTaskInstances if they don't exist
+        2. Check for submissions
+        3. Return data if submitted, raise AwaitingAsyncTaskCallback if not
+        """
+        db = self.resources.session
+        collection_address = self.execution_node.address
+
+        # Verify this is a manual task address
+        if not ManualTaskAddress.is_manual_task_address(collection_address):
+            raise ValueError(f"Invalid manual task address: {collection_address}")
+
+        connection_key = ManualTaskAddress.get_connection_key(collection_address)
+
+        # Get manual tasks for this connection
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+
+        if not manual_tasks:
+            return []
+
+        # Check/create manual task instances for ACCESS configs only
+        self._ensure_manual_task_instances(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.access_privacy_request,
+        )
+
+        # Check if all manual task instances have submissions for ACCESS configs only
+        submitted_data = self._get_submitted_data(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.access_privacy_request,
+        )
+
+        if submitted_data is not None:
+            result: List[Row] = [submitted_data] if submitted_data else []
+            self.request_task.access_data = result
+
+            # Mark request task as complete and write execution log
+            self.log_end(ActionType.access)
+            return result
+
+        # Set privacy request status to requires_input if not already set
+        if self.resources.request.status != PrivacyRequestStatus.requires_input:
+            self.resources.request.status = PrivacyRequestStatus.requires_input
+            self.resources.request.save(db)
+
+        # This should trigger log_awaiting_processing via the @retry decorator
+        raise AwaitingAsyncTaskCallback(
+            f"Manual task for {connection_key} requires user input"
+        )
+
+    def _ensure_manual_task_instances(
+        self,
+        db: Session,
+        manual_tasks: List[ManualTask],
+        privacy_request: PrivacyRequest,
+        allowed_config_type: "ManualTaskConfigurationType",
+    ) -> None:
+        """Create ManualTaskInstances for configs matching `allowed_config_type` if they don't exist."""
+
+        for manual_task in manual_tasks:
+            # ------------------------------------------------------------------
+            # Short-circuit: if instances already exist for this task & entity
+            # (no matter what config version they were created for) we should reuse
+            # them instead of creating a brand-new one that would result in
+            # duplicates when configurations are versioned after the privacy
+            # request has started.
+            # ------------------------------------------------------------------
+            existing_task_instance = (
+                db.query(ManualTaskInstance)
+                .filter(
+                    ManualTaskInstance.task_id == manual_task.id,
+                    ManualTaskInstance.entity_id == privacy_request.id,
+                    ManualTaskInstance.entity_type
+                    == ManualTaskEntityType.privacy_request,
+                )
+                .first()
+            )
+            if existing_task_instance:
+                # An instance already exists for this privacy request – no need
+                # to create another one tied to a newer config version.
+                continue
+
+            # Check each active config for instances (now we know none exist yet)
+            for config in manual_task.configs:
+                if not config.is_current or config.config_type != allowed_config_type:
+                    # Skip configs that are not current or not relevant for this request type
+                    continue
+
+                ManualTaskInstance.create(
+                    db=db,
+                    data={
+                        "task_id": manual_task.id,
+                        "config_id": config.id,
+                        "entity_id": privacy_request.id,
+                        "entity_type": ManualTaskEntityType.privacy_request.value,
+                        "status": StatusType.pending.value,
+                    },
+                )
+
+    # pylint: disable=too-many-branches,too-many-nested-blocks
+    def _get_submitted_data(
+        self,
+        db: Session,
+        manual_tasks: List[ManualTask],
+        privacy_request: PrivacyRequest,
+        allowed_config_type: "ManualTaskConfigurationType",
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Check if all manual task instances have submissions for ALL fields and return aggregated data
+        Returns None if any field submissions are missing (all fields must be completed or skipped)
+        """
+        aggregated_data: Dict[str, Any] = {}
+
+        def _format_size(size_bytes: int) -> str:
+            units = ["B", "KB", "MB", "GB", "TB"]
+            size = float(size_bytes)
+            for unit in units:
+                if size < 1024.0:
+                    return f"{size:.1f} {unit}"
+                size /= 1024.0
+            return f"{size:.1f} PB"
+
+        for manual_task in manual_tasks:
+
+            candidate_instances: list[ManualTaskInstance] = (
+                db.query(ManualTaskInstance)
+                .filter(
+                    ManualTaskInstance.task_id == manual_task.id,
+                    ManualTaskInstance.entity_id == privacy_request.id,
+                    ManualTaskInstance.entity_type
+                    == ManualTaskEntityType.privacy_request,
+                )
+                .all()
+            )
+
+            if not candidate_instances:
+                return None  # No instance yet for this manual task
+
+            for inst in candidate_instances:
+                # Skip instances tied to other request types
+                if not inst.config or inst.config.config_type != allowed_config_type:
+                    continue
+
+                all_fields = inst.config.field_definitions or []
+
+                # Every field must have a submission
+                if not all(inst.get_submission_for_field(f.id) for f in all_fields):
+                    return None  # At least one instance still incomplete
+
+                # Ensure status set
+                if inst.status != StatusType.completed:
+                    inst.status = StatusType.completed
+                    inst.save(db)
+
+                # Aggregate submission data from this instance
+                for submission in inst.submissions:
+                    if not submission.field or not submission.field.field_key:
+                        continue
+
+                    field_key = submission.field.field_key
+
+                    if not isinstance(submission.data, dict):
+                        continue
+
+                    data_dict: Dict[str, Any] = submission.data
+
+                    field_type = data_dict.get("field_type")
+
+                    if field_type == ManualTaskFieldType.attachment.value:
+                        attachment_map: Dict[str, Dict[str, Any]] = {}
+                        for attachment in submission.attachments or []:
+                            if (
+                                attachment.attachment_type
+                                == AttachmentType.include_with_access_package
+                            ):
+                                try:
+                                    size, url = attachment.retrieve_attachment()
+                                    attachment_map[attachment.file_name] = {
+                                        "url": str(url) if url else None,
+                                        "size": (
+                                            _format_size(size) if size else "Unknown"
+                                        ),
+                                    }
+                                except (
+                                    Exception
+                                ) as exc:  # pylint: disable=broad-exception-caught
+                                    logger.warning(
+                                        "Error retrieving attachment {}: {}",
+                                        attachment.file_name,
+                                        str(exc),
+                                    )
+
+                        aggregated_data[field_key] = attachment_map or None
+                    else:
+                        aggregated_data[field_key] = data_dict.get("value")
+
+        return aggregated_data if aggregated_data else None
+
+    def dry_run_task(self) -> int:
+        """Return estimated row count for dry run - manual tasks don't have predictable counts"""
+        return 1  # Placeholder - manual tasks generate variable data
+
+    # NEW METHOD: Provide erasure support for manual tasks
+    @retry(action_type=ActionType.erasure, default_return=0)
+    def erasure_request(
+        self,
+        retrieved_data: List[Row],
+        *erasure_prereqs: int,  # noqa: D401, pylint: disable=unused-argument
+    ) -> int:
+        """Execute manual-task-driven erasure logic.
+
+        Mirrors access_request behaviour but returns the number of rows masked (always 0)
+        once all required manual task submissions are present. If submissions are
+        incomplete the privacy request is paused awaiting user input.
+        """
+        db = self.resources.session
+        collection_address = self.execution_node.address
+
+        # Validate manual task address
+        if not ManualTaskAddress.is_manual_task_address(collection_address):
+            raise ValueError(f"Invalid manual task address: {collection_address}")
+
+        connection_key = ManualTaskAddress.get_connection_key(collection_address)
+
+        # Fetch relevant manual tasks for this connection
+        manual_tasks = get_manual_tasks_for_connection_config(db, connection_key)
+        if not manual_tasks:
+            # No manual tasks defined – nothing to erase
+            self.log_end(ActionType.erasure)
+            return 0
+
+        # Create ManualTaskInstances for ERASURE configs only
+        self._ensure_manual_task_instances(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.erasure_privacy_request,
+        )
+
+        # Check for full submissions – reuse helper used by access flow, filtering ERASURE configs
+        submissions_complete = self._get_submitted_data(
+            db,
+            manual_tasks,
+            self.resources.request,
+            ManualTaskConfigurationType.erasure_privacy_request,
+        )
+
+        # If any field submissions are missing, pause processing
+        if submissions_complete is None:
+            if self.resources.request.status != PrivacyRequestStatus.requires_input:
+                self.resources.request.status = PrivacyRequestStatus.requires_input
+                self.resources.request.save(db)
+            raise AwaitingAsyncTaskCallback(
+                f"Manual erasure task for {connection_key} requires user input"
+            )
+
+        # Mark rows_masked = 0 (manual tasks do not mask data directly)
+        if self.request_task.id:
+            # Storing result for DSR 3.0; SQLAlchemy column typing triggers mypy warning
+            self.request_task.rows_masked = 0  # type: ignore[assignment]
+
+        # Mark successful completion
+        self.log_end(ActionType.erasure)
+        return 0