ethyca-fides 2.66.2b2__py2.py3-none-any.whl → 2.66.2rc0__py2.py3-none-any.whl

fides/api/service/privacy_request/request_runner_service.py

@@ -14,7 +14,6 @@ from fides.api.common_exceptions import (
     ClientUnsuccessfulException,
     IdentityNotFoundException,
     ManualWebhookFieldsUnset,
-    MaskingSecretsExpired,
     MessageDispatchException,
     NoCachedManualWebhookEntry,
     PrivacyRequestExit,
@@ -76,7 +75,6 @@ from fides.api.task.graph_task import (
 from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
 from fides.api.tasks import DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
-from fides.api.util.cache import get_all_masking_secret_keys
 from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
@@ -480,21 +478,12 @@ def run_privacy_request(
                     connection_configs
                 )
 
-                # If the privacy request requires manual finalization and has not yet been finalized, we exit here
-                if (
-                    privacy_request.status
-                    == PrivacyRequestStatus.requires_manual_finalization
-                    and privacy_request.finalized_at is None
-                ):
-                    return
-
                 # Access CHECKPOINT
                 if (
                     policy.get_rules_for_action(action_type=ActionType.access)
                     or policy.get_rules_for_action(action_type=ActionType.erasure)
                 ) and can_run_checkpoint(
-                    request_checkpoint=CurrentStep.access,
-                    from_checkpoint=resume_step,
+                    request_checkpoint=CurrentStep.access, from_checkpoint=resume_step
                 ):
                     privacy_request.cache_failed_checkpoint_details(CurrentStep.access)
                     access_runner(
@@ -539,12 +528,9 @@ def run_privacy_request(
                 if policy.get_rules_for_action(
                     action_type=ActionType.erasure
                 ) and can_run_checkpoint(
-                    request_checkpoint=CurrentStep.erasure,
-                    from_checkpoint=resume_step,
+                    request_checkpoint=CurrentStep.erasure, from_checkpoint=resume_step
                 ):
                     privacy_request.cache_failed_checkpoint_details(CurrentStep.erasure)
-                    _verify_masking_secrets(policy, privacy_request, resume_step)
-
                     # We only need to run the erasure once until masking strategies are handled
                     erasure_runner(
                         privacy_request=privacy_request,
@@ -666,111 +652,58 @@ def run_privacy_request(
                 if not proceed:
                     return
 
-            # Request finalization CHECKPOINT
-            if can_run_checkpoint(
-                request_checkpoint=CurrentStep.finalization,
-                from_checkpoint=resume_step,
-            ):
-                privacy_request.cache_failed_checkpoint_details(
-                    CurrentStep.finalization,
-                )
-                if privacy_request.status != PrivacyRequestStatus.error:
-                    erasure_rules = policy.get_rules_for_action(
-                        action_type=ActionType.erasure
-                    )
-                    if (
-                        privacy_request.finalized_at is None
-                        and erasure_rules
-                        and CONFIG.execution.erasure_request_finalization_required
-                    ):
-                        logger.info(
-                            "Marking privacy request '{}' as requires manual finalization.",
-                            privacy_request.id,
-                        )
-                        privacy_request.status = (
-                            PrivacyRequestStatus.requires_manual_finalization
-                        )
-                        privacy_request.save(db=session)
-                        return
-
-                    # Finally, mark the request as complete
-                    if privacy_request.finalized_at:
-                        logger.info(
-                            "Marking privacy request '{}' as finalized.",
-                            privacy_request.id,
-                        )
-                        privacy_request.add_success_execution_log(
-                            session,
-                            connection_key=None,
-                            dataset_name="Request finalized",
-                            collection_name=None,
-                            message=f"Request finalized for privacy request: {privacy_request.id}",
-                            action_type=privacy_request.policy.get_action_type(),  # type: ignore
-                        )
-
-                    logger.info(
-                        "Marking privacy request '{}' as complete.",
-                        privacy_request.id,
-                    )
-                    AuditLog.create(
-                        db=session,
-                        data={
-                            "user_id": "system",
-                            "privacy_request_id": privacy_request.id,
-                            "action": AuditLogAction.finished,
-                            "message": "",
-                        },
+            legacy_request_completion_enabled = ConfigProxy(
+                session
+            ).notifications.send_request_completion_notification
+
+            action_type = (
+                MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
+                if policy.get_rules_for_action(action_type=ActionType.access)
+                else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
+            )
+
+            if message_send_enabled(
+                session,
+                privacy_request.property_id,
+                action_type,
+                legacy_request_completion_enabled,
+            ) and not policy.get_rules_for_action(action_type=ActionType.consent):
+                try:
+                    if not access_result_urls:
+                        # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
+                        # from the database because the Privacy Request would have exited
+                        # processing and lost access to the access_result_urls in memory
+                        access_result_urls = (
+                            privacy_request.access_result_urls or {}
+                        ).get("access_result_urls", [])
+                    initiate_privacy_request_completion_email(
+                        session,
+                        policy,
+                        access_result_urls,
+                        identity_data,
+                        privacy_request.property_id,
                     )
-                    privacy_request.status = PrivacyRequestStatus.complete
-                    privacy_request.finished_processing_at = datetime.utcnow()
-                    privacy_request.save(db=session)
-
-                    # Send a final email to the user confirming request completion
-                    if privacy_request.status == PrivacyRequestStatus.complete:
-                        legacy_request_completion_enabled = ConfigProxy(
-                            session
-                        ).notifications.send_request_completion_notification
-
-                        action_type = (
-                            MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
-                            if policy.get_rules_for_action(
-                                action_type=ActionType.access
-                            )
-                            else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
-                        )
-
-                        if message_send_enabled(
-                            session,
-                            privacy_request.property_id,
-                            action_type,
-                            legacy_request_completion_enabled,
-                        ) and not policy.get_rules_for_action(
-                            action_type=ActionType.consent
-                        ):
-                            if not access_result_urls:
-                                # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
-                                # from the database because the Privacy Request would have exited
-                                # processing and lost access to the access_result_urls in memory
-                                access_result_urls = (
-                                    privacy_request.access_result_urls or {}
-                                ).get("access_result_urls", [])
-
-                            try:
-                                initiate_privacy_request_completion_email(
-                                    session,
-                                    policy,
-                                    access_result_urls,
-                                    identity_data,
-                                    privacy_request.property_id,
-                                )
-                            except (
-                                IdentityNotFoundException,
-                                MessageDispatchException,
-                            ) as e:
-                                privacy_request.error_processing(db=session)
-                                # If dev mode, log traceback
-                                _log_exception(e, CONFIG.dev_mode)
-                                return
+                except (IdentityNotFoundException, MessageDispatchException) as e:
+                    privacy_request.error_processing(db=session)
+                    # If dev mode, log traceback
+                    _log_exception(e, CONFIG.dev_mode)
+                    return
+
+            # Only mark as complete if not in error state
+            if privacy_request.status != PrivacyRequestStatus.error:
+                privacy_request.finished_processing_at = datetime.utcnow()
+                AuditLog.create(
+                    db=session,
+                    data={
+                        "user_id": "system",
+                        "privacy_request_id": privacy_request.id,
+                        "action": AuditLogAction.finished,
+                        "message": "",
+                    },
+                )
+                privacy_request.status = PrivacyRequestStatus.complete
+                logger.info("Privacy request run completed.")
+                privacy_request.save(db=session)
 
 
 def initiate_privacy_request_completion_email(
@@ -1067,25 +1000,3 @@ def run_webhooks_and_report_status(
             return False
 
     return True
-
-
-def _verify_masking_secrets(
-    policy: Policy, privacy_request: PrivacyRequest, resume_step: Optional[CurrentStep]
-) -> None:
-    """
-    Checks that the required masking secrets are still cached for the given request.
-    Raises an exception if masking secrets are needed for the given policy but they don't exist.
-    """
-
-    if resume_step is None:
-        return
-
-    # if masking can be performed without any masking secrets, we skip the cache check
-    if (
-        policy.generate_masking_secrets()
-        and not get_all_masking_secret_keys(privacy_request.id)
-        and not privacy_request.masking_secrets
-    ):
-        raise MaskingSecretsExpired(
-            f"The masking secrets for privacy request ID '{privacy_request.id}' have expired. Please submit a new erasure request."
-        )

fides/api/service/privacy_request/request_service.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import json
 from asyncio import sleep
 from datetime import datetime, timedelta
-from typing import Any, Dict, Optional, Set
+from typing import Any, Dict, List, Optional, Set
 
 from httpx import AsyncClient
 from loguru import logger
@@ -11,6 +11,7 @@ from sqlalchemy import text
 from sqlalchemy.sql.elements import TextClause
 
 from fides.api.common_exceptions import PrivacyRequestNotFound
+from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
     PrivacyRequest,
@@ -18,12 +19,14 @@ from fides.api.models.privacy_request import (
 )
 from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
+from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import (
     PrivacyRequestResponse,
     PrivacyRequestStatus,
 )
 from fides.api.schemas.redis_cache import Identity
+from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
 from fides.api.util.cache import (
@@ -66,6 +69,7 @@ def build_required_privacy_request_kwargs(
 
 def cache_data(
     privacy_request: PrivacyRequest,
+    policy: Policy,
     identity: Identity,
     encryption_key: Optional[str],
     drp_request_body: Optional[DrpPrivacyRequestCreate],
@@ -78,6 +82,23 @@ def cache_data(
     privacy_request.cache_custom_privacy_request_fields(custom_privacy_request_fields)
     privacy_request.cache_encryption(encryption_key)  # handles None already
 
+    # Store masking secrets in the cache
+    logger.info("Caching masking secrets for privacy request {}", privacy_request.id)
+    erasure_rules = policy.get_rules_for_action(action_type=ActionType.erasure)
+    unique_masking_strategies_by_name: Set[str] = set()
+    for rule in erasure_rules:
+        strategy_name: str = rule.masking_strategy["strategy"]  # type: ignore
+        configuration = rule.masking_strategy["configuration"]  # type: ignore
+        if strategy_name in unique_masking_strategies_by_name:
+            continue
+        unique_masking_strategies_by_name.add(strategy_name)
+        masking_strategy = MaskingStrategy.get_strategy(strategy_name, configuration)
+        if masking_strategy.secrets_required():
+            masking_secrets: List[MaskingSecretCache] = (
+                masking_strategy.generate_secrets_for_cache()
+            )
+            for masking_secret in masking_secrets:
+                privacy_request.cache_masking_secret(masking_secret)
     if drp_request_body:
         privacy_request.cache_drp_request_body(drp_request_body)
 

fides/api/task/create_request_tasks.py

@@ -33,8 +33,8 @@ from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
-from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
+    ManualTaskAddress,
     create_manual_task_instances_for_privacy_request,
 )
 from fides.api.util.logger_context_utils import log_context

fides/api/task/deprecated_graph_task.py

@@ -1,5 +1,4 @@
 # pylint: disable=too-many-lines
-from functools import partial
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import dask
@@ -253,33 +252,16 @@ def run_erasure_request_deprecated(  # pylint: disable = too-many-arguments
 
         access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
 
-        # Build the dask task graph for erasure, ensuring we also pass along the
-        # upstream access data that may be required by the connector (e.g. BigQuery
-        # delete statements).  We accomplish this by partially applying the
-        # `inputs` kwarg on each task's `erasure_request` method.  The resulting
-        # callable accepts the original positional arguments expected by Dask.
-
-        dsk: Dict[CollectionAddress, Any] = {}
-        for k, t in env.items():
-            # Collect upstream access data in the same order as the input keys
-            upstream_access_data: List[List[Row]] = [
-                access_request_data.get(str(input_key), [])
-                for input_key in t.execution_node.input_keys
-            ]
-
-            # Bind the `inputs` keyword argument so it is supplied when the task executes
-            erasure_fn_with_inputs = partial(
-                t.erasure_request, inputs=upstream_access_data
-            )
-
-            # Build the task tuple: (callable, retrieved_data, *prereqs)
-            dsk[k] = (
-                erasure_fn_with_inputs,
+        dsk: Dict[CollectionAddress, Any] = {
+            k: (
+                t.erasure_request,
                 access_request_data.get(
                     str(k), []
-                ),  # Data retrieved for this collection
+                ),  # Pass in the results of the access request for this collection
                 *_evaluate_erasure_dependencies(t, erasure_end_nodes),
             )
+            for k, t in env.items()
+        }
 
         # root node returns 0 to be consistent with the output of the other erasure tasks
         dsk[ROOT_COLLECTION_ADDRESS] = 0

fides/api/task/execute_request_tasks.py

@@ -29,8 +29,8 @@ from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
 )
-from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
+from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.task.task_resources import TaskResources
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.util.cache import cache_task_tracking_key
@@ -298,11 +298,17 @@ def run_access_node(
                         )
                         # Currently, upstream tasks and "input keys" (which are built by data dependencies)
                         # are the same, but they may not be the same in the future.
-                        upstream_access_data: List[List[Row]] = (
-                            _build_upstream_access_data(
+                        ordered_upstream_tasks: List[Optional[RequestTask]] = (
+                            _order_tasks_by_input_key(
                                 graph_task.execution_node.input_keys, upstream_results
                             )
                         )
+                        # Pass in access data dependencies in the same order as the input keys.
+                        # If we don't have access data for an upstream node, pass in an empty list
+                        upstream_access_data: List[List[Row]] = [
+                            upstream.get_access_data() if upstream else []
+                            for upstream in ordered_upstream_tasks
+                        ]
                         # Run the main access function
                         graph_task.access_request(*upstream_access_data)
 
@@ -353,7 +359,7 @@ def run_erasure_node(
                     session,
                 ) as resources:
                     # Build GraphTask resource to facilitate execution
-                    erasure_graph_task: GraphTask = create_graph_task(
+                    graph_task: GraphTask = create_graph_task(
                         session, request_task, resources
                     )
                     # Get access data that was saved in the erasure format that was collected from the
@@ -362,24 +368,8 @@ def run_erasure_node(
                         request_task.get_data_for_erasures() or []
                     )
 
-                    upstream_access_data: List[List[Row]] = []
-
-                    try:
-                        upstream_access_data = (
-                            get_upstream_access_data_for_erasure_task(
-                                request_task, session, resources
-                            )
-                        )
-                    except Exception as e:
-                        logger.error(
-                            f"Unable to get upstream access data for erasure task {request_task.collection_address}: {e}"
-                        )
-
-                    # Run the main erasure function, passing along the upstream access data.
-                    # The extra data is currently only needed for generating BigQuery delete statements.
-                    erasure_graph_task.erasure_request(
-                        retrieved_data, inputs=upstream_access_data
-                    )
+                    # Run the main erasure function!
+                    graph_task.erasure_request(retrieved_data)
 
     queue_downstream_tasks_with_retries(
         self,
@@ -497,73 +487,6 @@ def _order_tasks_by_input_key(
     return tasks
 
 
-def get_upstream_access_data_for_erasure_task(
-    erasure_request_task: RequestTask,
-    session: Session,
-    resources: TaskResources,
-) -> List[List[Row]]:
-    """
-    Retrieves upstream access data for a given erasure request task.
-
-    This function finds the corresponding access task for the erasure task,
-    creates a GraphTask to extract input keys, and builds the upstream access data
-    needed for erasure operations (particularly for BigQuery delete statements).
-
-    Args:
-        erasure_request_task: The erasure task that needs upstream access data
-        session: Database session for querying
-        resources: Task resources for creating GraphTask
-
-    Returns:
-        List[List[Row]]: Upstream access data ordered by input keys
-
-    Raises:
-        Exception: If the corresponding access task cannot be found
-    """
-    # Get the corresponding access task for the current erasure task
-    access_request_task = (
-        session.query(RequestTask)
-        .filter(
-            RequestTask.privacy_request_id == erasure_request_task.privacy_request_id,
-            RequestTask.collection_address == erasure_request_task.collection_address,
-            RequestTask.action_type == ActionType.access,
-        )
-        .first()
-    )
-
-    if not access_request_task:
-        raise Exception(
-            f"Unable to find access request task for erasure task {erasure_request_task.collection_address}"
-        )
-
-    # Convert the request task to a GraphTask to get the input_keys
-    access_graph_task: GraphTask = create_graph_task(
-        session, access_request_task, resources
-    )
-
-    # Build and return the upstream access data
-    return _build_upstream_access_data(
-        access_graph_task.execution_node.input_keys,
-        access_request_task.upstream_tasks_objects(session),
-    )
-
-
-def _build_upstream_access_data(
-    input_keys: List[CollectionAddress],
-    upstream_tasks: Query,
-) -> List[List[Row]]:
-    """
-    Helper function to build the access data for the current node.
-    The access data is passed in the same order as the input keys.
-    If we don't have access data for an upstream node, return an empty list.
-    """
-
-    ordered_upstream: List[Optional[RequestTask]] = _order_tasks_by_input_key(
-        input_keys, upstream_tasks
-    )
-    return [task.get_access_data() if task else [] for task in ordered_upstream]
-
-
 mapping = {
     ActionType.access.value: run_access_node,
     ActionType.erasure.value: run_erasure_node,

fides/api/task/filter_results.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from fides.api.graph.config import CollectionAddress, FieldPath
 from fides.api.graph.graph import DatasetGraph
-from fides.api.task.manual.manual_task_address import ManualTaskAddress
+from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.util.collection_util import Row
 
 

fides/api/task/graph_task.py

@@ -421,7 +421,6 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         action_type: ActionType,
         ex: Optional[BaseException] = None,
         success_override_msg: Optional[str] = None,
-        record_count: Optional[int] = None,
     ) -> None:
         """On completion activities"""
         if ex:
@@ -441,23 +440,8 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 mark_current_and_downstream_nodes_as_failed(request_task, db)
         else:
             logger.info("Ending {}, {}", self.resources.request.id, self.key)
-
-            # Build standardized success message with record count
-            base_message = (
-                str(success_override_msg) if success_override_msg else "success"
-            )
-            if record_count is not None:
-                if action_type == ActionType.access:
-                    message = f"{base_message} - retrieved {record_count} records"
-                elif action_type == ActionType.erasure:
-                    message = f"{base_message} - masked {record_count} records"
-                else:
-                    message = f"{base_message} - processed {record_count} records"
-            else:
-                message = base_message
-
             self.update_status(
-                message,
+                str(success_override_msg) if success_override_msg else "success",
                 build_affected_field_logs(
                     self.execution_node, self.resources.policy, action_type
                 ),
@@ -653,11 +637,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(
-            ActionType.access,
-            success_override_msg=success_message,
-            record_count=len(filtered_output),
-        )
+        self.log_end(ActionType.access, success_override_msg=success_message)
         return filtered_output
 
     @retry(action_type=ActionType.erasure, default_return=0)
@@ -665,15 +645,9 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         self,
         retrieved_data: List[Row],
         *erasure_prereqs: int,  # TODO Remove when we stop support for DSR 2.0. DSR 3.0 enforces with downstream_tasks.
-        inputs: Optional[
-            List[List[Row]]
-        ] = None,  # Upstream data from corresponding access task
     ) -> int:
         """Run erasure request"""
 
-        if inputs is None:
-            inputs = []
-
         # if there is no primary key specified in the graph node configuration
         # note this in the execution log and perform no erasures on this node
         if (
@@ -720,10 +694,6 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             )
             return 0
 
-        formatted_input_data: NodeInput = self.pre_process_input_data(
-            *inputs, group_dependent_fields=True
-        )
-
         # Use execution context to capture postprocessor messages
         with collect_execution_log_messages() as messages:
             output = self.connector.mask_data(
@@ -732,7 +702,6 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 self.resources.request,
                 self.resources.privacy_request_task,
                 retrieved_data,
-                formatted_input_data,
             )
 
         if self.request_task.id:
@@ -751,11 +720,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(
-            ActionType.erasure,
-            success_override_msg=success_message,
-            record_count=output,
-        )
+        self.log_end(ActionType.erasure, success_override_msg=success_message)
         return output
 
     @retry(action_type=ActionType.consent, default_return=False)