ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0__py2.py3-none-any.whl

fides/api/task/conditional_dependencies/schemas.py

@@ -0,0 +1,54 @@
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class Operator(str, Enum):
+    eq = "eq"
+    neq = "neq"
+    lt = "lt"
+    lte = "lte"
+    gt = "gt"
+    gte = "gte"
+    exists = "exists"
+    not_exists = "not_exists"
+    list_contains = "list_contains"
+    not_in_list = "not_in_list"
+
+
+class GroupOperator(str, Enum):
+    and_ = "and"
+    or_ = "or"
+
+
+# Leaf condition (e.g., "user.name exists", "user.created_at >= 2024-01-01")
+class ConditionLeaf(BaseModel):
+    field_address: str = Field(
+        description="Field path to check (e.g., 'user.name', 'billing.subscription.status')"
+    )
+    operator: Operator = Field(description="Operator to apply")
+    value: Optional[
+        Union[str, int, float, bool, List[Union[str, int, float, bool]]]
+    ] = Field(default=None, description="Expected value for comparison")
+
+
+# Nested logical condition (AND, OR)
+class ConditionGroup(BaseModel):
+    logical_operator: GroupOperator = Field(
+        description="Logical operator: 'and' or 'or'"
+    )
+    conditions: list["Condition"] = Field(
+        description="List of conditions or nested groups"
+    )
+
+    @model_validator(mode="after")
+    def validate_conditions(self) -> "ConditionGroup":
+        if not self.conditions:
+            raise ValueError("conditions list cannot be empty")
+        return self
+
+
+# Use model_rebuild to allow recursive nesting
+Condition = Union[ConditionLeaf, ConditionGroup]
+ConditionGroup.model_rebuild()

fides/api/task/create_request_tasks.py

@@ -33,8 +33,8 @@ from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    ManualTaskAddress,
     create_manual_task_instances_for_privacy_request,
 )
 from fides.api.util.logger_context_utils import log_context

fides/api/task/deprecated_graph_task.py

@@ -1,4 +1,5 @@
 # pylint: disable=too-many-lines
+from functools import partial
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import dask
@@ -252,16 +253,33 @@ def run_erasure_request_deprecated(  # pylint: disable = too-many-arguments
 
         access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
 
-        dsk: Dict[CollectionAddress, Any] = {
-            k: (
-                t.erasure_request,
+        # Build the dask task graph for erasure, ensuring we also pass along the
+        # upstream access data that may be required by the connector (e.g. BigQuery
+        # delete statements).  We accomplish this by partially applying the
+        # `inputs` kwarg on each task's `erasure_request` method.  The resulting
+        # callable accepts the original positional arguments expected by Dask.
+
+        dsk: Dict[CollectionAddress, Any] = {}
+        for k, t in env.items():
+            # Collect upstream access data in the same order as the input keys
+            upstream_access_data: List[List[Row]] = [
+                access_request_data.get(str(input_key), [])
+                for input_key in t.execution_node.input_keys
+            ]
+
+            # Bind the `inputs` keyword argument so it is supplied when the task executes
+            erasure_fn_with_inputs = partial(
+                t.erasure_request, inputs=upstream_access_data
+            )
+
+            # Build the task tuple: (callable, retrieved_data, *prereqs)
+            dsk[k] = (
+                erasure_fn_with_inputs,
                 access_request_data.get(
                     str(k), []
-                ),  # Pass in the results of the access request for this collection
+                ),  # Data retrieved for this collection
                 *_evaluate_erasure_dependencies(t, erasure_end_nodes),
             )
-            for k, t in env.items()
-        }
 
         # root node returns 0 to be consistent with the output of the other erasure tasks
         dsk[ROOT_COLLECTION_ADDRESS] = 0

fides/api/task/execute_request_tasks.py

@@ -29,13 +29,14 @@ from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
 )
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
-from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.task.task_resources import TaskResources
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.util.cache import cache_task_tracking_key
 from fides.api.util.collection_util import Row
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
+from fides.api.util.memory_watchdog import memory_limiter
 
 # DSR 3.0 task functions
 
@@ -255,6 +256,7 @@ def queue_downstream_tasks(
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
+@memory_limiter
 @log_context(
     capture_args={
         "privacy_request_id": LoggerContextKeys.privacy_request_id,
@@ -298,17 +300,11 @@ def run_access_node(
                         )
                         # Currently, upstream tasks and "input keys" (which are built by data dependencies)
                         # are the same, but they may not be the same in the future.
-                        ordered_upstream_tasks: List[Optional[RequestTask]] = (
-                            _order_tasks_by_input_key(
+                        upstream_access_data: List[List[Row]] = (
+                            _build_upstream_access_data(
                                 graph_task.execution_node.input_keys, upstream_results
                             )
                         )
-                        # Pass in access data dependencies in the same order as the input keys.
-                        # If we don't have access data for an upstream node, pass in an empty list
-                        upstream_access_data: List[List[Row]] = [
-                            upstream.get_access_data() if upstream else []
-                            for upstream in ordered_upstream_tasks
-                        ]
                         # Run the main access function
                         graph_task.access_request(*upstream_access_data)
 
@@ -325,6 +321,7 @@ def run_access_node(
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
+@memory_limiter
 @log_context(
     capture_args={
         "privacy_request_id": LoggerContextKeys.privacy_request_id,
@@ -359,7 +356,7 @@ def run_erasure_node(
                     session,
                 ) as resources:
                     # Build GraphTask resource to facilitate execution
-                    graph_task: GraphTask = create_graph_task(
+                    erasure_graph_task: GraphTask = create_graph_task(
                         session, request_task, resources
                     )
                     # Get access data that was saved in the erasure format that was collected from the
@@ -368,8 +365,24 @@ def run_erasure_node(
                         request_task.get_data_for_erasures() or []
                     )
 
-                    # Run the main erasure function!
-                    graph_task.erasure_request(retrieved_data)
+                    upstream_access_data: List[List[Row]] = []
+
+                    try:
+                        upstream_access_data = (
+                            get_upstream_access_data_for_erasure_task(
+                                request_task, session, resources
+                            )
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Unable to get upstream access data for erasure task {request_task.collection_address}: {e}"
+                        )
+
+                    # Run the main erasure function, passing along the upstream access data.
+                    # The extra data is currently only needed for generating BigQuery delete statements.
+                    erasure_graph_task.erasure_request(
+                        retrieved_data, inputs=upstream_access_data
+                    )
 
     queue_downstream_tasks_with_retries(
         self,
@@ -381,6 +394,7 @@ def run_erasure_node(
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
+@memory_limiter
 @log_context(
     capture_args={
         "privacy_request_id": LoggerContextKeys.privacy_request_id,
@@ -487,6 +501,73 @@ def _order_tasks_by_input_key(
     return tasks
 
 
+def get_upstream_access_data_for_erasure_task(
+    erasure_request_task: RequestTask,
+    session: Session,
+    resources: TaskResources,
+) -> List[List[Row]]:
+    """
+    Retrieves upstream access data for a given erasure request task.
+
+    This function finds the corresponding access task for the erasure task,
+    creates a GraphTask to extract input keys, and builds the upstream access data
+    needed for erasure operations (particularly for BigQuery delete statements).
+
+    Args:
+        erasure_request_task: The erasure task that needs upstream access data
+        session: Database session for querying
+        resources: Task resources for creating GraphTask
+
+    Returns:
+        List[List[Row]]: Upstream access data ordered by input keys
+
+    Raises:
+        Exception: If the corresponding access task cannot be found
+    """
+    # Get the corresponding access task for the current erasure task
+    access_request_task = (
+        session.query(RequestTask)
+        .filter(
+            RequestTask.privacy_request_id == erasure_request_task.privacy_request_id,
+            RequestTask.collection_address == erasure_request_task.collection_address,
+            RequestTask.action_type == ActionType.access,
+        )
+        .first()
+    )
+
+    if not access_request_task:
+        raise Exception(
+            f"Unable to find access request task for erasure task {erasure_request_task.collection_address}"
+        )
+
+    # Convert the request task to a GraphTask to get the input_keys
+    access_graph_task: GraphTask = create_graph_task(
+        session, access_request_task, resources
+    )
+
+    # Build and return the upstream access data
+    return _build_upstream_access_data(
+        access_graph_task.execution_node.input_keys,
+        access_request_task.upstream_tasks_objects(session),
+    )
+
+
+def _build_upstream_access_data(
+    input_keys: List[CollectionAddress],
+    upstream_tasks: Query,
+) -> List[List[Row]]:
+    """
+    Helper function to build the access data for the current node.
+    The access data is passed in the same order as the input keys.
+    If we don't have access data for an upstream node, return an empty list.
+    """
+
+    ordered_upstream: List[Optional[RequestTask]] = _order_tasks_by_input_key(
+        input_keys, upstream_tasks
+    )
+    return [task.get_access_data() if task else [] for task in ordered_upstream]
+
+
 mapping = {
     ActionType.access.value: run_access_node,
     ActionType.erasure.value: run_erasure_node,

fides/api/task/filter_results.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from fides.api.graph.config import CollectionAddress, FieldPath
 from fides.api.graph.graph import DatasetGraph
-from fides.api.task.manual.manual_task_utils import ManualTaskAddress
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.util.collection_util import Row
 
 

fides/api/task/graph_task.py

@@ -18,6 +18,7 @@ from fides.api.common_exceptions import (
     NotSupportedForCollection,
     PrivacyRequestErasureEmailSendRequired,
     SkippingConsentPropagation,
+    TableNotFound,
 )
 from fides.api.graph.config import (
     ROOT_COLLECTION_ADDRESS,
@@ -61,6 +62,7 @@ from fides.api.util.consent_util import (
 )
 from fides.api.util.logger import Pii
 from fides.api.util.logger_context_utils import LoggerContextKeys
+from fides.api.util.memory_watchdog import MemoryLimitExceeded
 from fides.api.util.saas_util import FIDESOPS_GROUPED_INPUTS
 from fides.config import CONFIG
 
@@ -70,6 +72,16 @@ EMPTY_REQUEST = PrivacyRequest()
 EMPTY_REQUEST_TASK = RequestTask()
 
 
+def _is_memory_limit_exceeded(exception: BaseException) -> bool:
+    """Check if the exception or any exception in its chain is a MemoryLimitExceeded."""
+    current_exception: Optional[BaseException] = exception
+    while current_exception:
+        if isinstance(current_exception, MemoryLimitExceeded):
+            return True
+        current_exception = current_exception.__cause__ or current_exception.__context__
+    return False
+
+
 def retry(
     action_type: ActionType,
     default_return: Any,
@@ -126,6 +138,7 @@ def retry(
                     CollectionDisabled,
                     ActionDisabled,
                     NotSupportedForCollection,
+                    TableNotFound,
                 ) as exc:
                     logger.warning(
                         "{} - Skipping collection {} for privacy_request: {}",
@@ -144,7 +157,31 @@ def retry(
                     self.log_skipped(action_type, exc)
                     self.cache_system_status_for_preferences()
                     return default_return
+                except MemoryLimitExceeded as ex:
+                    # Hard failure – mark task & downstream as errored and abort.
+                    logger.error(
+                        "Memory watchdog exceeded ({}%). Aborting {} {} without retry.",
+                        ex.memory_percent,
+                        method_name,
+                        self.execution_node.address,
+                    )
+                    # Persist error status and create execution logs before raising
+                    self.log_end(action_type, ex)
+                    self.add_error_status_for_consent_reporting()
+                    raise
                 except BaseException as ex:  # pylint: disable=W0703
+                    # Check if this exception was caused by memory limit exceeded
+                    if _is_memory_limit_exceeded(ex):
+                        logger.error(
+                            "Memory watchdog exceeded (wrapped exception). Aborting {} {} without retry.",
+                            method_name,
+                            self.execution_node.address,
+                        )
+                        # Persist error status and create execution logs before raising
+                        self.log_end(action_type, ex)
+                        self.add_error_status_for_consent_reporting()
+                        raise
+
                     traceback.print_exc()
                     func_delay *= CONFIG.execution.task_retry_backoff
                     logger.warning(
@@ -421,6 +458,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         action_type: ActionType,
         ex: Optional[BaseException] = None,
         success_override_msg: Optional[str] = None,
+        record_count: Optional[int] = None,
     ) -> None:
         """On completion activities"""
         if ex:
@@ -440,8 +478,23 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 mark_current_and_downstream_nodes_as_failed(request_task, db)
         else:
             logger.info("Ending {}, {}", self.resources.request.id, self.key)
+
+            # Build standardized success message with record count
+            base_message = (
+                str(success_override_msg) if success_override_msg else "success"
+            )
+            if record_count is not None:
+                if action_type == ActionType.access:
+                    message = f"{base_message} - retrieved {record_count} records"
+                elif action_type == ActionType.erasure:
+                    message = f"{base_message} - masked {record_count} records"
+                else:
+                    message = f"{base_message} - processed {record_count} records"
+            else:
+                message = base_message
+
             self.update_status(
-                str(success_override_msg) if success_override_msg else "success",
+                message,
                 build_affected_field_logs(
                     self.execution_node, self.resources.policy, action_type
                 ),
@@ -537,12 +590,21 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         # For access request results, mutate rows in-place to remove non-matching
         # array elements.  We already iterated over `output` above, so reuse the same
         # loop structure to keep cache locality.
+        logger.info(
+            "Filtering {} rows in {} for matching array elements.",
+            len(output),
+            self.execution_node.address,
+        )
         for row in output:
+            filter_element_match(row, post_processed_node_input_data)
+
+        if len(output) > 0:
             logger.info(
-                "Filtering row in {} for matching array elements.",
+                "Filtering completed for {} rows in {}. Post-processed node size: {}",
+                len(output),
                 self.execution_node.address,
+                len(post_processed_node_input_data),
             )
-            filter_element_match(row, post_processed_node_input_data)
 
         if self.request_task.id:
             # Saves intermediate access results for DSR 3.0 directly on the Request Task
@@ -637,7 +699,11 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(ActionType.access, success_override_msg=success_message)
+        self.log_end(
+            ActionType.access,
+            success_override_msg=success_message,
+            record_count=len(filtered_output),
+        )
         return filtered_output
 
     @retry(action_type=ActionType.erasure, default_return=0)
@@ -645,9 +711,15 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         self,
         retrieved_data: List[Row],
         *erasure_prereqs: int,  # TODO Remove when we stop support for DSR 2.0. DSR 3.0 enforces with downstream_tasks.
+        inputs: Optional[
+            List[List[Row]]
+        ] = None,  # Upstream data from corresponding access task
     ) -> int:
         """Run erasure request"""
 
+        if inputs is None:
+            inputs = []
+
         # if there is no primary key specified in the graph node configuration
         # note this in the execution log and perform no erasures on this node
         if (
@@ -694,6 +766,10 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             )
             return 0
 
+        formatted_input_data: NodeInput = self.pre_process_input_data(
+            *inputs, group_dependent_fields=True
+        )
+
         # Use execution context to capture postprocessor messages
         with collect_execution_log_messages() as messages:
             output = self.connector.mask_data(
@@ -702,6 +778,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 self.resources.request,
                 self.resources.privacy_request_task,
                 retrieved_data,
+                formatted_input_data,
             )
 
         if self.request_task.id:
@@ -720,7 +797,11 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(ActionType.erasure, success_override_msg=success_message)
+        self.log_end(
+            ActionType.erasure,
+            success_override_msg=success_message,
+            record_count=output,
+        )
         return output
 
     @retry(action_type=ActionType.consent, default_return=False)

fides/api/task/manual/manual_task_address.py

@@ -0,0 +1,46 @@
+from fides.api.graph.config import CollectionAddress
+
+
+class ManualTaskAddress:
+    """Utility class for creating and parsing manual task addresses"""
+
+    MANUAL_DATA_COLLECTION = "manual_data"
+
+    @staticmethod
+    def create(connection_config_key: str) -> CollectionAddress:
+        """Create a CollectionAddress for manual data: {connection_key}:manual_data"""
+        return CollectionAddress(
+            dataset=connection_config_key,
+            collection=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+        )
+
+    @staticmethod
+    def _is_manual_data_collection(collection_name: str) -> bool:
+        """Check if collection name represents manual task data"""
+        return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
+
+    @staticmethod
+    def is_manual_task_address(address: CollectionAddress) -> bool:
+        """Check if address represents manual task data"""
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            _, _, collection_part = address.partition(":")
+            if not collection_part:
+                return False
+            return ManualTaskAddress._is_manual_data_collection(collection_part)
+
+        # Handle CollectionAddress object
+        return ManualTaskAddress._is_manual_data_collection(address.collection)
+
+    @staticmethod
+    def get_connection_key(address: CollectionAddress) -> str:
+        """Extract connection config key from manual task address"""
+        if not ManualTaskAddress.is_manual_task_address(address):
+            raise ValueError(f"Not a manual task address: {address}")
+
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.split(":")[0]
+
+        # Handle CollectionAddress object
+        return address.dataset