ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.66.3b0__py2.py3-none-any.whl

fides/api/task/conditional_dependencies/evaluator.py

@@ -0,0 +1,109 @@
+import operator as py_operator
+from typing import Any, Union
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.graph.config import FieldPath
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+    GroupOperator,
+    Operator,
+)
+
+operator_methods = {
+    Operator.exists: lambda a, _: a is not None,
+    Operator.not_exists: lambda a, _: a is None,
+    Operator.eq: py_operator.eq,
+    Operator.neq: py_operator.ne,
+    Operator.lt: lambda a, b: a < b if a is not None else False,
+    Operator.lte: lambda a, b: a <= b if a is not None else False,
+    Operator.gt: lambda a, b: a > b if a is not None else False,
+    Operator.gte: lambda a, b: a >= b if a is not None else False,
+    Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
+    Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
+}
+
+
+class ConditionEvaluator:
+    """Evaluates nested conditions for manual task creation"""
+
+    def __init__(self, db: Session):
+        self.db = db
+
+    def evaluate_rule(self, rule: Condition, data: Union[dict, Any]) -> bool:
+        """Evaluate a nested condition rule against input data"""
+        if isinstance(rule, ConditionLeaf):
+            return self._evaluate_leaf_condition(rule, data)
+        # ConditionGroup
+        return self._evaluate_group_condition(rule, data)
+
+    def _evaluate_leaf_condition(
+        self, condition: ConditionLeaf, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a leaf condition against input data"""
+        actual_value = self._get_nested_value(data, condition.field_address.split("."))
+        # Apply operator and return result
+        return self._apply_operator(actual_value, condition.operator, condition.value)
+
+    def _evaluate_group_condition(
+        self, group: ConditionGroup, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a group condition against input data"""
+        results = [
+            self.evaluate_rule(condition, data) for condition in group.conditions
+        ]
+
+        logical_operators = {GroupOperator.and_: all, GroupOperator.or_: any}
+        operator_func = logical_operators.get(group.logical_operator)
+
+        if operator_func is None:
+            logger.warning(f"Unknown logical operator: {group.logical_operator}")
+            return False
+
+        return operator_func(results)
+
+    def _get_nested_value(self, data: Union[dict, Any], keys: list[str]) -> Any:
+        """Get nested value from data using dot notation
+
+        Supports both simple dictionary access and Fides reference structures:
+        - Simple dict: data["user"]["name"]
+        - Fides FieldAddress: data.get_field_value(FieldAddress("dataset", "collection", "field_address"))
+        - Fides Collection: data.get_field_value(FieldPath("field_address", "subfield"))
+        """
+        if not keys:
+            return data
+
+        current = data
+
+        # Try Fides reference structures first
+        if hasattr(current, "get_field_value"):
+            try:
+                field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
+                return current.get_field_value(field_path)
+            except (AttributeError, ValueError):
+                pass
+
+        # Fall back to dictionary access
+        for key in keys:
+            if not isinstance(current, dict):
+                current = current.get(key, {}) if hasattr(current, "get") else None
+            else:
+                current = current.get(key, {})
+
+        return current if current != {} else None
+
+    def _apply_operator(
+        self, actual_value: Any, operator: Operator, expected_value: Any
+    ) -> bool:
+        """Apply operator to actual and expected values"""
+
+        # Get the method for the operator and execute it
+        operator_method = operator_methods.get(operator)
+        if operator_method is None:
+            logger.warning(f"Unknown operator: {operator}")
+            return False
+
+        return operator_method(actual_value, expected_value)

fides/api/task/conditional_dependencies/schemas.py

@@ -0,0 +1,54 @@
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class Operator(str, Enum):
+    eq = "eq"
+    neq = "neq"
+    lt = "lt"
+    lte = "lte"
+    gt = "gt"
+    gte = "gte"
+    exists = "exists"
+    not_exists = "not_exists"
+    list_contains = "list_contains"
+    not_in_list = "not_in_list"
+
+
+class GroupOperator(str, Enum):
+    and_ = "and"
+    or_ = "or"
+
+
+# Leaf condition (e.g., "user.name exists", "user.created_at >= 2024-01-01")
+class ConditionLeaf(BaseModel):
+    field_address: str = Field(
+        description="Field path to check (e.g., 'user.name', 'billing.subscription.status')"
+    )
+    operator: Operator = Field(description="Operator to apply")
+    value: Optional[
+        Union[str, int, float, bool, List[Union[str, int, float, bool]]]
+    ] = Field(default=None, description="Expected value for comparison")
+
+
+# Nested logical condition (AND, OR)
+class ConditionGroup(BaseModel):
+    logical_operator: GroupOperator = Field(
+        description="Logical operator: 'and' or 'or'"
+    )
+    conditions: list["Condition"] = Field(
+        description="List of conditions or nested groups"
+    )
+
+    @model_validator(mode="after")
+    def validate_conditions(self) -> "ConditionGroup":
+        if not self.conditions:
+            raise ValueError("conditions list cannot be empty")
+        return self
+
+
+# Use model_rebuild to allow recursive nesting
+Condition = Union[ConditionLeaf, ConditionGroup]
+ConditionGroup.model_rebuild()

fides/api/task/create_request_tasks.py

@@ -33,8 +33,8 @@ from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    ManualTaskAddress,
     create_manual_task_instances_for_privacy_request,
 )
 from fides.api.util.logger_context_utils import log_context

fides/api/task/deprecated_graph_task.py

@@ -1,4 +1,5 @@
 # pylint: disable=too-many-lines
+from functools import partial
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import dask
@@ -252,16 +253,33 @@ def run_erasure_request_deprecated(  # pylint: disable = too-many-arguments
 
         access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
 
-        dsk: Dict[CollectionAddress, Any] = {
-            k: (
-                t.erasure_request,
+        # Build the dask task graph for erasure, ensuring we also pass along the
+        # upstream access data that may be required by the connector (e.g. BigQuery
+        # delete statements).  We accomplish this by partially applying the
+        # `inputs` kwarg on each task's `erasure_request` method.  The resulting
+        # callable accepts the original positional arguments expected by Dask.
+
+        dsk: Dict[CollectionAddress, Any] = {}
+        for k, t in env.items():
+            # Collect upstream access data in the same order as the input keys
+            upstream_access_data: List[List[Row]] = [
+                access_request_data.get(str(input_key), [])
+                for input_key in t.execution_node.input_keys
+            ]
+
+            # Bind the `inputs` keyword argument so it is supplied when the task executes
+            erasure_fn_with_inputs = partial(
+                t.erasure_request, inputs=upstream_access_data
+            )
+
+            # Build the task tuple: (callable, retrieved_data, *prereqs)
+            dsk[k] = (
+                erasure_fn_with_inputs,
                 access_request_data.get(
                     str(k), []
-                ),  # Pass in the results of the access request for this collection
+                ),  # Data retrieved for this collection
                 *_evaluate_erasure_dependencies(t, erasure_end_nodes),
             )
-            for k, t in env.items()
-        }
 
         # root node returns 0 to be consistent with the output of the other erasure tasks
         dsk[ROOT_COLLECTION_ADDRESS] = 0

fides/api/task/execute_request_tasks.py

@@ -29,8 +29,8 @@ from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
 )
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
-from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.task.task_resources import TaskResources
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.util.cache import cache_task_tracking_key
@@ -298,17 +298,11 @@ def run_access_node(
                         )
                         # Currently, upstream tasks and "input keys" (which are built by data dependencies)
                         # are the same, but they may not be the same in the future.
-                        ordered_upstream_tasks: List[Optional[RequestTask]] = (
-                            _order_tasks_by_input_key(
+                        upstream_access_data: List[List[Row]] = (
+                            _build_upstream_access_data(
                                 graph_task.execution_node.input_keys, upstream_results
                             )
                         )
-                        # Pass in access data dependencies in the same order as the input keys.
-                        # If we don't have access data for an upstream node, pass in an empty list
-                        upstream_access_data: List[List[Row]] = [
-                            upstream.get_access_data() if upstream else []
-                            for upstream in ordered_upstream_tasks
-                        ]
                         # Run the main access function
                         graph_task.access_request(*upstream_access_data)
 
@@ -359,7 +353,7 @@ def run_erasure_node(
                     session,
                 ) as resources:
                     # Build GraphTask resource to facilitate execution
-                    graph_task: GraphTask = create_graph_task(
+                    erasure_graph_task: GraphTask = create_graph_task(
                         session, request_task, resources
                     )
                     # Get access data that was saved in the erasure format that was collected from the
@@ -368,8 +362,24 @@ def run_erasure_node(
                         request_task.get_data_for_erasures() or []
                     )
 
-                    # Run the main erasure function!
-                    graph_task.erasure_request(retrieved_data)
+                    upstream_access_data: List[List[Row]] = []
+
+                    try:
+                        upstream_access_data = (
+                            get_upstream_access_data_for_erasure_task(
+                                request_task, session, resources
+                            )
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Unable to get upstream access data for erasure task {request_task.collection_address}: {e}"
+                        )
+
+                    # Run the main erasure function, passing along the upstream access data.
+                    # The extra data is currently only needed for generating BigQuery delete statements.
+                    erasure_graph_task.erasure_request(
+                        retrieved_data, inputs=upstream_access_data
+                    )
 
     queue_downstream_tasks_with_retries(
         self,
@@ -487,6 +497,73 @@ def _order_tasks_by_input_key(
     return tasks
 
 
+def get_upstream_access_data_for_erasure_task(
+    erasure_request_task: RequestTask,
+    session: Session,
+    resources: TaskResources,
+) -> List[List[Row]]:
+    """
+    Retrieves upstream access data for a given erasure request task.
+
+    This function finds the corresponding access task for the erasure task,
+    creates a GraphTask to extract input keys, and builds the upstream access data
+    needed for erasure operations (particularly for BigQuery delete statements).
+
+    Args:
+        erasure_request_task: The erasure task that needs upstream access data
+        session: Database session for querying
+        resources: Task resources for creating GraphTask
+
+    Returns:
+        List[List[Row]]: Upstream access data ordered by input keys
+
+    Raises:
+        Exception: If the corresponding access task cannot be found
+    """
+    # Get the corresponding access task for the current erasure task
+    access_request_task = (
+        session.query(RequestTask)
+        .filter(
+            RequestTask.privacy_request_id == erasure_request_task.privacy_request_id,
+            RequestTask.collection_address == erasure_request_task.collection_address,
+            RequestTask.action_type == ActionType.access,
+        )
+        .first()
+    )
+
+    if not access_request_task:
+        raise Exception(
+            f"Unable to find access request task for erasure task {erasure_request_task.collection_address}"
+        )
+
+    # Convert the request task to a GraphTask to get the input_keys
+    access_graph_task: GraphTask = create_graph_task(
+        session, access_request_task, resources
+    )
+
+    # Build and return the upstream access data
+    return _build_upstream_access_data(
+        access_graph_task.execution_node.input_keys,
+        access_request_task.upstream_tasks_objects(session),
+    )
+
+
+def _build_upstream_access_data(
+    input_keys: List[CollectionAddress],
+    upstream_tasks: Query,
+) -> List[List[Row]]:
+    """
+    Helper function to build the access data for the current node.
+    The access data is passed in the same order as the input keys.
+    If we don't have access data for an upstream node, return an empty list.
+    """
+
+    ordered_upstream: List[Optional[RequestTask]] = _order_tasks_by_input_key(
+        input_keys, upstream_tasks
+    )
+    return [task.get_access_data() if task else [] for task in ordered_upstream]
+
+
 mapping = {
     ActionType.access.value: run_access_node,
     ActionType.erasure.value: run_erasure_node,

fides/api/task/filter_results.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from fides.api.graph.config import CollectionAddress, FieldPath
 from fides.api.graph.graph import DatasetGraph
-from fides.api.task.manual.manual_task_utils import ManualTaskAddress
+from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.util.collection_util import Row
 
 

fides/api/task/graph_task.py

@@ -421,6 +421,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         action_type: ActionType,
         ex: Optional[BaseException] = None,
         success_override_msg: Optional[str] = None,
+        record_count: Optional[int] = None,
     ) -> None:
         """On completion activities"""
         if ex:
@@ -440,8 +441,23 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 mark_current_and_downstream_nodes_as_failed(request_task, db)
         else:
             logger.info("Ending {}, {}", self.resources.request.id, self.key)
+
+            # Build standardized success message with record count
+            base_message = (
+                str(success_override_msg) if success_override_msg else "success"
+            )
+            if record_count is not None:
+                if action_type == ActionType.access:
+                    message = f"{base_message} - retrieved {record_count} records"
+                elif action_type == ActionType.erasure:
+                    message = f"{base_message} - masked {record_count} records"
+                else:
+                    message = f"{base_message} - processed {record_count} records"
+            else:
+                message = base_message
+
             self.update_status(
-                str(success_override_msg) if success_override_msg else "success",
+                message,
                 build_affected_field_logs(
                     self.execution_node, self.resources.policy, action_type
                 ),
@@ -637,7 +653,11 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(ActionType.access, success_override_msg=success_message)
+        self.log_end(
+            ActionType.access,
+            success_override_msg=success_message,
+            record_count=len(filtered_output),
+        )
         return filtered_output
 
     @retry(action_type=ActionType.erasure, default_return=0)
@@ -645,9 +665,15 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         self,
         retrieved_data: List[Row],
         *erasure_prereqs: int,  # TODO Remove when we stop support for DSR 2.0. DSR 3.0 enforces with downstream_tasks.
+        inputs: Optional[
+            List[List[Row]]
+        ] = None,  # Upstream data from corresponding access task
     ) -> int:
         """Run erasure request"""
 
+        if inputs is None:
+            inputs = []
+
         # if there is no primary key specified in the graph node configuration
         # note this in the execution log and perform no erasures on this node
         if (
@@ -694,6 +720,10 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             )
             return 0
 
+        formatted_input_data: NodeInput = self.pre_process_input_data(
+            *inputs, group_dependent_fields=True
+        )
+
         # Use execution context to capture postprocessor messages
         with collect_execution_log_messages() as messages:
             output = self.connector.mask_data(
@@ -702,6 +732,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 self.resources.request,
                 self.resources.privacy_request_task,
                 retrieved_data,
+                formatted_input_data,
             )
 
         if self.request_task.id:
@@ -720,7 +751,11 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         if messages:
             success_message = "\n".join(messages)
 
-        self.log_end(ActionType.erasure, success_override_msg=success_message)
+        self.log_end(
+            ActionType.erasure,
+            success_override_msg=success_message,
+            record_count=output,
+        )
         return output
 
     @retry(action_type=ActionType.consent, default_return=False)

fides/api/task/manual/manual_task_address.py

@@ -0,0 +1,46 @@
+from fides.api.graph.config import CollectionAddress
+
+
+class ManualTaskAddress:
+    """Utility class for creating and parsing manual task addresses"""
+
+    MANUAL_DATA_COLLECTION = "manual_data"
+
+    @staticmethod
+    def create(connection_config_key: str) -> CollectionAddress:
+        """Create a CollectionAddress for manual data: {connection_key}:manual_data"""
+        return CollectionAddress(
+            dataset=connection_config_key,
+            collection=ManualTaskAddress.MANUAL_DATA_COLLECTION,
+        )
+
+    @staticmethod
+    def _is_manual_data_collection(collection_name: str) -> bool:
+        """Check if collection name represents manual task data"""
+        return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
+
+    @staticmethod
+    def is_manual_task_address(address: CollectionAddress) -> bool:
+        """Check if address represents manual task data"""
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            _, _, collection_part = address.partition(":")
+            if not collection_part:
+                return False
+            return ManualTaskAddress._is_manual_data_collection(collection_part)
+
+        # Handle CollectionAddress object
+        return ManualTaskAddress._is_manual_data_collection(address.collection)
+
+    @staticmethod
+    def get_connection_key(address: CollectionAddress) -> str:
+        """Extract connection config key from manual task address"""
+        if not ManualTaskAddress.is_manual_task_address(address):
+            raise ValueError(f"Not a manual task address: {address}")
+
+        if isinstance(address, str):
+            # Handle string format "connection_key:collection_name"
+            return address.split(":")[0]
+
+        # Handle CollectionAddress object
+        return address.dataset