ethyca-fides 2.66.1rc0__py2.py3-none-any.whl → 2.66.2b0__py2.py3-none-any.whl

fides/api/service/connectors/rds_mysql_connector.py

@@ -158,6 +158,7 @@ class RDSMySQLConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS MySQL"""
         return 0

fides/api/service/connectors/rds_postgres_connector.py

@@ -147,6 +147,7 @@ class RDSPostgresConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS Postgres"""
         return 0

fides/api/service/connectors/s3_connector.py

@@ -66,6 +66,7 @@ class S3Connector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for S3"""
         return 0

fides/api/service/connectors/saas_connector.py

@@ -517,6 +517,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Return the number of rows that have been updated."""
         self.set_privacy_request_state(privacy_request, node, request_task)

fides/api/service/connectors/scylla_connector.py

@@ -150,6 +150,7 @@ class ScyllaConnector(BaseConnector[Cluster]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)

fides/api/service/connectors/sql_connector.py

@@ -27,11 +27,13 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration import ConnectionConfigSecretsSchema
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
 from fides.api.util.collection_util import Row
 from fides.config import get_config
+from fides.config.config_proxy import ConfigProxy
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     Dataset as CtlDataset,
@@ -58,6 +60,23 @@ class SQLConnector(BaseConnector[Engine]):
             )
         self.ssh_server: sshtunnel._ForwardServer = None
 
+    def should_dry_run(self, mode_to_check: SqlDryRunMode) -> bool:
+        """
+        Check if SQL dry run is enabled for the specified mode.
+
+        Args:
+            mode_to_check: The SqlDryRunMode to check for
+
+        Returns:
+            bool: True if the current mode matches the mode to check
+        """
+        from fides.api.api.deps import get_autoclose_db_session as get_db
+
+        with get_db() as db:
+            config_proxy = ConfigProxy(db)
+            current_mode = getattr(config_proxy.execution, "sql_dry_run", None)
+            return current_mode == mode_to_check
+
     @staticmethod
     def cursor_result_to_rows(results: CursorResult) -> List[Row]:
         """Convert SQLAlchemy results to a list of dictionaries"""
@@ -140,6 +159,10 @@ class SQLConnector(BaseConnector[Engine]):
         if query is None:
             return []
 
+        if self.should_dry_run(SqlDryRunMode.access):
+            logger.warning(f"SQL DRY RUN - Would execute SQL: {query}")
+            return []
+
         with client.connect() as connection:
             self.set_schema(connection)
             results = connection.execute(query)
@@ -160,6 +183,10 @@ class SQLConnector(BaseConnector[Engine]):
         if stmt is None:
             return []
 
+        if self.should_dry_run(SqlDryRunMode.access):
+            logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
+            return []
+
         logger.info("Starting data retrieval for {}", node.address)
         with client.connect() as connection:
             self.set_schema(connection)
@@ -178,20 +205,25 @@ class SQLConnector(BaseConnector[Engine]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records masked"""
         query_config = self.query_config(node)
         update_ct = 0
         client = self.client()
+
         for row in rows:
             update_stmt: Optional[TextClause] = query_config.generate_update_stmt(
                 row, policy, privacy_request
             )
             if update_stmt is not None:
-                with client.connect() as connection:
-                    self.set_schema(connection)
-                    results: LegacyCursorResult = connection.execute(update_stmt)
-                    update_ct = update_ct + results.rowcount
+                if self.should_dry_run(SqlDryRunMode.erasure):
+                    logger.warning(f"SQL DRY RUN - Would execute SQL: {update_stmt}")
+                else:
+                    with client.connect() as connection:
+                        self.set_schema(connection)
+                        results: LegacyCursorResult = connection.execute(update_stmt)
+                        update_ct = update_ct + results.rowcount
         return update_ct
 
     def close(self) -> None:

fides/api/service/connectors/website_connector.py

@@ -75,6 +75,7 @@ class WebsiteConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for website connector"""
         return 0

fides/api/service/privacy_request/request_runner_service.py

@@ -14,6 +14,7 @@ from fides.api.common_exceptions import (
     ClientUnsuccessfulException,
     IdentityNotFoundException,
     ManualWebhookFieldsUnset,
+    MaskingSecretsExpired,
     MessageDispatchException,
     NoCachedManualWebhookEntry,
     PrivacyRequestExit,
@@ -75,6 +76,7 @@ from fides.api.task.graph_task import (
 from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
 from fides.api.tasks import DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
+from fides.api.util.cache import get_all_masking_secret_keys
 from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
@@ -531,6 +533,8 @@ def run_privacy_request(
                     request_checkpoint=CurrentStep.erasure, from_checkpoint=resume_step
                 ):
                     privacy_request.cache_failed_checkpoint_details(CurrentStep.erasure)
+                    _verify_masking_secrets(policy, privacy_request, resume_step)
+
                     # We only need to run the erasure once until masking strategies are handled
                     erasure_runner(
                         privacy_request=privacy_request,
@@ -1000,3 +1004,25 @@ def run_webhooks_and_report_status(
             return False
 
     return True
+
+
+def _verify_masking_secrets(
+    policy: Policy, privacy_request: PrivacyRequest, resume_step: Optional[CurrentStep]
+) -> None:
+    """
+    Checks that the required masking secrets are still cached for the given request.
+    Raises an exception if masking secrets are needed for the given policy but they don't exist.
+    """
+
+    if resume_step is None:
+        return
+
+    # if masking can be performed without any masking secrets, we skip the cache check
+    if (
+        policy.generate_masking_secrets()
+        and not get_all_masking_secret_keys(privacy_request.id)
+        and not privacy_request.masking_secrets
+    ):
+        raise MaskingSecretsExpired(
+            f"The masking secrets for privacy request ID '{privacy_request.id}' have expired. Please submit a new erasure request."
+        )

fides/api/service/privacy_request/request_service.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import json
 from asyncio import sleep
 from datetime import datetime, timedelta
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, Optional, Set
 
 from httpx import AsyncClient
 from loguru import logger
@@ -11,7 +11,6 @@ from sqlalchemy import text
 from sqlalchemy.sql.elements import TextClause
 
 from fides.api.common_exceptions import PrivacyRequestNotFound
-from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
     PrivacyRequest,
@@ -19,14 +18,12 @@ from fides.api.models.privacy_request import (
 )
 from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
-from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import (
     PrivacyRequestResponse,
     PrivacyRequestStatus,
 )
 from fides.api.schemas.redis_cache import Identity
-from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
 from fides.api.util.cache import (
@@ -69,7 +66,6 @@ def build_required_privacy_request_kwargs(
 
 def cache_data(
     privacy_request: PrivacyRequest,
-    policy: Policy,
     identity: Identity,
     encryption_key: Optional[str],
     drp_request_body: Optional[DrpPrivacyRequestCreate],
@@ -82,23 +78,6 @@ def cache_data(
     privacy_request.cache_custom_privacy_request_fields(custom_privacy_request_fields)
     privacy_request.cache_encryption(encryption_key)  # handles None already
 
-    # Store masking secrets in the cache
-    logger.info("Caching masking secrets for privacy request {}", privacy_request.id)
-    erasure_rules = policy.get_rules_for_action(action_type=ActionType.erasure)
-    unique_masking_strategies_by_name: Set[str] = set()
-    for rule in erasure_rules:
-        strategy_name: str = rule.masking_strategy["strategy"]  # type: ignore
-        configuration = rule.masking_strategy["configuration"]  # type: ignore
-        if strategy_name in unique_masking_strategies_by_name:
-            continue
-        unique_masking_strategies_by_name.add(strategy_name)
-        masking_strategy = MaskingStrategy.get_strategy(strategy_name, configuration)
-        if masking_strategy.secrets_required():
-            masking_secrets: List[MaskingSecretCache] = (
-                masking_strategy.generate_secrets_for_cache()
-            )
-            for masking_secret in masking_secrets:
-                privacy_request.cache_masking_secret(masking_secret)
     if drp_request_body:
         privacy_request.cache_drp_request_body(drp_request_body)
 

fides/api/task/conditional_dependencies/evaluator.py

@@ -0,0 +1,109 @@
+import operator as py_operator
+from typing import Any, Union
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.graph.config import FieldPath
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+    GroupOperator,
+    Operator,
+)
+
+operator_methods = {
+    Operator.exists: lambda a, _: a is not None,
+    Operator.not_exists: lambda a, _: a is None,
+    Operator.eq: py_operator.eq,
+    Operator.neq: py_operator.ne,
+    Operator.lt: lambda a, b: a < b if a is not None else False,
+    Operator.lte: lambda a, b: a <= b if a is not None else False,
+    Operator.gt: lambda a, b: a > b if a is not None else False,
+    Operator.gte: lambda a, b: a >= b if a is not None else False,
+    Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
+    Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
+}
+
+
+class ConditionEvaluator:
+    """Evaluates nested conditions for manual task creation"""
+
+    def __init__(self, db: Session):
+        self.db = db
+
+    def evaluate_rule(self, rule: Condition, data: Union[dict, Any]) -> bool:
+        """Evaluate a nested condition rule against input data"""
+        if isinstance(rule, ConditionLeaf):
+            return self._evaluate_leaf_condition(rule, data)
+        # ConditionGroup
+        return self._evaluate_group_condition(rule, data)
+
+    def _evaluate_leaf_condition(
+        self, condition: ConditionLeaf, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a leaf condition against input data"""
+        actual_value = self._get_nested_value(data, condition.field_address.split("."))
+        # Apply operator and return result
+        return self._apply_operator(actual_value, condition.operator, condition.value)
+
+    def _evaluate_group_condition(
+        self, group: ConditionGroup, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a group condition against input data"""
+        results = [
+            self.evaluate_rule(condition, data) for condition in group.conditions
+        ]
+
+        logical_operators = {GroupOperator.and_: all, GroupOperator.or_: any}
+        operator_func = logical_operators.get(group.logical_operator)
+
+        if operator_func is None:
+            logger.warning(f"Unknown logical operator: {group.logical_operator}")
+            return False
+
+        return operator_func(results)
+
+    def _get_nested_value(self, data: Union[dict, Any], keys: list[str]) -> Any:
+        """Get nested value from data using dot notation
+
+        Supports both simple dictionary access and Fides reference structures:
+        - Simple dict: data["user"]["name"]
+        - Fides FieldAddress: data.get_field_value(FieldAddress("dataset", "collection", "field_address"))
+        - Fides Collection: data.get_field_value(FieldPath("field_address", "subfield"))
+        """
+        if not keys:
+            return data
+
+        current = data
+
+        # Try Fides reference structures first
+        if hasattr(current, "get_field_value"):
+            try:
+                field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
+                return current.get_field_value(field_path)
+            except (AttributeError, ValueError):
+                pass
+
+        # Fall back to dictionary access
+        for key in keys:
+            if not isinstance(current, dict):
+                current = current.get(key, {}) if hasattr(current, "get") else None
+            else:
+                current = current.get(key, {})
+
+        return current if current != {} else None
+
+    def _apply_operator(
+        self, actual_value: Any, operator: Operator, expected_value: Any
+    ) -> bool:
+        """Apply operator to actual and expected values"""
+
+        # Get the method for the operator and execute it
+        operator_method = operator_methods.get(operator)
+        if operator_method is None:
+            logger.warning(f"Unknown operator: {operator}")
+            return False
+
+        return operator_method(actual_value, expected_value)

fides/api/task/conditional_dependencies/schemas.py

@@ -0,0 +1,54 @@
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class Operator(str, Enum):
+    eq = "eq"
+    neq = "neq"
+    lt = "lt"
+    lte = "lte"
+    gt = "gt"
+    gte = "gte"
+    exists = "exists"
+    not_exists = "not_exists"
+    list_contains = "list_contains"
+    not_in_list = "not_in_list"
+
+
+class GroupOperator(str, Enum):
+    and_ = "and"
+    or_ = "or"
+
+
+# Leaf condition (e.g., "user.name exists", "user.created_at >= 2024-01-01")
+class ConditionLeaf(BaseModel):
+    field_address: str = Field(
+        description="Field path to check (e.g., 'user.name', 'billing.subscription.status')"
+    )
+    operator: Operator = Field(description="Operator to apply")
+    value: Optional[
+        Union[str, int, float, bool, List[Union[str, int, float, bool]]]
+    ] = Field(default=None, description="Expected value for comparison")
+
+
+# Nested logical condition (AND, OR)
+class ConditionGroup(BaseModel):
+    logical_operator: GroupOperator = Field(
+        description="Logical operator: 'and' or 'or'"
+    )
+    conditions: list["Condition"] = Field(
+        description="List of conditions or nested groups"
+    )
+
+    @model_validator(mode="after")
+    def validate_conditions(self) -> "ConditionGroup":
+        if not self.conditions:
+            raise ValueError("conditions list cannot be empty")
+        return self
+
+
+# Use model_rebuild to allow recursive nesting
+Condition = Union[ConditionLeaf, ConditionGroup]
+ConditionGroup.model_rebuild()

fides/api/task/deprecated_graph_task.py

@@ -1,4 +1,5 @@
 # pylint: disable=too-many-lines
+from functools import partial
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import dask
@@ -252,16 +253,33 @@ def run_erasure_request_deprecated(  # pylint: disable = too-many-arguments
 
         access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
 
-        dsk: Dict[CollectionAddress, Any] = {
-            k: (
-                t.erasure_request,
+        # Build the dask task graph for erasure, ensuring we also pass along the
+        # upstream access data that may be required by the connector (e.g. BigQuery
+        # delete statements).  We accomplish this by partially applying the
+        # `inputs` kwarg on each task's `erasure_request` method.  The resulting
+        # callable accepts the original positional arguments expected by Dask.
+
+        dsk: Dict[CollectionAddress, Any] = {}
+        for k, t in env.items():
+            # Collect upstream access data in the same order as the input keys
+            upstream_access_data: List[List[Row]] = [
+                access_request_data.get(str(input_key), [])
+                for input_key in t.execution_node.input_keys
+            ]
+
+            # Bind the `inputs` keyword argument so it is supplied when the task executes
+            erasure_fn_with_inputs = partial(
+                t.erasure_request, inputs=upstream_access_data
+            )
+
+            # Build the task tuple: (callable, retrieved_data, *prereqs)
+            dsk[k] = (
+                erasure_fn_with_inputs,
                 access_request_data.get(
                     str(k), []
-                ),  # Pass in the results of the access request for this collection
+                ),  # Data retrieved for this collection
                 *_evaluate_erasure_dependencies(t, erasure_end_nodes),
             )
-            for k, t in env.items()
-        }
 
         # root node returns 0 to be consistent with the output of the other erasure tasks
         dsk[ROOT_COLLECTION_ADDRESS] = 0

fides/api/task/execute_request_tasks.py

@@ -298,17 +298,11 @@ def run_access_node(
                         )
                         # Currently, upstream tasks and "input keys" (which are built by data dependencies)
                         # are the same, but they may not be the same in the future.
-                        ordered_upstream_tasks: List[Optional[RequestTask]] = (
-                            _order_tasks_by_input_key(
+                        upstream_access_data: List[List[Row]] = (
+                            _build_upstream_access_data(
                                 graph_task.execution_node.input_keys, upstream_results
                             )
                         )
-                        # Pass in access data dependencies in the same order as the input keys.
-                        # If we don't have access data for an upstream node, pass in an empty list
-                        upstream_access_data: List[List[Row]] = [
-                            upstream.get_access_data() if upstream else []
-                            for upstream in ordered_upstream_tasks
-                        ]
                         # Run the main access function
                         graph_task.access_request(*upstream_access_data)
 
@@ -359,7 +353,7 @@ def run_erasure_node(
                     session,
                 ) as resources:
                     # Build GraphTask resource to facilitate execution
-                    graph_task: GraphTask = create_graph_task(
+                    erasure_graph_task: GraphTask = create_graph_task(
                         session, request_task, resources
                     )
                     # Get access data that was saved in the erasure format that was collected from the
@@ -368,8 +362,24 @@ def run_erasure_node(
                         request_task.get_data_for_erasures() or []
                     )
 
-                    # Run the main erasure function!
-                    graph_task.erasure_request(retrieved_data)
+                    upstream_access_data: List[List[Row]] = []
+
+                    try:
+                        upstream_access_data = (
+                            get_upstream_access_data_for_erasure_task(
+                                request_task, session, resources
+                            )
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Unable to get upstream access data for erasure task {request_task.collection_address}: {e}"
+                        )
+
+                    # Run the main erasure function, passing along the upstream access data.
+                    # The extra data is currently only needed for generating BigQuery delete statements.
+                    erasure_graph_task.erasure_request(
+                        retrieved_data, inputs=upstream_access_data
+                    )
 
     queue_downstream_tasks_with_retries(
         self,
@@ -487,6 +497,73 @@ def _order_tasks_by_input_key(
     return tasks
 
 
+def get_upstream_access_data_for_erasure_task(
+    erasure_request_task: RequestTask,
+    session: Session,
+    resources: TaskResources,
+) -> List[List[Row]]:
+    """
+    Retrieves upstream access data for a given erasure request task.
+
+    This function finds the corresponding access task for the erasure task,
+    creates a GraphTask to extract input keys, and builds the upstream access data
+    needed for erasure operations (particularly for BigQuery delete statements).
+
+    Args:
+        erasure_request_task: The erasure task that needs upstream access data
+        session: Database session for querying
+        resources: Task resources for creating GraphTask
+
+    Returns:
+        List[List[Row]]: Upstream access data ordered by input keys
+
+    Raises:
+        Exception: If the corresponding access task cannot be found
+    """
+    # Get the corresponding access task for the current erasure task
+    access_request_task = (
+        session.query(RequestTask)
+        .filter(
+            RequestTask.privacy_request_id == erasure_request_task.privacy_request_id,
+            RequestTask.collection_address == erasure_request_task.collection_address,
+            RequestTask.action_type == ActionType.access,
+        )
+        .first()
+    )
+
+    if not access_request_task:
+        raise Exception(
+            f"Unable to find access request task for erasure task {erasure_request_task.collection_address}"
+        )
+
+    # Convert the request task to a GraphTask to get the input_keys
+    access_graph_task: GraphTask = create_graph_task(
+        session, access_request_task, resources
+    )
+
+    # Build and return the upstream access data
+    return _build_upstream_access_data(
+        access_graph_task.execution_node.input_keys,
+        access_request_task.upstream_tasks_objects(session),
+    )
+
+
+def _build_upstream_access_data(
+    input_keys: List[CollectionAddress],
+    upstream_tasks: Query,
+) -> List[List[Row]]:
+    """
+    Helper function to build the access data for the current node.
+    The access data is passed in the same order as the input keys.
+    If we don't have access data for an upstream node, return an empty list.
+    """
+
+    ordered_upstream: List[Optional[RequestTask]] = _order_tasks_by_input_key(
+        input_keys, upstream_tasks
+    )
+    return [task.get_access_data() if task else [] for task in ordered_upstream]
+
+
 mapping = {
     ActionType.access.value: run_access_node,
     ActionType.erasure.value: run_erasure_node,