ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/task/conditional_dependencies/util.py

@@ -0,0 +1,111 @@
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+# pylint: disable=too-many-return-statements
+def transform_value_for_evaluation(value: Any) -> Any:
+    """
+    Transforms a value to its evaluation-ready form.
+
+    Handles various data types defensively to ensure compatibility with
+    the operators defined in task.conditional_dependencies.operators.py
+
+    Args:
+        value: The value to transform
+
+    Returns:
+        The transformed value compatible with evaluation operators
+    """
+    if value is None:
+        return None
+
+    # Handle Pydantic models first (before checking for other attributes)
+    if isinstance(value, BaseModel):
+        return value.model_dump(mode="json")
+
+    # Handle enums (must come before other checks as enums might have other attributes)
+    if hasattr(value, "value") and not isinstance(value, BaseModel):
+        return value.value
+
+    # Handle date/time types
+    if isinstance(value, datetime):
+        return value.isoformat()
+
+    # Handle UUIDs
+    if isinstance(value, UUID):
+        return str(value)
+
+    # Handle bytes types - convert to strings for string operators
+    if isinstance(value, (bytes, bytearray)):
+        try:
+            return value.decode("utf-8")
+        except UnicodeDecodeError:
+            # If decode fails, use repr for a string representation
+            return repr(value)
+
+    # Handle complex numbers - convert to string as they can't be compared with < > etc
+    if isinstance(value, complex):
+        return str(value)
+
+    # Handle set/frozenset/tuple - convert to lists for list operators
+    # (operators.py list operations check isinstance(x, list))
+    if isinstance(value, (set, frozenset, tuple)):
+        return list(value)
+
+    # Return value as-is if no transformation needed
+    return value
+
+
+def extract_nested_field_value(data: Any, field_path: list[str]) -> Any:
+    """
+    Extracts a value from a dictionary by following the path.
+
+    Args:
+        data: The dictionary to extract from
+        field_path: The path to extract the value from
+
+    Returns:
+        The extracted value
+    """
+    current: Any = data
+    for part in field_path:
+        if current is None:
+            return None
+        if hasattr(current, part):
+            current = getattr(current, part)
+        elif isinstance(current, list) and part.isdigit():
+            if 0 <= int(part) < len(current):
+                current = current[int(part)]
+            else:
+                return None
+        elif isinstance(current, dict) and part in current:
+            current = current[part]
+        else:
+            # If we can't find the part, return None
+            return None
+    return current
+
+
+def set_nested_value(path: list[str], value: Any) -> dict[str, Any]:
+    """
+    Returns a dictionary with a nested value set at the path.
+
+    Args:
+        path: List of keys to traverse (e.g., ["policy", "key"])
+        value: The value to set at the end of the path
+    """
+    result: dict[str, Any] = {}
+    final_key = path[-1]
+    current = result
+
+    for key in path[:-1]:
+        if key not in current:
+            current[key] = {}
+        current = current[key]
+
+    # Set the final value
+    current[final_key] = value
+    return result

fides/api/task/execute_request_tasks.py

@@ -3,7 +3,7 @@ from typing import Callable, List, Optional, Tuple
 from celery.app.task import Task
 from loguru import logger
 from sqlalchemy.exc import OperationalError
-from sqlalchemy.orm import Query, Session
+from sqlalchemy.orm import Query, Session, selectinload
 from tenacity import (
     RetryCallState,
     retry,
@@ -21,8 +21,12 @@ from fides.api.common_exceptions import (
 )
 from fides.api.graph.config import TERMINATOR_ADDRESS, CollectionAddress
 from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest, RequestTask
-from fides.api.models.worker_task import ExecutionLogStatus
+from fides.api.models.worker_task import (
+    RESUMABLE_EXECUTION_LOG_STATUSES,
+    ExecutionLogStatus,
+)
 from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.task.graph_task import (
@@ -45,11 +49,23 @@ def get_privacy_request_and_task(
     session: Session, privacy_request_id: str, privacy_request_task_id: str
 ) -> Tuple[PrivacyRequest, RequestTask]:
     """
-    Retrieves and validates a privacy request and its associated task
+    Retrieves and validates a privacy request and its associated task.
+    Eager loads policy.rules to avoid N+1 queries during manual task execution.
     """
 
-    privacy_request: Optional[PrivacyRequest] = PrivacyRequest.get(
-        db=session, object_id=privacy_request_id
+    # Eager load relationships to avoid N+1 queries during manual task execution:
+    # - policy.rules: accessed by get_policy_convenience_fields()
+    # - provided_identities: accessed by get_persisted_identity()
+    # - custom_fields: accessed by get_persisted_custom_privacy_request_fields()
+    privacy_request: Optional[PrivacyRequest] = (
+        session.query(PrivacyRequest)
+        .options(
+            selectinload(PrivacyRequest.policy).selectinload(Policy.rules),  # type: ignore[attr-defined]
+            selectinload(PrivacyRequest.provided_identities),  # type: ignore[attr-defined]
+            selectinload(PrivacyRequest.custom_fields),  # type: ignore[attr-defined]
+        )
+        .filter(PrivacyRequest.id == privacy_request_id)
+        .first()
     )
     request_task: Optional[RequestTask] = RequestTask.get(
         db=session, object_id=privacy_request_task_id
@@ -162,7 +178,7 @@ def can_run_task_body(
     if request_task.is_root_task:
         # Shouldn't be possible but adding as a catch-all
         return False
-    if request_task.status != ExecutionLogStatus.pending:
+    if request_task.status not in RESUMABLE_EXECUTION_LOG_STATUSES:
         logger_method(request_task)(
             "Skipping {} task {} with status {}.",
             request_task.action_type,
@@ -274,15 +290,18 @@ def run_access_node(
 
     try:
         with self.get_new_session() as session:
-            privacy_request, request_task, upstream_results = (
-                run_prerequisite_task_checks(
-                    session, privacy_request_id, privacy_request_task_id
-                )
+            (
+                privacy_request,
+                request_task,
+                upstream_results,
+            ) = run_prerequisite_task_checks(
+                session, privacy_request_id, privacy_request_task_id
             )
             with logger.contextualize(
                 privacy_request_source=(
                     privacy_request.source.value if privacy_request.source else None
-                )
+                ),
+                collection=request_task.collection_address,
             ):
                 log_task_starting(request_task)
 
@@ -343,7 +362,8 @@ def run_erasure_node(
         with logger.contextualize(
             privacy_request_source=(
                 privacy_request.source.value if privacy_request.source else None
-            )
+            ),
+            collection=request_task.collection_address,
         ):
             log_task_starting(request_task)
 
@@ -416,7 +436,8 @@ def run_consent_node(
         with logger.contextualize(
             privacy_request_source=(
                 privacy_request.source.value if privacy_request.source else None
-            )
+            ),
+            collection=request_task.collection_address,
         ):
             log_task_starting(request_task)
 

fides/api/task/filter_results.py

@@ -6,10 +6,43 @@ from loguru import logger
 
 from fides.api.graph.config import CollectionAddress, FieldPath
 from fides.api.graph.graph import DatasetGraph
+from fides.api.service.storage.util import is_attachment_field
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.util.collection_util import Row
 
 
+def should_skip_data_category_filtering(
+    node_address: str, results: List[Dict[str, Optional[Any]]]
+) -> bool:
+    """
+    Determine if results should skip data category filtering.
+
+    This includes:
+    1. Manual task data - controlled by field definitions, not data categories
+    2. Attachment data - should be preserved regardless of data categories (from polling or manual tasks)
+
+    :param node_address: The collection address being processed
+    :param results: The results data to check
+    :return: True if filtering should be skipped, False otherwise
+    """
+    # Check if this is a manual task address
+    if ManualTaskAddress.is_manual_task_address(node_address):
+        return True
+
+    # Check if results contain attachment data
+    if not results:
+        return False
+
+    # Verify it looks like actual attachment data
+    for result in results:
+        if isinstance(result, dict) and any(
+            is_attachment_field(value) for value in result.values()
+        ):
+            return True
+
+    return False
+
+
 def filter_data_categories(
     access_request_results: Dict[str, List[Dict[str, Optional[Any]]]],
     target_categories: Set[str],
@@ -38,8 +71,8 @@ def filter_data_categories(
         if not results:
             continue
 
-        # Skip manual task data - it doesn't need filtering since it's controlled by field definitions
-        if ManualTaskAddress.is_manual_task_address(node_address):
+        # Skip data that doesn't need filtering (manual tasks and attachment data)
+        if should_skip_data_category_filtering(node_address, results):
             filtered_access_results[node_address].extend(results)
             continue
 

fides/api/task/graph_runners.py

@@ -1,6 +1,5 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import PrivacyRequestExit
@@ -19,37 +18,8 @@ from fides.api.task.deprecated_graph_task import (
     run_consent_request_deprecated,
     run_erasure_request_deprecated,
 )
+from fides.api.task.scheduler_utils import use_dsr_3_0_scheduler
 from fides.api.util.collection_util import Row
-from fides.config import CONFIG
-
-
-def use_dsr_3_0_scheduler(
-    privacy_request: PrivacyRequest, action_type: ActionType
-) -> bool:
-    """Return whether we should use the DSR 3.0 scheduler.
-
-    Override if we have a partially processed Privacy Request that was already run on
-    DSR 2.0 so we can finish processing it on 2.0.
-
-    """
-    use_dsr_3_0 = CONFIG.execution.use_dsr_3_0
-
-    prev_results: Dict[str, Optional[List[Row]]] = (
-        privacy_request.get_raw_access_results()
-    )
-    existing_tasks_count: int = privacy_request.get_tasks_by_action(action_type).count()
-
-    if prev_results and use_dsr_3_0 and not existing_tasks_count:
-        # If we've previously tried to process this Privacy Request using DSR 2.0, continue doing so
-        # for access and erasure requests
-        logger.info(
-            "Overriding scheduler to run privacy request {} using DSR 2.0 as it's "
-            "already partially processed",
-            privacy_request.id,
-        )
-        use_dsr_3_0 = False
-
-    return use_dsr_3_0
 
 
 def access_runner(

fides/api/task/graph_task.py

@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
 from fides.api.api.deps import get_autoclose_db_session as get_db
 from fides.api.common_exceptions import (
     ActionDisabled,
+    AwaitingAsyncProcessing,
     AwaitingAsyncTask,
     CollectionDisabled,
     NotSupportedForCollection,
@@ -47,6 +48,7 @@ from fides.api.service.execution_context import collect_execution_log_messages
 from fides.api.task.consolidate_query_matches import consolidate_query_matches
 from fides.api.task.filter_element_match import filter_element_match
 from fides.api.task.refine_target_path import FieldPathNodeInput
+from fides.api.task.scheduler_utils import use_dsr_3_0_scheduler
 from fides.api.task.task_resources import TaskResources
 from fides.api.util.cache import get_cache
 from fides.api.util.collection_util import (
@@ -60,7 +62,6 @@ from fides.api.util.collection_util import (
 from fides.api.util.consent_util import (
     add_errored_system_status_for_consent_reporting_on_preferences,
 )
-from fides.api.util.logger import Pii
 from fides.api.util.logger_context_utils import LoggerContextKeys
 from fides.api.util.memory_watchdog import MemoryLimitExceeded
 from fides.api.util.saas_util import FIDESOPS_GROUPED_INPUTS
@@ -95,6 +96,7 @@ def retry(
     If we exceed the number of TASK_RETRY_COUNT retries, we re-raise the exception to stop execution of the privacy request.
     """
 
+    # pylint: disable=too-many-return-statements
     def decorator(func: Callable) -> Callable:
         @wraps(func)
         def result(*args: Any, **kwargs: Any) -> Any:
@@ -114,8 +116,18 @@ def retry(
                         self.log_start(action_type)
                     # Run access or erasure request
                     return func(*args, **kwargs)
+                except AwaitingAsyncProcessing as ex:
+                    logger.warning(
+                        "Request Task {} {} {} async processing in progress",
+                        self.request_task.id if self.request_task.id else None,
+                        method_name,
+                        self.execution_node.address,
+                    )
+                    # Log the async processing status and exit without retrying.
+                    self.log_async_processing(action_type, ex)
+                    # Request Task put in "polling" status and exited, external system is processing
+                    return None
                 except AwaitingAsyncTask as ex:
-                    traceback.print_exc()
                     logger.warning(
                         "Request Task {} {} {} awaiting async continuation",
                         self.request_task.id if self.request_task.id else None,
@@ -416,9 +428,13 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             # created to keep these in sync.
             # TODO remove conditional above alongside deprecating DSR 2.0
             if self.request_task.id:
-                # Merge the request task to ensure we keep the changes made
-                # to self.request_task that haven't yet been committed to the database
+                # Merge the request_task into the current session to make it persistent,
+                # then refresh its `async_type` to load the latest state from the
+                # database. This is crucial for async tasks where `async_type` might be
+                # updated by another process, and avoids overwriting local data like
+                # `access_data`.
                 request_task = db.merge(self.request_task)
+                db.refresh(request_task, attribute_names=["async_type"])
                 request_task.update_status(db, status)
                 self.request_task = request_task
 
@@ -453,6 +469,21 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             message, [], action_type, ExecutionLogStatus.awaiting_processing
         )
 
+    def log_async_processing(
+        self,
+        action_type: ActionType,
+        ex: Optional[BaseException],
+        extra_message: Optional[str] = None,
+    ) -> None:
+        """On async processing activities - external system is actively processing, Fides is polling"""
+        logger.info("Polling for async results on node {}", self.key)
+
+        message = str(ex)
+        if extra_message:
+            message = f"{message}. {extra_message}"
+
+        self.update_status(message, [], action_type, ExecutionLogStatus.polling)
+
     def log_skipped(self, action_type: ActionType, ex: str) -> None:
         """Log that a collection was skipped.  For now, this is because a collection has been disabled."""
         logger.info("Skipping node {}", self.key)
@@ -469,11 +500,11 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
     ) -> None:
         """On completion activities"""
         if ex:
-            logger.warning(
-                "Ending {}, {} with failure {}",
+            logger.error(
+                "Ending {}, {} with failure: {}",
                 self.resources.request.id,
                 self.key,
-                Pii(ex),
+                ex,
             )
             self.update_status(str(ex), [], action_type, ExecutionLogStatus.error)
             # For DSR 3.0, Hooking into the GraphTask.log_end method to also mark the current
@@ -589,7 +620,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         # TODO Remove when we stop support for DSR 2.0
         # Save data to build masking requests for DSR 2.0 in Redis.
         # Results saved with matching array elements preserved
-        if not CONFIG.execution.use_dsr_3_0:
+        if not use_dsr_3_0_scheduler(self.resources.request, ActionType.access):
             self.resources.cache_results_with_placeholders(
                 f"access_request__{self.key}", placeholder_output
             )
@@ -619,7 +650,8 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
 
         # TODO Remove when we stop support for DSR 2.0
         # Saves intermediate access results for DSR 2.0 in Redis
-        if not CONFIG.execution.use_dsr_3_0:
+        # Only cache for existing DSR 2.0 requests
+        if not use_dsr_3_0_scheduler(self.resources.request, ActionType.access):
             self.resources.cache_object(f"access_request__{self.key}", output)
 
         # Return filtered rows with non-matched array data removed.
@@ -692,7 +724,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                     self.execution_node,
                     self.resources.policy,
                     self.resources.request,
-                    self.resources.privacy_request_task,
+                    self.request_task,
                     formatted_input_data,
                 )
 
@@ -853,7 +885,6 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         privacy_request_id = self.resources.request.id
 
         with get_db() as db:
-
             privacy_preferences = db.query(PrivacyPreferenceHistory).filter(
                 PrivacyPreferenceHistory.privacy_request_id == privacy_request_id
             )
@@ -974,7 +1005,6 @@ def build_affected_field_logs(
     policy_id = policy.id
 
     with get_db() as db:
-
         rules = db.query(Rule).filter(Rule.policy_id == policy_id)
 
         targeted_field_paths: Dict[FieldAddress, str] = {}

fides/api/task/manual/manual_task_conditional_evaluation.py

@@ -8,13 +8,21 @@ from fides.api.models.manual_task import ManualTask
 from fides.api.models.manual_task.conditional_dependency import (
     ManualTaskConditionalDependency,
 )
+from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.task.conditional_dependencies.evaluator import ConditionEvaluator
+from fides.api.task.conditional_dependencies.privacy_request.privacy_request_data import (
+    PrivacyRequestDataTransformer,
+)
 from fides.api.task.conditional_dependencies.schemas import EvaluationResult
+from fides.api.task.conditional_dependencies.util import extract_nested_field_value
 from fides.api.util.collection_util import Row
 
 
 def extract_conditional_dependency_data_from_inputs(
-    *inputs: list[Row], manual_task: ManualTask, input_keys: list[CollectionAddress]
+    *inputs: list[Row],
+    manual_task: ManualTask,
+    input_keys: list[CollectionAddress],
+    privacy_request: PrivacyRequest,
 ) -> dict[str, Any]:
     """
     Extract data for conditional dependency field addresses from input data.
@@ -33,14 +41,35 @@ def extract_conditional_dependency_data_from_inputs(
 
     conditional_data: dict[str, Any] = {}
 
-    # Get all conditional dependencies field addresses
+    # Get all task input conditional dependencies field addresses
     field_addresses = [
         dependency.field_address
         for dependency in manual_task.conditional_dependencies
         if dependency.field_address
+        and not dependency.field_address.startswith("privacy_request.")
     ]
 
-    # if no field addresses, return empty conditional data
+    # Get all privacy request conditional dependencies field addresses
+    privacy_request_field_addresses = {
+        dependency.field_address
+        for dependency in manual_task.conditional_dependencies
+        if dependency.field_address
+        and dependency.field_address.startswith("privacy_request.")
+    }
+    # If there are any privacy request conditional dependencies field addresses,
+    # transform the privacy request data into a dictionary structure for evaluation
+    if privacy_request_field_addresses:
+        privacy_request_data_transformer = PrivacyRequestDataTransformer(
+            privacy_request
+        )
+        conditional_privacy_request_data = (
+            privacy_request_data_transformer.to_evaluation_data(
+                privacy_request_field_addresses
+            )
+        )
+        conditional_data = {**conditional_privacy_request_data}
+
+    # if no field addresses, return conditional data which may contain privacy request data or be empty
     # This will allow the manual task to be executed if there are no conditional dependencies
     if not field_addresses:
         return conditional_data
@@ -103,30 +132,6 @@ def parse_field_address(field_address: str) -> tuple[str, list[str]]:
     return source_collection_key, field_path
 
 
-def extract_nested_field_value(data: Any, field_path: list[str]) -> Any:
-    """
-    Extract a nested field value by traversing the field path.
-
-    Args:
-        data: The data to extract from (usually a dict)
-        field_path: List of field names to traverse (e.g., ["profile", "preferences", "theme"])
-
-    Returns:
-        The value at the end of the field path, or None if not found
-    """
-    if not field_path:
-        return data
-
-    current = data
-    for field_name in field_path:
-        if isinstance(current, dict) and field_name in current:
-            current = current[field_name]
-        else:
-            return None
-
-    return current
-
-
 def set_nested_value(field_address: str, value: Any) -> dict[str, Any]:
     """
     Set a field value in the conditional data structure.
@@ -181,7 +186,7 @@ def evaluate_conditional_dependencies(
     """
     # Get the root condition for this manual task
     root_condition = ManualTaskConditionalDependency.get_root_condition(
-        db, manual_task.id
+        db, manual_task_id=manual_task.id
     )
 
     if not root_condition:

fides/api/task/manual/manual_task_graph_task.py

@@ -158,7 +158,10 @@ class ManualTaskGraphTask(GraphTask):
 
         # Extract conditional dependency data from inputs
         conditional_data = extract_conditional_dependency_data_from_inputs(
-            *inputs, manual_task=manual_task, input_keys=self.execution_node.input_keys
+            *inputs,
+            manual_task=manual_task,
+            input_keys=self.execution_node.input_keys,
+            privacy_request=self.resources.request,
         )
         # Evaluate conditional dependencies
         evaluation_result = evaluate_conditional_dependencies(