ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0__py2.py3-none-any.whl

fides/api/service/privacy_request/request_runner_service.py

@@ -14,6 +14,7 @@ from fides.api.common_exceptions import (
     ClientUnsuccessfulException,
     IdentityNotFoundException,
     ManualWebhookFieldsUnset,
+    MaskingSecretsExpired,
     MessageDispatchException,
     NoCachedManualWebhookEntry,
     PrivacyRequestExit,
@@ -75,9 +76,11 @@ from fides.api.task.graph_task import (
 from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
 from fides.api.tasks import DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
+from fides.api.util.cache import get_all_masking_secret_keys
 from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
+from fides.api.util.memory_watchdog import memory_limiter
 from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_TRANSFER_TO_PARENT,
     V1_URL_PREFIX,
@@ -356,8 +359,8 @@ def upload_and_save_access_results(  # pylint: disable=R0912
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
-# TODO: Add log_context back in, this is just for some temporary testing
-# @log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
+@memory_limiter
+@log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
 def run_privacy_request(
     self: DatabaseTask,
     privacy_request_id: str,
@@ -478,12 +481,21 @@ def run_privacy_request(
                     connection_configs
                 )
 
+                # If the privacy request requires manual finalization and has not yet been finalized, we exit here
+                if (
+                    privacy_request.status
+                    == PrivacyRequestStatus.requires_manual_finalization
+                    and privacy_request.finalized_at is None
+                ):
+                    return
+
                 # Access CHECKPOINT
                 if (
                     policy.get_rules_for_action(action_type=ActionType.access)
                     or policy.get_rules_for_action(action_type=ActionType.erasure)
                 ) and can_run_checkpoint(
-                    request_checkpoint=CurrentStep.access, from_checkpoint=resume_step
+                    request_checkpoint=CurrentStep.access,
+                    from_checkpoint=resume_step,
                 ):
                     privacy_request.cache_failed_checkpoint_details(CurrentStep.access)
                     access_runner(
@@ -528,9 +540,12 @@ def run_privacy_request(
                 if policy.get_rules_for_action(
                     action_type=ActionType.erasure
                 ) and can_run_checkpoint(
-                    request_checkpoint=CurrentStep.erasure, from_checkpoint=resume_step
+                    request_checkpoint=CurrentStep.erasure,
+                    from_checkpoint=resume_step,
                 ):
                     privacy_request.cache_failed_checkpoint_details(CurrentStep.erasure)
+                    _verify_masking_secrets(policy, privacy_request, resume_step)
+
                     # We only need to run the erasure once until masking strategies are handled
                     erasure_runner(
                         privacy_request=privacy_request,
@@ -652,58 +667,111 @@ def run_privacy_request(
                 if not proceed:
                     return
 
-            legacy_request_completion_enabled = ConfigProxy(
-                session
-            ).notifications.send_request_completion_notification
-
-            action_type = (
-                MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
-                if policy.get_rules_for_action(action_type=ActionType.access)
-                else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
-            )
-
-            if message_send_enabled(
-                session,
-                privacy_request.property_id,
-                action_type,
-                legacy_request_completion_enabled,
-            ) and not policy.get_rules_for_action(action_type=ActionType.consent):
-                try:
-                    if not access_result_urls:
-                        # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
-                        # from the database because the Privacy Request would have exited
-                        # processing and lost access to the access_result_urls in memory
-                        access_result_urls = (
-                            privacy_request.access_result_urls or {}
-                        ).get("access_result_urls", [])
-                    initiate_privacy_request_completion_email(
-                        session,
-                        policy,
-                        access_result_urls,
-                        identity_data,
-                        privacy_request.property_id,
-                    )
-                except (IdentityNotFoundException, MessageDispatchException) as e:
-                    privacy_request.error_processing(db=session)
-                    # If dev mode, log traceback
-                    _log_exception(e, CONFIG.dev_mode)
-                    return
-
-            # Only mark as complete if not in error state
-            if privacy_request.status != PrivacyRequestStatus.error:
-                privacy_request.finished_processing_at = datetime.utcnow()
-                AuditLog.create(
-                    db=session,
-                    data={
-                        "user_id": "system",
-                        "privacy_request_id": privacy_request.id,
-                        "action": AuditLogAction.finished,
-                        "message": "",
-                    },
+            # Request finalization CHECKPOINT
+            if can_run_checkpoint(
+                request_checkpoint=CurrentStep.finalization,
+                from_checkpoint=resume_step,
+            ):
+                privacy_request.cache_failed_checkpoint_details(
+                    CurrentStep.finalization,
                 )
-                privacy_request.status = PrivacyRequestStatus.complete
-                logger.info("Privacy request run completed.")
-                privacy_request.save(db=session)
+                if privacy_request.status != PrivacyRequestStatus.error:
+                    erasure_rules = policy.get_rules_for_action(
+                        action_type=ActionType.erasure
+                    )
+                    if (
+                        privacy_request.finalized_at is None
+                        and erasure_rules
+                        and CONFIG.execution.erasure_request_finalization_required
+                    ):
+                        logger.info(
+                            "Marking privacy request '{}' as requires manual finalization.",
+                            privacy_request.id,
+                        )
+                        privacy_request.status = (
+                            PrivacyRequestStatus.requires_manual_finalization
+                        )
+                        privacy_request.save(db=session)
+                        return
+
+                    # Finally, mark the request as complete
+                    if privacy_request.finalized_at:
+                        logger.info(
+                            "Marking privacy request '{}' as finalized.",
+                            privacy_request.id,
+                        )
+                        privacy_request.add_success_execution_log(
+                            session,
+                            connection_key=None,
+                            dataset_name="Request finalized",
+                            collection_name=None,
+                            message=f"Request finalized for privacy request: {privacy_request.id}",
+                            action_type=privacy_request.policy.get_action_type(),  # type: ignore
+                        )
+
+                    logger.info(
+                        "Marking privacy request '{}' as complete.",
+                        privacy_request.id,
+                    )
+                    AuditLog.create(
+                        db=session,
+                        data={
+                            "user_id": "system",
+                            "privacy_request_id": privacy_request.id,
+                            "action": AuditLogAction.finished,
+                            "message": "",
+                        },
+                    )
+                    privacy_request.status = PrivacyRequestStatus.complete
+                    privacy_request.finished_processing_at = datetime.utcnow()
+                    privacy_request.save(db=session)
+
+                    # Send a final email to the user confirming request completion
+                    if privacy_request.status == PrivacyRequestStatus.complete:
+                        legacy_request_completion_enabled = ConfigProxy(
+                            session
+                        ).notifications.send_request_completion_notification
+
+                        action_type = (
+                            MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
+                            if policy.get_rules_for_action(
+                                action_type=ActionType.access
+                            )
+                            else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
+                        )
+
+                        if message_send_enabled(
+                            session,
+                            privacy_request.property_id,
+                            action_type,
+                            legacy_request_completion_enabled,
+                        ) and not policy.get_rules_for_action(
+                            action_type=ActionType.consent
+                        ):
+                            if not access_result_urls:
+                                # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
+                                # from the database because the Privacy Request would have exited
+                                # processing and lost access to the access_result_urls in memory
+                                access_result_urls = (
+                                    privacy_request.access_result_urls or {}
+                                ).get("access_result_urls", [])
+
+                            try:
+                                initiate_privacy_request_completion_email(
+                                    session,
+                                    policy,
+                                    access_result_urls,
+                                    identity_data,
+                                    privacy_request.property_id,
+                                )
+                            except (
+                                IdentityNotFoundException,
+                                MessageDispatchException,
+                            ) as e:
+                                privacy_request.error_processing(db=session)
+                                # If dev mode, log traceback
+                                _log_exception(e, CONFIG.dev_mode)
+                                return
 
 
 def initiate_privacy_request_completion_email(
@@ -1000,3 +1068,25 @@ def run_webhooks_and_report_status(
             return False
 
     return True
+
+
+def _verify_masking_secrets(
+    policy: Policy, privacy_request: PrivacyRequest, resume_step: Optional[CurrentStep]
+) -> None:
+    """
+    Checks that the required masking secrets are still cached for the given request.
+    Raises an exception if masking secrets are needed for the given policy but they don't exist.
+    """
+
+    if resume_step is None:
+        return
+
+    # if masking can be performed without any masking secrets, we skip the cache check
+    if (
+        policy.generate_masking_secrets()
+        and not get_all_masking_secret_keys(privacy_request.id)
+        and not privacy_request.masking_secrets
+    ):
+        raise MaskingSecretsExpired(
+            f"The masking secrets for privacy request ID '{privacy_request.id}' have expired. Please submit a new erasure request."
+        )

fides/api/service/privacy_request/request_service.py

@@ -8,10 +8,10 @@ from typing import Any, Dict, List, Optional, Set
 from httpx import AsyncClient
 from loguru import logger
 from sqlalchemy import text
+from sqlalchemy.orm import Session
 from sqlalchemy.sql.elements import TextClause
 
 from fides.api.common_exceptions import PrivacyRequestNotFound
-from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
     PrivacyRequest,
@@ -19,14 +19,12 @@ from fides.api.models.privacy_request import (
 )
 from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
-from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import (
     PrivacyRequestResponse,
     PrivacyRequestStatus,
 )
 from fides.api.schemas.redis_cache import Identity
-from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
 from fides.api.util.cache import (
@@ -34,6 +32,9 @@ from fides.api.util.cache import (
     celery_tasks_in_flight,
     get_async_task_tracking_cache_key,
     get_cache,
+    get_privacy_request_retry_count,
+    increment_privacy_request_retry_count,
+    reset_privacy_request_retry_count,
 )
 from fides.api.util.lock import redis_lock
 from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS, V1_URL_PREFIX
@@ -69,7 +70,6 @@ def build_required_privacy_request_kwargs(
 
 def cache_data(
     privacy_request: PrivacyRequest,
-    policy: Policy,
     identity: Identity,
     encryption_key: Optional[str],
     drp_request_body: Optional[DrpPrivacyRequestCreate],
@@ -82,23 +82,6 @@ def cache_data(
     privacy_request.cache_custom_privacy_request_fields(custom_privacy_request_fields)
     privacy_request.cache_encryption(encryption_key)  # handles None already
 
-    # Store masking secrets in the cache
-    logger.info("Caching masking secrets for privacy request {}", privacy_request.id)
-    erasure_rules = policy.get_rules_for_action(action_type=ActionType.erasure)
-    unique_masking_strategies_by_name: Set[str] = set()
-    for rule in erasure_rules:
-        strategy_name: str = rule.masking_strategy["strategy"]  # type: ignore
-        configuration = rule.masking_strategy["configuration"]  # type: ignore
-        if strategy_name in unique_masking_strategies_by_name:
-            continue
-        unique_masking_strategies_by_name.add(strategy_name)
-        masking_strategy = MaskingStrategy.get_strategy(strategy_name, configuration)
-        if masking_strategy.secrets_required():
-            masking_secrets: List[MaskingSecretCache] = (
-                masking_strategy.generate_secrets_for_cache()
-            )
-            for masking_secret in masking_secrets:
-                privacy_request.cache_masking_secret(masking_secret)
     if drp_request_body:
         privacy_request.cache_drp_request_body(drp_request_body)
 
@@ -371,10 +354,17 @@ def initiate_interrupted_task_requeue_poll() -> None:
 
 
 def get_cached_task_id(entity_id: str) -> Optional[str]:
-    """Gets the cached task ID for a privacy request or request task by ID."""
+    """Gets the cached task ID for a privacy request or request task by ID.
+
+    Raises Exception if cache operations fail, allowing callers to handle cache failures appropriately.
+    """
     cache: FidesopsRedis = get_cache()
-    task_id = cache.get(get_async_task_tracking_cache_key(entity_id))
-    return task_id
+    try:
+        task_id = cache.get(get_async_task_tracking_cache_key(entity_id))
+        return task_id
+    except Exception as exc:
+        logger.error(f"Failed to get cached task ID for entity {entity_id}: {exc}")
+        raise
 
 
 REQUEUE_INTERRUPTED_TASKS_LOCK = "requeue_interrupted_tasks_lock"
@@ -414,6 +404,115 @@ def _get_task_ids_from_dsr_queue(
     return queued_tasks_ids
 
 
+def _cancel_interrupted_tasks_and_error_privacy_request(
+    db: Session, privacy_request: PrivacyRequest, error_message: Optional[str] = None
+) -> None:
+    """
+    Cancel all tasks associated with an interrupted privacy request and set the privacy request to error state.
+
+    This function:
+    1. Logs the error message (either provided or default)
+    2. Revokes the main privacy request task and all associated request tasks
+    3. Sets the privacy request status to error
+    4. Creates an error log entry
+
+    Args:
+        db: Database session
+        privacy_request: The privacy request to cancel and error
+        error_message: Optional error message to log. If not provided, uses default message.
+    """
+    if error_message:
+        logger.error(error_message)
+    else:
+        logger.error(
+            f"Canceling interrupted tasks and marking privacy request {privacy_request.id} as error"
+        )
+
+    # Cancel all associated Celery tasks
+    privacy_request.cancel_celery_tasks()
+
+    # Set privacy request to error state using the existing method
+    try:
+        privacy_request.error_processing(db)
+        logger.info(
+            f"Privacy request {privacy_request.id} marked as error due to task interruption"
+        )
+    except Exception as exc:
+        logger.error(
+            f"Failed to mark privacy request {privacy_request.id} as error: {exc}"
+        )
+
+
+def _handle_privacy_request_requeue(
+    db: Session, privacy_request: PrivacyRequest
+) -> None:
+    """Handle retry logic for a privacy request - either requeue or cancel based on retry count."""
+    try:
+        # Check retry count and either requeue or cancel based on limit
+        current_retry_count = get_privacy_request_retry_count(privacy_request.id)
+        max_retries = CONFIG.execution.privacy_request_requeue_retry_count
+
+        if current_retry_count < max_retries:
+            # Increment retry count and attempt requeue
+            new_retry_count = increment_privacy_request_retry_count(privacy_request.id)
+            logger.info(
+                f"Requeuing privacy request {privacy_request.id} "
+                f"(attempt {new_retry_count}/{max_retries})"
+            )
+
+            from fides.service.privacy_request.privacy_request_service import (  # pylint: disable=cyclic-import
+                PrivacyRequestError,
+                _requeue_privacy_request,
+            )
+
+            try:
+                _requeue_privacy_request(db, privacy_request)
+            except PrivacyRequestError as exc:
+                # If requeue fails, cancel tasks and set to error state
+                _cancel_interrupted_tasks_and_error_privacy_request(
+                    db, privacy_request, exc.message
+                )
+        else:
+            # Exceeded retry limit, cancel tasks and set to error state
+            _cancel_interrupted_tasks_and_error_privacy_request(
+                db,
+                privacy_request,
+                f"Privacy request {privacy_request.id} exceeded max retry attempts "
+                f"({max_retries}), canceling tasks and setting to error state",
+            )
+            # Reset retry count since we're giving up
+            reset_privacy_request_retry_count(privacy_request.id)
+
+    except Exception as cache_exc:
+        # If cache operations fail (Redis down, network issues, etc.), fail safe by canceling
+        _cancel_interrupted_tasks_and_error_privacy_request(
+            db,
+            privacy_request,
+            f"Cache operation failed for privacy request {privacy_request.id}, "
+            f"failing safe by canceling tasks: {cache_exc}",
+        )
+
+
+def _get_request_task_ids_in_progress(
+    db: Session, privacy_request_id: str
+) -> List[str]:
+    """Get the IDs of request tasks that are currently in progress for a privacy request."""
+    request_tasks_in_progress = (
+        db.query(RequestTask.id)
+        .filter(RequestTask.privacy_request_id == privacy_request_id)
+        .filter(
+            RequestTask.status.in_(
+                [
+                    ExecutionLogStatus.in_processing,
+                    ExecutionLogStatus.pending,
+                ]
+            )
+        )
+        .all()
+    )
+    return [task[0] for task in request_tasks_in_progress]
+
+
 # pylint: disable=too-many-branches
 @celery_app.task(base=DatabaseTask, bind=True)
 def requeue_interrupted_tasks(self: DatabaseTask) -> None:
@@ -463,17 +562,40 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
             )
 
             # Get task IDs from the queue in a memory-efficient way
-            queued_tasks_ids = _get_task_ids_from_dsr_queue(redis_conn)
+            try:
+                queued_tasks_ids = _get_task_ids_from_dsr_queue(redis_conn)
+            except Exception as queue_exc:
+                logger.warning(
+                    f"Failed to get task IDs from queue, skipping queue state checks: {queue_exc}"
+                )
+                return
 
             # Check each privacy request
             for privacy_request in in_progress_requests:
                 should_requeue = False
                 logger.debug(f"Checking tasks for privacy request {privacy_request.id}")
 
-                task_id = get_cached_task_id(privacy_request.id)
+                try:
+                    task_id = get_cached_task_id(privacy_request.id)
+                except Exception as cache_exc:
+                    # If we can't get the task ID due to cache failure, fail safe by canceling
+                    _cancel_interrupted_tasks_and_error_privacy_request(
+                        db,
+                        privacy_request,
+                        f"Cache failure when getting task ID for privacy request {privacy_request.id}, "
+                        f"failing safe by canceling tasks: {cache_exc}",
+                    )
+                    continue
 
                 # If the task ID is not cached, we can't check if it's running
+                # This means the request is stuck - cancel it
                 if not task_id:
+                    _cancel_interrupted_tasks_and_error_privacy_request(
+                        db,
+                        privacy_request,
+                        f"No task ID found for privacy request {privacy_request.id}, "
+                        f"request is stuck without a running task - canceling",
+                    )
                     continue
 
                 # Check if the main privacy request task is active
@@ -491,30 +613,36 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                         )
                         should_requeue = True
 
-                    request_tasks_in_progress = (
-                        db.query(RequestTask.id)
-                        .filter(RequestTask.privacy_request_id == privacy_request.id)
-                        .filter(
-                            RequestTask.status.in_(
-                                [
-                                    ExecutionLogStatus.in_processing,
-                                    ExecutionLogStatus.pending,
-                                ]
-                            )
-                        )
-                        .all()
+                    request_task_ids_in_progress = _get_request_task_ids_in_progress(
+                        db, privacy_request.id
                     )
-                    request_task_ids_in_progress = [
-                        task[0] for task in request_tasks_in_progress
-                    ]
 
                     # Check each individual request task
                     for request_task_id in request_task_ids_in_progress:
-                        subtask_id = get_cached_task_id(request_task_id)
+                        try:
+                            subtask_id = get_cached_task_id(request_task_id)
+                        except Exception as cache_exc:
+                            # If we can't get the subtask ID due to cache failure, fail safe by canceling
+                            _cancel_interrupted_tasks_and_error_privacy_request(
+                                db,
+                                privacy_request,
+                                f"Cache failure when getting subtask ID for request task {request_task_id} "
+                                f"(privacy request {privacy_request.id}), failing safe by canceling tasks: {cache_exc}",
+                            )
+                            should_requeue = False
+                            break
 
                         # If the task ID is not cached, we can't check if it's running
+                        # This means the subtask is stuck - cancel the entire privacy request
                         if not subtask_id:
-                            continue
+                            _cancel_interrupted_tasks_and_error_privacy_request(
+                                db,
+                                privacy_request,
+                                f"No task ID found for request task {request_task_id} "
+                                f"(privacy request {privacy_request.id}), subtask is stuck - canceling privacy request",
+                            )
+                            should_requeue = False
+                            break
 
                         if (
                             subtask_id not in queued_tasks_ids
@@ -528,12 +656,4 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
 
                 # Requeue the privacy request if needed
                 if should_requeue:
-                    from fides.service.privacy_request.privacy_request_service import (  # pylint: disable=cyclic-import
-                        PrivacyRequestError,
-                        _requeue_privacy_request,
-                    )
-
-                    try:
-                        _requeue_privacy_request(db, privacy_request)
-                    except PrivacyRequestError as exc:
-                        logger.error(exc.message)
+                    _handle_privacy_request_requeue(db, privacy_request)

fides/api/task/conditional_dependencies/evaluator.py

@@ -0,0 +1,109 @@
+import operator as py_operator
+from typing import Any, Union
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.graph.config import FieldPath
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+    GroupOperator,
+    Operator,
+)
+
+operator_methods = {
+    Operator.exists: lambda a, _: a is not None,
+    Operator.not_exists: lambda a, _: a is None,
+    Operator.eq: py_operator.eq,
+    Operator.neq: py_operator.ne,
+    Operator.lt: lambda a, b: a < b if a is not None else False,
+    Operator.lte: lambda a, b: a <= b if a is not None else False,
+    Operator.gt: lambda a, b: a > b if a is not None else False,
+    Operator.gte: lambda a, b: a >= b if a is not None else False,
+    Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
+    Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
+}
+
+
+class ConditionEvaluator:
+    """Evaluates nested conditions for manual task creation"""
+
+    def __init__(self, db: Session):
+        self.db = db
+
+    def evaluate_rule(self, rule: Condition, data: Union[dict, Any]) -> bool:
+        """Evaluate a nested condition rule against input data"""
+        if isinstance(rule, ConditionLeaf):
+            return self._evaluate_leaf_condition(rule, data)
+        # ConditionGroup
+        return self._evaluate_group_condition(rule, data)
+
+    def _evaluate_leaf_condition(
+        self, condition: ConditionLeaf, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a leaf condition against input data"""
+        actual_value = self._get_nested_value(data, condition.field_address.split("."))
+        # Apply operator and return result
+        return self._apply_operator(actual_value, condition.operator, condition.value)
+
+    def _evaluate_group_condition(
+        self, group: ConditionGroup, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a group condition against input data"""
+        results = [
+            self.evaluate_rule(condition, data) for condition in group.conditions
+        ]
+
+        logical_operators = {GroupOperator.and_: all, GroupOperator.or_: any}
+        operator_func = logical_operators.get(group.logical_operator)
+
+        if operator_func is None:
+            logger.warning(f"Unknown logical operator: {group.logical_operator}")
+            return False
+
+        return operator_func(results)
+
+    def _get_nested_value(self, data: Union[dict, Any], keys: list[str]) -> Any:
+        """Get nested value from data using dot notation
+
+        Supports both simple dictionary access and Fides reference structures:
+        - Simple dict: data["user"]["name"]
+        - Fides FieldAddress: data.get_field_value(FieldAddress("dataset", "collection", "field_address"))
+        - Fides Collection: data.get_field_value(FieldPath("field_address", "subfield"))
+        """
+        if not keys:
+            return data
+
+        current = data
+
+        # Try Fides reference structures first
+        if hasattr(current, "get_field_value"):
+            try:
+                field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
+                return current.get_field_value(field_path)
+            except (AttributeError, ValueError):
+                pass
+
+        # Fall back to dictionary access
+        for key in keys:
+            if not isinstance(current, dict):
+                current = current.get(key, {}) if hasattr(current, "get") else None
+            else:
+                current = current.get(key, {})
+
+        return current if current != {} else None
+
+    def _apply_operator(
+        self, actual_value: Any, operator: Operator, expected_value: Any
+    ) -> bool:
+        """Apply operator to actual and expected values"""
+
+        # Get the method for the operator and execute it
+        operator_method = operator_methods.get(operator)
+        if operator_method is None:
+            logger.warning(f"Unknown operator: {operator}")
+            return False
+
+        return operator_method(actual_value, expected_value)