ethyca-fides 2.67.0rc1__py2.py3-none-any.whl → 2.67.1b0__py2.py3-none-any.whl

fides/api/task/graph_task.py

@@ -18,6 +18,7 @@ from fides.api.common_exceptions import (
     NotSupportedForCollection,
     PrivacyRequestErasureEmailSendRequired,
     SkippingConsentPropagation,
+    TableNotFound,
 )
 from fides.api.graph.config import (
     ROOT_COLLECTION_ADDRESS,
@@ -61,6 +62,7 @@ from fides.api.util.consent_util import (
 )
 from fides.api.util.logger import Pii
 from fides.api.util.logger_context_utils import LoggerContextKeys
+from fides.api.util.memory_watchdog import MemoryLimitExceeded
 from fides.api.util.saas_util import FIDESOPS_GROUPED_INPUTS
 from fides.config import CONFIG
 
@@ -70,6 +72,16 @@ EMPTY_REQUEST = PrivacyRequest()
 EMPTY_REQUEST_TASK = RequestTask()
 
 
+def _is_memory_limit_exceeded(exception: BaseException) -> bool:
+    """Check if the exception or any exception in its chain is a MemoryLimitExceeded."""
+    current_exception: Optional[BaseException] = exception
+    while current_exception:
+        if isinstance(current_exception, MemoryLimitExceeded):
+            return True
+        current_exception = current_exception.__cause__ or current_exception.__context__
+    return False
+
+
 def retry(
     action_type: ActionType,
     default_return: Any,
@@ -126,6 +138,7 @@ def retry(
                     CollectionDisabled,
                     ActionDisabled,
                     NotSupportedForCollection,
+                    TableNotFound,
                 ) as exc:
                     logger.warning(
                         "{} - Skipping collection {} for privacy_request: {}",
@@ -144,7 +157,31 @@ def retry(
                     self.log_skipped(action_type, exc)
                     self.cache_system_status_for_preferences()
                     return default_return
+                except MemoryLimitExceeded as ex:
+                    # Hard failure – mark task & downstream as errored and abort.
+                    logger.error(
+                        "Memory watchdog exceeded ({}%). Aborting {} {} without retry.",
+                        ex.memory_percent,
+                        method_name,
+                        self.execution_node.address,
+                    )
+                    # Persist error status and create execution logs before raising
+                    self.log_end(action_type, ex)
+                    self.add_error_status_for_consent_reporting()
+                    raise
                 except BaseException as ex:  # pylint: disable=W0703
+                    # Check if this exception was caused by memory limit exceeded
+                    if _is_memory_limit_exceeded(ex):
+                        logger.error(
+                            "Memory watchdog exceeded (wrapped exception). Aborting {} {} without retry.",
+                            method_name,
+                            self.execution_node.address,
+                        )
+                        # Persist error status and create execution logs before raising
+                        self.log_end(action_type, ex)
+                        self.add_error_status_for_consent_reporting()
+                        raise
+
                     traceback.print_exc()
                     func_delay *= CONFIG.execution.task_retry_backoff
                     logger.warning(
@@ -553,12 +590,21 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         # For access request results, mutate rows in-place to remove non-matching
         # array elements.  We already iterated over `output` above, so reuse the same
         # loop structure to keep cache locality.
+        logger.info(
+            "Filtering {} rows in {} for matching array elements.",
+            len(output),
+            self.execution_node.address,
+        )
         for row in output:
+            filter_element_match(row, post_processed_node_input_data)
+
+        if len(output) > 0:
             logger.info(
-                "Filtering row in {} for matching array elements.",
+                "Filtering completed for {} rows in {}. Post-processed node size: {}",
+                len(output),
                 self.execution_node.address,
+                len(post_processed_node_input_data),
             )
-            filter_element_match(row, post_processed_node_input_data)
 
         if self.request_task.id:
             # Saves intermediate access results for DSR 3.0 directly on the Request Task

fides/api/util/cache.py

@@ -334,6 +334,62 @@ def cache_task_tracking_key(request_id: str, celery_task_id: str) -> None:
         )
 
 
+def get_privacy_request_retry_cache_key(privacy_request_id: str) -> str:
+    """Get cache key for tracking privacy request requeue retry attempts."""
+    return f"id-{privacy_request_id}-privacy-request-retry-count"
+
+
+def get_privacy_request_retry_count(privacy_request_id: str) -> int:
+    """Get the current retry count for a privacy request requeue attempts.
+
+    Raises Exception if cache operations fail, allowing callers to handle cache failures appropriately.
+    """
+    cache: FidesopsRedis = get_cache()
+    try:
+        retry_count = cache.get(get_privacy_request_retry_cache_key(privacy_request_id))
+        return int(retry_count) if retry_count else 0
+    except Exception as exc:
+        logger.error(
+            f"Failed to get retry count for privacy request {privacy_request_id}: {exc}"
+        )
+        raise
+
+
+def increment_privacy_request_retry_count(privacy_request_id: str) -> int:
+    """Increment and return the retry count for a privacy request requeue attempts.
+
+    Raises Exception if cache operations fail, allowing callers to handle cache failures appropriately.
+    """
+    cache: FidesopsRedis = get_cache()
+    cache_key = get_privacy_request_retry_cache_key(privacy_request_id)
+
+    try:
+        # Increment the counter, will be 1 if key doesn't exist
+        new_count = cache.incr(cache_key)
+        # Set expiry to prevent cache buildup (24 hours)
+        cache.expire(cache_key, 86400)
+        return new_count
+    except Exception as exc:
+        logger.error(
+            f"Failed to increment retry count for privacy request {privacy_request_id}: {exc}"
+        )
+        raise
+
+
+def reset_privacy_request_retry_count(privacy_request_id: str) -> None:
+    """Reset the retry count for a privacy request requeue attempts.
+
+    Silently fails if cache operations fail since this is cleanup.
+    """
+    cache: FidesopsRedis = get_cache()
+    try:
+        cache.delete(get_privacy_request_retry_cache_key(privacy_request_id))
+    except Exception as exc:
+        logger.warning(
+            f"Failed to reset retry count for privacy request {privacy_request_id}: {exc}"
+        )
+
+
 def celery_tasks_in_flight(celery_task_ids: List[str]) -> bool:
     """Returns True if supplied Celery Tasks appear to be in-flight"""
     if not celery_task_ids:

fides/api/util/memory_watchdog.py

@@ -0,0 +1,286 @@
+"""
+Memory watchdog utilities used to proactively interrupt long-running Celery tasks that approach the
+container's memory limits.
+
+The watchdog runs in a background thread and periodically samples the overall system memory usage
+(using `psutil`). When usage exceeds the configured threshold, it immediately sends `SIGUSR1` to
+the current process. The signal handler raises `MemoryLimitExceeded` which can be caught by the
+calling code to perform clean-up and let Celery record the task as failed.
+
+This allows tasks to terminate gracefully with proper error logging, rather than being forcefully
+killed by the system's OOM killer which would result in a WorkerLostError and incomplete cleanup.
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import threading
+import time
+from functools import wraps
+from types import FrameType, TracebackType
+from typing import Any, Callable, Literal, Optional, Type, TypeVar, Union
+
+import psutil  # type: ignore
+from loguru import logger
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def get_memory_watchdog_enabled() -> bool:
+    """
+    Get the memory_watchdog_enabled setting from the application configuration.
+
+    Returns:
+        bool: True if memory_watchdog_enabled is enabled, False otherwise (defaults to False)
+    """
+    try:
+        from fides.api.api.deps import get_autoclose_db_session as get_db
+        from fides.config.config_proxy import ConfigProxy
+
+        with get_db() as db:
+            config_proxy = ConfigProxy(db)
+            # ConfigProxy returns None when no config record exists, so we must handle None explicitly
+            value = getattr(config_proxy.execution, "memory_watchdog_enabled")
+            return value if value is not None else False
+    except Exception:  # pragma: no cover
+        # default to disabled for backward compatibility
+        return False
+
+
+class MemoryLimitExceeded(RuntimeError):
+    """Raised when the watchdog detects sustained memory usage above the threshold."""
+
+    def __init__(self, message: str, *, memory_percent: Optional[float] = None) -> None:
+        super().__init__(message)
+        self.memory_percent: Optional[float] = memory_percent
+
+
+class MemoryWatchdog:
+    """Background memory monitor that enables graceful task termination.
+
+    Monitors system memory usage and triggers controlled shutdown when usage exceeds the threshold,
+    preventing forceful termination by the system's OOM killer.
+
+    Parameters
+    ----------
+    threshold:
+        Percentage of *overall* system memory usage at which the watchdog triggers.
+    check_interval:
+        How often (in seconds) to sample ``psutil.virtual_memory()``.
+    grace_period:
+        Deprecated - kept for compatibility but defaults to 0 for immediate action.
+    """
+
+    def __init__(
+        self, *, threshold: int = 90, check_interval: float = 0.5, grace_period: int = 0
+    ) -> None:
+        self.threshold = threshold
+        self.check_interval = check_interval
+        self.grace_period = grace_period
+
+        self._thread: Optional[threading.Thread] = None
+        self._monitoring = threading.Event()
+        self._original_handler: Union[
+            Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None
+        ] = None
+
+    # ---------------------------------------------------------------------
+    # Public helpers
+    # ---------------------------------------------------------------------
+    def start(self) -> None:
+        """Start the background monitoring thread and register the signal handler."""
+        logger.debug(
+            "Starting memory watchdog - threshold={}%, check_interval={}s",
+            self.threshold,
+            self.check_interval,
+        )
+        self._original_handler = signal.signal(signal.SIGUSR1, self._signal_handler)
+        self._monitoring.set()
+        self._thread = threading.Thread(
+            target=self._run, name="memory-watchdog", daemon=True
+        )
+        self._thread.start()
+
+    def stop(self) -> None:
+        """Stop the monitoring thread and restore the previous signal handler."""
+        logger.debug("Stopping memory watchdog")
+        self._monitoring.clear()
+        if self._thread and self._thread.is_alive():
+            self._thread.join(timeout=2)
+        if self._original_handler is not None:
+            signal.signal(signal.SIGUSR1, self._original_handler)
+            self._original_handler = None
+
+    # ------------------------------------------------------------------
+    # Context-manager support
+    # ------------------------------------------------------------------
+    def __enter__(self) -> "MemoryWatchdog":
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Literal[False]:
+        self.stop()
+        # Do not suppress exceptions
+        return False
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+    def _signal_handler(
+        self, signum: int, frame: Optional[FrameType]
+    ) -> None:  # noqa: D401 – Celery uses signal handlers
+        """Convert the signal into an exception on the main thread."""
+        current_percent = _system_memory_percent()
+        logger.error("Memory limit exceeded: {}%", current_percent)
+        self.stop()
+        raise MemoryLimitExceeded(
+            "Memory usage exceeded threshold", memory_percent=current_percent
+        )
+
+    def _run(self) -> None:
+        """Background thread body."""
+        while self._monitoring.is_set():
+            try:
+                mem_percent = _system_memory_percent()
+
+                if mem_percent > self.threshold:
+                    # Trigger graceful termination before the system OOM killer can intervene
+                    logger.error(
+                        "Memory usage {}% above threshold {}% - sending SIGUSR1 to terminate task gracefully (prevents OOM kill)",
+                        mem_percent,
+                        self.threshold,
+                    )
+                    os.kill(os.getpid(), signal.SIGUSR1)
+                    return  # stop monitoring after signal
+
+                time.sleep(self.check_interval)
+            except Exception as exc:  # pragma: no cover – best-effort logging
+                logger.exception("Uncaught error in memory watchdog: {}", exc)
+                break
+
+
+# -----------------------------------------------------------------------------
+# Decorator for Celery tasks (or any function)
+# -----------------------------------------------------------------------------
+
+
+def memory_limiter(
+    _func: Optional[F] = None,
+    *,
+    threshold: int = 90,
+    check_interval: float = 0.5,
+    grace_period: int = 0,
+) -> Union[F, Callable[[F], F]]:
+    """Decorator that runs the wrapped callable under a :class:`MemoryWatchdog`.
+
+    Enables graceful task termination when memory usage is high, preventing WorkerLostError
+    from system OOM killer intervention.
+
+    Can be applied **with or without arguments**::
+
+        @memory_limiter
+        def task_a():
+            ...
+
+        @memory_limiter(threshold=85)
+        def task_b():
+            ...
+    """
+
+    def decorator(func: F) -> F:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            # Check if memory watchdog is enabled
+            if not get_memory_watchdog_enabled():
+                logger.debug(
+                    "Memory watchdog disabled by configuration - running task without monitoring"
+                )
+                return func(*args, **kwargs)
+
+            watchdog = MemoryWatchdog(
+                threshold=threshold,
+                check_interval=check_interval,
+                grace_period=grace_period,
+            )
+            watchdog.start()
+            try:
+                return func(*args, **kwargs)
+            except MemoryLimitExceeded as exc:
+                logger.error(
+                    "Task terminated gracefully due to memory pressure: {}%",
+                    exc.memory_percent,
+                )
+                raise
+            finally:
+                watchdog.stop()
+
+        return wrapper  # type: ignore[return-value]
+
+    # If called without arguments, the first positional argument is the function
+    if _func is not None and callable(_func):
+        return decorator(_func)
+
+    return decorator
+
+
+# -----------------------------------------------------------------------------
+# Platform helpers – cgroup-aware memory measurement
+# -----------------------------------------------------------------------------
+
+# Docker / Kubernetes containers are usually memory-constrained via cgroups.  If we only look at
+# host-wide memory usage the watchdog will never fire, because the host may still have plenty of
+# free RAM even though the container is about to be OOM-killed.  The helpers below read the
+# relevant cgroup files (v2 first, then v1) and compute the percentage of the *container limit* in
+# use.  When cgroup information is unavailable (e.g. running directly on the host) we return
+# `None` and fall back to `psutil`.
+
+
+def _cgroup_memory_percent() -> Optional[float]:
+    """Return container memory usage as *percentage* of the cgroup limit or `None` when not
+    running inside a memory-limited cgroup."""
+
+    def _read(path: str) -> Optional[int]:
+        try:
+            with open(path, "r", encoding="utf-8") as fh:
+                raw = fh.read().strip()
+                if raw == "" or raw.lower() == "max":  # v2 "max" means unlimited
+                    return None
+                return int(raw)
+        except (FileNotFoundError, PermissionError, ValueError):  # pragma: no cover
+            return None
+
+    # cgroups v2 (unified)
+    usage = _read("/sys/fs/cgroup/memory.current")
+    limit = _read("/sys/fs/cgroup/memory.max")
+    if usage is not None and limit and limit > 0:
+        return usage / limit * 100
+
+    # cgroups v1
+    usage = _read("/sys/fs/cgroup/memory/memory.usage_in_bytes")
+    limit = _read("/sys/fs/cgroup/memory/memory.limit_in_bytes")
+    if (
+        usage is not None and limit and 0 < limit < (1 << 60)
+    ):  # ignore enormous "unlimited" value
+        return usage / limit * 100
+
+    return None
+
+
+def _system_memory_percent() -> float:
+    """Best-effort *current* memory usage percentage for the running worker.
+
+    1. Use cgroup metrics when they are available and meaningful.
+    2. Fall back to host-wide `psutil.virtual_memory().percent` otherwise."""
+
+    cgroup_percent = _cgroup_memory_percent()
+    if cgroup_percent is not None:
+        return cgroup_percent
+
+    # Fall back to host memory usage
+    return psutil.virtual_memory().percent

fides/config/execution_settings.py

@@ -57,6 +57,10 @@ class ExecutionSettings(FidesSettings):
         default=300,
         description="Seconds between polling for interrupted tasks to requeue",
     )
+    privacy_request_requeue_retry_count: int = Field(
+        default=3,
+        description="The number of times a privacy request will be requeued when its tasks are interrupted before being marked as error",
+    )
     use_dsr_3_0: bool = Field(
         default=False,
         description="Temporary flag to switch to using DSR 3.0 to process your tasks.",
@@ -77,4 +81,8 @@ class ExecutionSettings(FidesSettings):
         default="US/Eastern",
         description="The timezone to send batch emails for DSR email integration.",
     )
+    memory_watchdog_enabled: bool = Field(
+        default=False,
+        description="Whether the memory watchdog is enabled to monitor and gracefully terminate tasks that approach memory limits.",
+    )
     model_config = SettingsConfigDict(env_prefix=ENV_PREFIX)

fides/config/utils.py

@@ -61,6 +61,7 @@ CONFIG_KEY_ALLOWLIST = {
         "task_retry_backoff",
         "require_manual_request_approval",
         "subject_identity_verification_required",
+        "memory_watchdog_enabled",
         "sql_dry_run",
     ],
     "storage": [

fides/ui-build/static/admin/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/d9924caa849931b3.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d9924caa849931b3.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-90e8ec1fc5c6455b.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-5c3a63bb1697f34c.js" defer=""></script><script src="/_next/static/chunks/pages/404-2d803dab6a00f353.js" defer=""></script><script src="/_next/static/ZIM71ZcqBBeTYHc-MN9_n/_buildManifest.js" defer=""></script><script src="/_next/static/ZIM71ZcqBBeTYHc-MN9_n/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"ZIM71ZcqBBeTYHc-MN9_n","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/d9924caa849931b3.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d9924caa849931b3.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-90e8ec1fc5c6455b.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-750d6bd16c971bb9.js" defer=""></script><script src="/_next/static/chunks/pages/404-2d803dab6a00f353.js" defer=""></script><script src="/_next/static/v1eqRIfzld3di00TTnVM9/_buildManifest.js" defer=""></script><script src="/_next/static/v1eqRIfzld3di00TTnVM9/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"v1eqRIfzld3di00TTnVM9","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>