ethyca-fides 2.67.0rc2__tar.gz → 2.67.1b1__tar.gz

{ethyca_fides-2.67.0rc2/src/ethyca_fides.egg-info → ethyca_fides-2.67.1b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ethyca_fides
-Version: 2.67.0rc2
+Version: 2.67.1b1
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1/src/ethyca_fides.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ethyca_fides
-Version: 2.67.0rc2
+Version: 2.67.1b1
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/ethyca_fides.egg-info/SOURCES.txt

@@ -737,6 +737,7 @@ src/fides/api/util/lock.py
 src/fides/api/util/logger.py
 src/fides/api/util/logger_context_utils.py
 src/fides/api/util/matching_queue.py
+src/fides/api/util/memory_watchdog.py
 src/fides/api/util/querytoken.py
 src/fides/api/util/saas_util.py
 src/fides/api/util/scylla_util.py
@@ -870,8 +871,6 @@ src/fides/ui-build/static/admin/properties.html
 src/fides/ui-build/static/admin/systems.html
 src/fides/ui-build/static/admin/taxonomy.html
 src/fides/ui-build/static/admin/user-management.html
-src/fides/ui-build/static/admin/_next/static/5x65uIwZtfTiu6ITZ4wqq/_buildManifest.js
-src/fides/ui-build/static/admin/_next/static/5x65uIwZtfTiu6ITZ4wqq/_ssgManifest.js
 src/fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js
 src/fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js
 src/fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js
@@ -1065,6 +1064,8 @@ src/fides/ui-build/static/admin/_next/static/media/inter-vietnamese-500-normal.7
 src/fides/ui-build/static/admin/_next/static/media/inter-vietnamese-600-normal.eb82de1d.woff2
 src/fides/ui-build/static/admin/_next/static/media/inter-vietnamese-700-normal.00e401a9.woff2
 src/fides/ui-build/static/admin/_next/static/media/logo-white.c9bf4196.svg
+src/fides/ui-build/static/admin/_next/static/sFDtXdzSc8wR5M8bdSVpg/_buildManifest.js
+src/fides/ui-build/static/admin/_next/static/sFDtXdzSc8wR5M8bdSVpg/_ssgManifest.js
 src/fides/ui-build/static/admin/add-systems/manual.html
 src/fides/ui-build/static/admin/add-systems/multiple.html
 src/fides/ui-build/static/admin/consent/configure.html

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-31T12:03:42-0400",
+ "date": "2025-07-31T10:13:21-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "bbc0caadf698e6921fea436d503659506ca6bcce",
- "version": "2.67.0rc2"
+ "full-revisionid": "fdf4599646a5637c08ee8ba13674b64a0aa5ad42",
+ "version": "2.67.1b1"
 }
 '''  # END VERSION_JSON
 

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/models/manual_task/manual_task.py

@@ -383,15 +383,13 @@ class ManualTaskInstance(Base):
 
     @property
     def incomplete_fields(self) -> list["ManualTaskConfigField"]:
-        """Get all fields that haven't been completed yet.
-        A field is considered incomplete if:
-        1. It's required and has no submission
+        """Get all fields that have no submission.
         Returns:
             list[ManualTaskConfigField]: List of incomplete fields
         """
         return [
             field
-            for field in self.required_fields
+            for field in self.config.field_definitions
             if not self.get_submission_for_field(field.id)
         ]
 
@@ -401,8 +399,7 @@ class ManualTaskInstance(Base):
         return [
             field
             for field in self.config.field_definitions
-            if field.field_metadata.get("required", False)
-            and self.get_submission_for_field(field.id)
+            if self.get_submission_for_field(field.id)
         ]
 
     def get_submission_for_field(

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/models/privacy_request/privacy_request.py

@@ -47,8 +47,16 @@ from fides.api.models.attachment import (
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
 from fides.api.models.comment import Comment, CommentReference, CommentReferenceType
+from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.field_types import EncryptedLargeDataDescriptor
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskConfig,
+    ManualTaskConfigurationType,
+    ManualTaskEntityType,
+    ManualTaskInstance,
+)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.masking_secret import MaskingSecret
 from fides.api.models.policy import (
@@ -200,6 +208,14 @@ class PrivacyRequest(
         viewonly=True,
         uselist=True,
     )
+    manual_task_instances = relationship(
+        "ManualTaskInstance",
+        lazy="select",
+        passive_deletes="all",
+        primaryjoin="and_(ManualTaskInstance.entity_id==foreign(PrivacyRequest.id), "
+        "ManualTaskInstance.entity_type=='privacy_request')",
+        uselist=True,
+    )
     property_id = Column(String, nullable=True)
 
     cancel_reason = Column(String(200))
@@ -1009,6 +1025,38 @@ class PrivacyRequest(
                 request_task_celery_ids.append(request_task_id)
         return request_task_celery_ids
 
+    def cancel_celery_tasks(self) -> None:
+        """Cancel all Celery tasks associated with this privacy request.
+
+        This includes both the main privacy request task and any sub-tasks (Request Tasks).
+        """
+        task_ids: List[str] = []
+
+        # Add the main privacy request task ID
+        parent_task_id = self.get_cached_task_id()
+        if parent_task_id:
+            task_ids.append(parent_task_id)
+
+        # Add all request task IDs
+        request_task_celery_ids = self.get_request_task_celery_task_ids()
+        task_ids.extend(request_task_celery_ids)
+
+        if not task_ids:
+            return
+
+        # Revoke all Celery tasks in batch
+        logger.info(f"Revoking {len(task_ids)} tasks for privacy request {self.id}")
+        try:
+            # Use terminate=False to allow graceful shutdown if already running
+            celery_app.control.revoke(task_ids, terminate=False)
+            logger.info(
+                f"Successfully revoked {len(task_ids)} tasks for privacy request {self.id}"
+            )
+        except Exception as exc:
+            logger.warning(
+                f"Failed to revoke {len(task_ids)} tasks for privacy request {self.id}: {exc}"
+            )
+
     def cancel_processing(self, db: Session, cancel_reason: Optional[str]) -> None:
         """Cancels a privacy request.  Currently should only cancel 'pending' tasks
 
@@ -1021,19 +1069,7 @@ class PrivacyRequest(
             self.canceled_at = datetime.utcnow()
             self.save(db)
 
-            task_ids: List[str] = (
-                self.get_request_task_celery_task_ids()
-            )  # Celery tasks for sub tasks (DSR 3.0 Request Tasks)
-            parent_task_id = (
-                self.get_cached_task_id()
-            )  # Celery task for current Privacy Request
-            if parent_task_id:
-                task_ids.append(parent_task_id)
-
-            for celery_task_id in task_ids:
-                logger.info("Revoking task {} for request {}", celery_task_id, self.id)
-                # Only revokes if execution is not already in progress.
-                celery_app.control.revoke(celery_task_id, terminate=False)
+            self.cancel_celery_tasks()
 
     def error_processing(self, db: Session) -> None:
         """Mark privacy request as errored, and note time processing was finished"""
@@ -1155,6 +1191,68 @@ class PrivacyRequest(
             db, manual_webhook_id, "erasure_manual_webhook"
         )
 
+    def create_manual_task_instances(
+        self, db: Session, connection_configs_with_manual_tasks: list[ConnectionConfig]
+    ) -> list[ManualTaskInstance]:
+        """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
+        # Early return if no relevant policy rules
+        policy_rules = {
+            ActionType.access: bool(
+                self.policy.get_rules_for_action(action_type=ActionType.access)
+            ),
+            ActionType.erasure: bool(
+                self.policy.get_rules_for_action(action_type=ActionType.erasure)
+            ),
+        }
+
+        if not any(policy_rules.values()):
+            return []
+
+        # Build configuration types using list comprehension
+        config_types = [
+            (
+                ManualTaskConfigurationType.access_privacy_request
+                if action_type == ActionType.access
+                else ManualTaskConfigurationType.erasure_privacy_request
+            )
+            for action_type, has_rules in policy_rules.items()
+            if has_rules
+        ]
+
+        # Get all relevant manual tasks and configs in one query
+        connection_config_ids = [cc.id for cc in connection_configs_with_manual_tasks]
+        manual_tasks_with_configs = (
+            db.query(ManualTask, ManualTaskConfig)
+            .join(ManualTaskConfig, ManualTask.id == ManualTaskConfig.task_id)
+            .filter(
+                ManualTask.parent_entity_id.in_(connection_config_ids),
+                ManualTask.parent_entity_type == "connection_config",
+                ManualTaskConfig.is_current.is_(True),
+                ManualTaskConfig.config_type.in_(config_types),
+            )
+            .all()
+        )
+
+        # Get existing config IDs to avoid duplicates
+        existing_config_ids = {
+            instance.config_id for instance in self.manual_task_instances
+        }
+
+        # Create instances using list comprehension and filter out existing ones
+        return [
+            ManualTaskInstance.create(
+                db=db,
+                data={
+                    "entity_id": self.id,
+                    "entity_type": ManualTaskEntityType.privacy_request,
+                    "task_id": manual_task.id,
+                    "config_id": config.id,
+                },
+            )
+            for manual_task, config in manual_tasks_with_configs
+            if config.id not in existing_config_ids
+        ]
+
     def get_existing_request_task(
         self,
         db: Session,

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/schemas/application_config.py

@@ -70,6 +70,7 @@ class ExecutionApplicationConfig(FidesSchema):
     subject_identity_verification_required: Optional[bool] = None
     disable_consent_identity_verification: Optional[bool] = None
     require_manual_request_approval: Optional[bool] = None
+    memory_watchdog_enabled: Optional[bool] = None
     sql_dry_run: Optional[SqlDryRunMode] = None
 
     model_config = ConfigDict(use_enum_values=True, extra="forbid")

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py

@@ -13,7 +13,7 @@ from loguru import logger
 
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
-from fides.api.util.storage_util import StorageJSONEncoder
+from fides.api.util.storage_util import StorageJSONEncoder, format_size
 
 DSR_DIRECTORY = Path(__file__).parent.resolve()
 
@@ -204,7 +204,7 @@ class DsrReportBuilder:
 
             file_size = attachment.get("file_size")
             if isinstance(file_size, (int, float)):
-                file_size = self._format_size(float(file_size))
+                file_size = format_size(float(file_size))
             else:
                 file_size = "Unknown"
 
@@ -321,22 +321,6 @@ class DsrReportBuilder:
 
         return datasets
 
-    def _format_size(self, size_bytes: float) -> str:
-        """
-        Format size in bytes to human readable format.
-
-        Args:
-            size_bytes: Size in bytes
-
-        Returns:
-            Formatted string with appropriate unit (B, KB, MB, GB)
-        """
-        for unit in ["B", "KB", "MB", "GB"]:
-            if size_bytes < 1024.0:
-                return f"{size_bytes:.1f} {unit}"
-            size_bytes /= 1024.0
-        return f"{size_bytes:.1f} TB"
-
     def generate(self) -> BytesIO:
         """
         Processes the request and DSR data to build zip file containing the DSR report.
@@ -395,7 +379,7 @@ class DsrReportBuilder:
 
         # Calculate time taken and file size
         time_taken = time_module.time() - start_time
-        file_size = self._format_size(float(len(self.baos.getvalue())))
+        file_size = format_size(float(len(self.baos.getvalue())))
 
         logger.bind(time_to_generate=time_taken, dsr_package_size=file_size).info(
             "DSR report generation complete."

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/service/privacy_request/request_runner_service.py

@@ -80,6 +80,7 @@ from fides.api.util.cache import get_all_masking_secret_keys
 from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
+from fides.api.util.memory_watchdog import memory_limiter
 from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_TRANSFER_TO_PARENT,
     V1_URL_PREFIX,
@@ -358,8 +359,8 @@ def upload_and_save_access_results(  # pylint: disable=R0912
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
-# TODO: Add log_context back in, this is just for some temporary testing
-# @log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
+@memory_limiter
+@log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
 def run_privacy_request(
     self: DatabaseTask,
     privacy_request_id: str,

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/service/privacy_request/request_service.py

@@ -3,11 +3,12 @@ from __future__ import annotations
 import json
 from asyncio import sleep
 from datetime import datetime, timedelta
-from typing import Any, Dict, Optional, Set
+from typing import Any, Dict, List, Optional, Set
 
 from httpx import AsyncClient
 from loguru import logger
 from sqlalchemy import text
+from sqlalchemy.orm import Session
 from sqlalchemy.sql.elements import TextClause
 
 from fides.api.common_exceptions import PrivacyRequestNotFound
@@ -31,6 +32,9 @@ from fides.api.util.cache import (
     celery_tasks_in_flight,
     get_async_task_tracking_cache_key,
     get_cache,
+    get_privacy_request_retry_count,
+    increment_privacy_request_retry_count,
+    reset_privacy_request_retry_count,
 )
 from fides.api.util.lock import redis_lock
 from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS, V1_URL_PREFIX
@@ -350,10 +354,17 @@ def initiate_interrupted_task_requeue_poll() -> None:
 
 
 def get_cached_task_id(entity_id: str) -> Optional[str]:
-    """Gets the cached task ID for a privacy request or request task by ID."""
+    """Gets the cached task ID for a privacy request or request task by ID.
+
+    Raises Exception if cache operations fail, allowing callers to handle cache failures appropriately.
+    """
     cache: FidesopsRedis = get_cache()
-    task_id = cache.get(get_async_task_tracking_cache_key(entity_id))
-    return task_id
+    try:
+        task_id = cache.get(get_async_task_tracking_cache_key(entity_id))
+        return task_id
+    except Exception as exc:
+        logger.error(f"Failed to get cached task ID for entity {entity_id}: {exc}")
+        raise
 
 
 REQUEUE_INTERRUPTED_TASKS_LOCK = "requeue_interrupted_tasks_lock"
@@ -393,6 +404,115 @@ def _get_task_ids_from_dsr_queue(
     return queued_tasks_ids
 
 
+def _cancel_interrupted_tasks_and_error_privacy_request(
+    db: Session, privacy_request: PrivacyRequest, error_message: Optional[str] = None
+) -> None:
+    """
+    Cancel all tasks associated with an interrupted privacy request and set the privacy request to error state.
+
+    This function:
+    1. Logs the error message (either provided or default)
+    2. Revokes the main privacy request task and all associated request tasks
+    3. Sets the privacy request status to error
+    4. Creates an error log entry
+
+    Args:
+        db: Database session
+        privacy_request: The privacy request to cancel and error
+        error_message: Optional error message to log. If not provided, uses default message.
+    """
+    if error_message:
+        logger.error(error_message)
+    else:
+        logger.error(
+            f"Canceling interrupted tasks and marking privacy request {privacy_request.id} as error"
+        )
+
+    # Cancel all associated Celery tasks
+    privacy_request.cancel_celery_tasks()
+
+    # Set privacy request to error state using the existing method
+    try:
+        privacy_request.error_processing(db)
+        logger.info(
+            f"Privacy request {privacy_request.id} marked as error due to task interruption"
+        )
+    except Exception as exc:
+        logger.error(
+            f"Failed to mark privacy request {privacy_request.id} as error: {exc}"
+        )
+
+
+def _handle_privacy_request_requeue(
+    db: Session, privacy_request: PrivacyRequest
+) -> None:
+    """Handle retry logic for a privacy request - either requeue or cancel based on retry count."""
+    try:
+        # Check retry count and either requeue or cancel based on limit
+        current_retry_count = get_privacy_request_retry_count(privacy_request.id)
+        max_retries = CONFIG.execution.privacy_request_requeue_retry_count
+
+        if current_retry_count < max_retries:
+            # Increment retry count and attempt requeue
+            new_retry_count = increment_privacy_request_retry_count(privacy_request.id)
+            logger.info(
+                f"Requeuing privacy request {privacy_request.id} "
+                f"(attempt {new_retry_count}/{max_retries})"
+            )
+
+            from fides.service.privacy_request.privacy_request_service import (  # pylint: disable=cyclic-import
+                PrivacyRequestError,
+                _requeue_privacy_request,
+            )
+
+            try:
+                _requeue_privacy_request(db, privacy_request)
+            except PrivacyRequestError as exc:
+                # If requeue fails, cancel tasks and set to error state
+                _cancel_interrupted_tasks_and_error_privacy_request(
+                    db, privacy_request, exc.message
+                )
+        else:
+            # Exceeded retry limit, cancel tasks and set to error state
+            _cancel_interrupted_tasks_and_error_privacy_request(
+                db,
+                privacy_request,
+                f"Privacy request {privacy_request.id} exceeded max retry attempts "
+                f"({max_retries}), canceling tasks and setting to error state",
+            )
+            # Reset retry count since we're giving up
+            reset_privacy_request_retry_count(privacy_request.id)
+
+    except Exception as cache_exc:
+        # If cache operations fail (Redis down, network issues, etc.), fail safe by canceling
+        _cancel_interrupted_tasks_and_error_privacy_request(
+            db,
+            privacy_request,
+            f"Cache operation failed for privacy request {privacy_request.id}, "
+            f"failing safe by canceling tasks: {cache_exc}",
+        )
+
+
+def _get_request_task_ids_in_progress(
+    db: Session, privacy_request_id: str
+) -> List[str]:
+    """Get the IDs of request tasks that are currently in progress for a privacy request."""
+    request_tasks_in_progress = (
+        db.query(RequestTask.id)
+        .filter(RequestTask.privacy_request_id == privacy_request_id)
+        .filter(
+            RequestTask.status.in_(
+                [
+                    ExecutionLogStatus.in_processing,
+                    ExecutionLogStatus.pending,
+                ]
+            )
+        )
+        .all()
+    )
+    return [task[0] for task in request_tasks_in_progress]
+
+
 # pylint: disable=too-many-branches
 @celery_app.task(base=DatabaseTask, bind=True)
 def requeue_interrupted_tasks(self: DatabaseTask) -> None:
@@ -442,17 +562,40 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
             )
 
             # Get task IDs from the queue in a memory-efficient way
-            queued_tasks_ids = _get_task_ids_from_dsr_queue(redis_conn)
+            try:
+                queued_tasks_ids = _get_task_ids_from_dsr_queue(redis_conn)
+            except Exception as queue_exc:
+                logger.warning(
+                    f"Failed to get task IDs from queue, skipping queue state checks: {queue_exc}"
+                )
+                return
 
             # Check each privacy request
             for privacy_request in in_progress_requests:
                 should_requeue = False
                 logger.debug(f"Checking tasks for privacy request {privacy_request.id}")
 
-                task_id = get_cached_task_id(privacy_request.id)
+                try:
+                    task_id = get_cached_task_id(privacy_request.id)
+                except Exception as cache_exc:
+                    # If we can't get the task ID due to cache failure, fail safe by canceling
+                    _cancel_interrupted_tasks_and_error_privacy_request(
+                        db,
+                        privacy_request,
+                        f"Cache failure when getting task ID for privacy request {privacy_request.id}, "
+                        f"failing safe by canceling tasks: {cache_exc}",
+                    )
+                    continue
 
                 # If the task ID is not cached, we can't check if it's running
+                # This means the request is stuck - cancel it
                 if not task_id:
+                    _cancel_interrupted_tasks_and_error_privacy_request(
+                        db,
+                        privacy_request,
+                        f"No task ID found for privacy request {privacy_request.id}, "
+                        f"request is stuck without a running task - canceling",
+                    )
                     continue
 
                 # Check if the main privacy request task is active
@@ -470,30 +613,36 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                         )
                         should_requeue = True
 
-                    request_tasks_in_progress = (
-                        db.query(RequestTask.id)
-                        .filter(RequestTask.privacy_request_id == privacy_request.id)
-                        .filter(
-                            RequestTask.status.in_(
-                                [
-                                    ExecutionLogStatus.in_processing,
-                                    ExecutionLogStatus.pending,
-                                ]
-                            )
-                        )
-                        .all()
+                    request_task_ids_in_progress = _get_request_task_ids_in_progress(
+                        db, privacy_request.id
                     )
-                    request_task_ids_in_progress = [
-                        task[0] for task in request_tasks_in_progress
-                    ]
 
                     # Check each individual request task
                     for request_task_id in request_task_ids_in_progress:
-                        subtask_id = get_cached_task_id(request_task_id)
+                        try:
+                            subtask_id = get_cached_task_id(request_task_id)
+                        except Exception as cache_exc:
+                            # If we can't get the subtask ID due to cache failure, fail safe by canceling
+                            _cancel_interrupted_tasks_and_error_privacy_request(
+                                db,
+                                privacy_request,
+                                f"Cache failure when getting subtask ID for request task {request_task_id} "
+                                f"(privacy request {privacy_request.id}), failing safe by canceling tasks: {cache_exc}",
+                            )
+                            should_requeue = False
+                            break
 
                         # If the task ID is not cached, we can't check if it's running
+                        # This means the subtask is stuck - cancel the entire privacy request
                         if not subtask_id:
-                            continue
+                            _cancel_interrupted_tasks_and_error_privacy_request(
+                                db,
+                                privacy_request,
+                                f"No task ID found for request task {request_task_id} "
+                                f"(privacy request {privacy_request.id}), subtask is stuck - canceling privacy request",
+                            )
+                            should_requeue = False
+                            break
 
                         if (
                             subtask_id not in queued_tasks_ids
@@ -507,12 +656,4 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
 
                 # Requeue the privacy request if needed
                 if should_requeue:
-                    from fides.service.privacy_request.privacy_request_service import (  # pylint: disable=cyclic-import
-                        PrivacyRequestError,
-                        _requeue_privacy_request,
-                    )
-
-                    try:
-                        _requeue_privacy_request(db, privacy_request)
-                    except PrivacyRequestError as exc:
-                        logger.error(exc.message)
+                    _handle_privacy_request_requeue(db, privacy_request)

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/task/create_request_tasks.py

@@ -35,7 +35,7 @@ from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    create_manual_task_instances_for_privacy_request,
+    get_connection_configs_with_manual_tasks,
 )
 from fides.api.util.logger_context_utils import log_context
 
@@ -92,7 +92,7 @@ def build_access_networkx_digraph(
     manual_nodes = [
         addr
         for addr in traversal_nodes.keys()
-        if addr.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
+        if ManualTaskAddress.is_manual_task_address(addr)
     ]
     for manual_node in manual_nodes:
         networkx_graph.add_edge(ROOT_COLLECTION_ADDRESS, manual_node)
@@ -472,7 +472,9 @@ def run_access_request(
             )
 
             # Snapshot manual task field instances for this privacy request
-            create_manual_task_instances_for_privacy_request(session, privacy_request)
+            privacy_request.create_manual_task_instances(
+                session, get_connection_configs_with_manual_tasks(session)
+            )
 
             # Save Access Request Tasks to the database
             ready_tasks = persist_new_access_request_tasks(

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/task/execute_request_tasks.py

@@ -36,6 +36,7 @@ from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.util.cache import cache_task_tracking_key
 from fides.api.util.collection_util import Row
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
+from fides.api.util.memory_watchdog import memory_limiter
 
 # DSR 3.0 task functions
 
@@ -255,6 +256,7 @@ def queue_downstream_tasks(
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
+@memory_limiter
 @log_context(
     capture_args={
         "privacy_request_id": LoggerContextKeys.privacy_request_id,
@@ -319,6 +321,7 @@ def run_access_node(
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
+@memory_limiter
 @log_context(
     capture_args={
         "privacy_request_id": LoggerContextKeys.privacy_request_id,
@@ -391,6 +394,7 @@ def run_erasure_node(
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
+@memory_limiter
 @log_context(
     capture_args={
         "privacy_request_id": LoggerContextKeys.privacy_request_id,

{ethyca_fides-2.67.0rc2 → ethyca_fides-2.67.1b1}/src/fides/api/task/filter_results.py

@@ -39,7 +39,7 @@ def filter_data_categories(
             continue
 
         # Skip manual task data - it doesn't need filtering since it's controlled by field definitions
-        if f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}" in node_address:
+        if ManualTaskAddress.is_manual_task_address(node_address):
             filtered_access_results[node_address].extend(results)
             continue