ethyca-fides 2.56.1b1__tar.gz → 2.56.1b2__tar.gz

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ethyca_fides
-Version: 2.56.1b1
+Version: 2.56.1b2
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/ethyca_fides.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ethyca-fides
-Version: 2.56.1b1
+Version: 2.56.1b2
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/ethyca_fides.egg-info/SOURCES.txt

@@ -504,6 +504,7 @@ src/fides/api/schemas/storage/storage.py
 src/fides/api/schemas/storage/storage_secrets_docs_only.py
 src/fides/api/schemas/third_party/__init__.py
 src/fides/api/service/__init__.py
+src/fides/api/service/deps.py
 src/fides/api/service/outbound_urn_registry.py
 src/fides/api/service/strategy.py
 src/fides/api/service/authentication/__init__.py
@@ -787,8 +788,8 @@ src/fides/ui-build/static/admin/properties.html
 src/fides/ui-build/static/admin/systems.html
 src/fides/ui-build/static/admin/taxonomy.html
 src/fides/ui-build/static/admin/user-management.html
-src/fides/ui-build/static/admin/_next/static/Yp4lTh1cgsDcOfwB95ehL/_buildManifest.js
-src/fides/ui-build/static/admin/_next/static/Yp4lTh1cgsDcOfwB95ehL/_ssgManifest.js
+src/fides/ui-build/static/admin/_next/static/SJ-oaW6pE4TEG81nDC-LR/_buildManifest.js
+src/fides/ui-build/static/admin/_next/static/SJ-oaW6pE4TEG81nDC-LR/_ssgManifest.js
 src/fides/ui-build/static/admin/_next/static/chunks/1241-438935108aed21e7.js
 src/fides/ui-build/static/admin/_next/static/chunks/1288-ba1df63e7fa2234a.js
 src/fides/ui-build/static/admin/_next/static/chunks/1827-2d2f6eb59ebca43f.js
@@ -841,7 +842,7 @@ src/fides/ui-build/static/admin/_next/static/chunks/8986-b955f08aba857064.js
 src/fides/ui-build/static/admin/_next/static/chunks/9111-36f3eb6456817560.js
 src/fides/ui-build/static/admin/_next/static/chunks/9255-2b1bafefb091a640.js
 src/fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js
-src/fides/ui-build/static/admin/_next/static/chunks/9659-4757cc92c551432f.js
+src/fides/ui-build/static/admin/_next/static/chunks/9659-7a6f150315045fdd.js
 src/fides/ui-build/static/admin/_next/static/chunks/9822-e0b4ee48b9ca9d6c.js
 src/fides/ui-build/static/admin/_next/static/chunks/c78d26b1-88a3e1bacb2a03c2.js
 src/fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js
@@ -851,7 +852,7 @@ src/fides/ui-build/static/admin/_next/static/chunks/main-app-de538a24cbd2733c.js
 src/fides/ui-build/static/admin/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js
 src/fides/ui-build/static/admin/_next/static/chunks/webpack-f8d056f5e13fe08a.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/404-cf2b98b932c99b1e.js
-src/fides/ui-build/static/admin/_next/static/chunks/pages/_app-817f3fe70282b855.js
+src/fides/ui-build/static/admin/_next/static/chunks/pages/_app-ec8f262156df2273.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/_error-c91b5eabd7ac9382.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-500ee461301938a8.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/ant-poc-833eede89996ded2.js

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-02-27T16:33:57+0000",
+ "date": "2025-02-28T13:28:31-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "3c01497d0f655c36d069ca0cd566daeeb0d59d70",
- "version": "2.56.1b1"
+ "full-revisionid": "dfcb91831d0ae08c20c88ebddce0057928076129",
+ "version": "2.56.1b2"
 }
 '''  # END VERSION_JSON
 

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/fides/api/api/deps.py

@@ -10,10 +10,6 @@ from fides.api.util.cache import get_cache as get_redis_connection
 from fides.config import CONFIG, FidesConfig
 from fides.config import get_config as get_app_config
 from fides.config.config_proxy import ConfigProxy
-from fides.service.dataset.dataset_config_service import DatasetConfigService
-from fides.service.dataset.dataset_service import DatasetService
-from fides.service.messaging.messaging_service import MessagingService
-from fides.service.privacy_request.privacy_request_service import PrivacyRequestService
 
 _engine = None
 
@@ -70,27 +66,3 @@ def get_cache() -> Generator:
             "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a Redis cache."
         )
     yield get_redis_connection()
-
-
-def get_messaging_service(
-    db: Session = Depends(get_db),
-    config: FidesConfig = Depends(get_config),
-    config_proxy: ConfigProxy = Depends(get_config_proxy),
-) -> MessagingService:
-    return MessagingService(db, config, config_proxy)
-
-
-def get_privacy_request_service(
-    db: Session = Depends(get_db),
-    config_proxy: ConfigProxy = Depends(get_config_proxy),
-    messaging_service: MessagingService = Depends(get_messaging_service),
-) -> PrivacyRequestService:
-    return PrivacyRequestService(db, config_proxy, messaging_service)
-
-
-def get_dataset_service(db: Session = Depends(get_db)) -> DatasetService:
-    return DatasetService(db)
-
-
-def get_dataset_config_service(db: Session = Depends(get_db)) -> DatasetConfigService:
-    return DatasetConfigService(db)

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/fides/api/api/v1/endpoints/consent_request_endpoints.py

@@ -21,7 +21,7 @@ from starlette.status import (
     HTTP_422_UNPROCESSABLE_ENTITY,
 )
 
-from fides.api.api.deps import get_config_proxy, get_db, get_messaging_service
+from fides.api.api.deps import get_config_proxy, get_db
 from fides.api.common_exceptions import (
     IdentityVerificationException,
     RedisNotConfigured,
@@ -50,6 +50,7 @@ from fides.api.schemas.privacy_request import (
     VerificationCode,
 )
 from fides.api.schemas.redis_cache import Identity
+from fides.api.service.deps import get_messaging_service
 from fides.api.util.api_router import APIRouter
 from fides.api.util.consent_util import (
     get_or_create_fides_user_device_id_provided_identity,

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/fides/api/api/v1/endpoints/dataset_config_endpoints.py

@@ -38,6 +38,7 @@ from fides.api.schemas.dataset import (
 )
 from fides.api.schemas.privacy_request import TestPrivacyRequest
 from fides.api.schemas.redis_cache import DatasetTestRequest
+from fides.api.service.deps import get_dataset_config_service
 from fides.api.util.api_router import APIRouter
 from fides.common.api.scope_registry import (
     DATASET_CREATE_OR_UPDATE,
@@ -93,9 +94,7 @@ def _get_connection_config(
 )
 def validate_dataset(
     dataset: FideslangDataset,
-    dataset_config_service: DatasetConfigService = Depends(
-        deps.get_dataset_config_service
-    ),
+    dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> ValidateDatasetResponse:
     """
@@ -134,9 +133,7 @@ def validate_dataset(
 def put_dataset_configs(
     dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=50)],  # type: ignore
     db: Session = Depends(deps.get_db),
-    dataset_config_service: DatasetConfigService = Depends(
-        deps.get_dataset_config_service
-    ),
+    dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> BulkPutDataset:
     """
@@ -185,9 +182,7 @@ def put_dataset_configs(
 )
 def patch_dataset_configs(
     dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=50)],  # type: ignore
-    dataset_config_service: DatasetConfigService = Depends(
-        deps.get_dataset_config_service
-    ),
+    dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> BulkPutDataset:
     """
@@ -223,9 +218,7 @@ def patch_dataset_configs(
 )
 def patch_datasets(
     datasets: Annotated[List[FideslangDataset], Field(max_length=50)],  # type: ignore
-    dataset_config_service: DatasetConfigService = Depends(
-        deps.get_dataset_config_service
-    ),
+    dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> BulkPutDataset:
     """
@@ -257,9 +250,7 @@ def patch_datasets(
 )
 async def patch_yaml_datasets(
     request: Request,
-    dataset_config_service: DatasetConfigService = Depends(
-        deps.get_dataset_config_service
-    ),
+    dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> BulkPutDataset:
     """
@@ -547,9 +538,7 @@ def dataset_reachability(
     *,
     db: Session = Depends(deps.get_db),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
-    dataset_config_service: DatasetConfigService = Depends(
-        deps.get_dataset_config_service
-    ),
+    dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     dataset_key: FidesKey,
     policy_key: Optional[FidesKey] = None,
 ) -> Dict[str, Any]:
@@ -595,9 +584,7 @@ def dataset_reachability(
 def test_connection_datasets(
     *,
     db: Session = Depends(deps.get_db),
-    dataset_config_service: DatasetConfigService = Depends(
-        deps.get_dataset_config_service
-    ),
+    dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
     dataset_key: FidesKey,
     test_request: DatasetTestRequest,

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/fides/api/api/v1/endpoints/generic_overrides.py

@@ -17,7 +17,7 @@ from starlette.status import (
     HTTP_422_UNPROCESSABLE_ENTITY,
 )
 
-from fides.api.api.deps import get_dataset_service, get_db
+from fides.api.api.deps import get_db
 from fides.api.common_exceptions import KeyOrNameAlreadyExists, ValidationError
 from fides.api.db.base_class import get_key_from_data
 from fides.api.db.crud import list_resource_query
@@ -34,6 +34,7 @@ from fides.api.schemas.taxonomy_extensions import (
     DataUse,
     DataUseCreateOrUpdate,
 )
+from fides.api.service.deps import get_dataset_service
 from fides.api.util.api_router import APIRouter
 from fides.api.util.errors import FidesError, ForbiddenIsDefaultTaxonomyError
 from fides.api.util.filter_utils import apply_filters_to_query

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -41,7 +41,6 @@ from starlette.status import (
 )
 
 from fides.api.api import deps
-from fides.api.api.deps import get_privacy_request_service
 from fides.api.api.v1.endpoints.dataset_config_endpoints import _get_connection_config
 from fides.api.api.v1.endpoints.manual_webhook_endpoints import (
     get_access_manual_webhook_or_404,
@@ -109,6 +108,7 @@ from fides.api.schemas.privacy_request import (
     ReviewPrivacyRequestIds,
     VerificationCode,
 )
+from fides.api.service.deps import get_messaging_service, get_privacy_request_service
 from fides.api.service.messaging.message_dispatch_service import EMAIL_JOIN_STRING
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
 from fides.api.task.filter_results import filter_data_categories
@@ -214,7 +214,7 @@ def get_privacy_request_or_error(
 def create_privacy_request(
     *,
     privacy_request_service: PrivacyRequestService = Depends(
-        deps.get_privacy_request_service
+        get_privacy_request_service
     ),
     data: Annotated[List[PrivacyRequestCreate], Field(max_length=50)],  # type: ignore
 ) -> BulkPostPrivacyRequests:
@@ -1232,7 +1232,7 @@ def restart_privacy_request_from_failure(
     *,
     db: Session = Depends(deps.get_db),
     privacy_request_service: PrivacyRequestService = Depends(
-        deps.get_privacy_request_service
+        get_privacy_request_service
     ),
 ) -> PrivacyRequestResponse:
     """Restart a privacy request from failure"""
@@ -1279,7 +1279,7 @@ def verify_identification_code(
     *,
     db: Session = Depends(deps.get_db),
     config_proxy: ConfigProxy = Depends(deps.get_config_proxy),
-    messaging_service: MessagingService = Depends(deps.get_messaging_service),
+    messaging_service: MessagingService = Depends(get_messaging_service),
     provided_code: VerificationCode,
 ) -> PrivacyRequestResponse:
     """Verify the supplied identity verification code.
@@ -1339,7 +1339,7 @@ def approve_privacy_request(
         scopes=[PRIVACY_REQUEST_REVIEW],
     ),
     privacy_request_service: PrivacyRequestService = Depends(
-        deps.get_privacy_request_service
+        get_privacy_request_service
     ),
     privacy_requests: ReviewPrivacyRequestIds,
 ) -> BulkReviewResponse:
@@ -1362,7 +1362,7 @@ def deny_privacy_request(
         scopes=[PRIVACY_REQUEST_REVIEW],
     ),
     privacy_request_service: PrivacyRequestService = Depends(
-        deps.get_privacy_request_service
+        get_privacy_request_service
     ),
     privacy_requests: DenyPrivacyRequests,
 ) -> BulkReviewResponse:
@@ -1399,7 +1399,7 @@ def mark_privacy_request_pre_approve_eligible(
     *,
     db: Session = Depends(deps.get_db),
     privacy_request_service: PrivacyRequestService = Depends(
-        deps.get_privacy_request_service
+        get_privacy_request_service
     ),
     webhook: PreApprovalWebhook = Security(
         verify_callback_oauth_pre_approval_webhook, scopes=[PRIVACY_REQUEST_REVIEW]

ethyca_fides-2.56.1b2/src/fides/api/service/deps.py

@@ -0,0 +1,39 @@
+"""
+Module to define service dependencies so service classes can be used
+via FastAPI dependency injection.
+"""
+
+from fastapi import Depends
+from sqlalchemy.orm import Session
+
+from fides.api.api.deps import get_config, get_config_proxy, get_db
+from fides.config import FidesConfig
+from fides.config.config_proxy import ConfigProxy
+from fides.service.dataset.dataset_config_service import DatasetConfigService
+from fides.service.dataset.dataset_service import DatasetService
+from fides.service.messaging.messaging_service import MessagingService
+from fides.service.privacy_request.privacy_request_service import PrivacyRequestService
+
+
+def get_messaging_service(
+    db: Session = Depends(get_db),
+    config: FidesConfig = Depends(get_config),
+    config_proxy: ConfigProxy = Depends(get_config_proxy),
+) -> MessagingService:
+    return MessagingService(db, config, config_proxy)
+
+
+def get_privacy_request_service(
+    db: Session = Depends(get_db),
+    config_proxy: ConfigProxy = Depends(get_config_proxy),
+    messaging_service: MessagingService = Depends(get_messaging_service),
+) -> PrivacyRequestService:
+    return PrivacyRequestService(db, config_proxy, messaging_service)
+
+
+def get_dataset_service(db: Session = Depends(get_db)) -> DatasetService:
+    return DatasetService(db)
+
+
+def get_dataset_config_service(db: Session = Depends(get_db)) -> DatasetConfigService:
+    return DatasetConfigService(db)

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b2}/src/fides/api/service/privacy_request/request_service.py

@@ -8,7 +8,6 @@ from typing import Any, Dict, List, Optional, Set
 from httpx import AsyncClient
 from loguru import logger
 from sqlalchemy import text
-from sqlalchemy.orm import Query
 from sqlalchemy.sql.elements import TextClause
 
 from fides.api.common_exceptions import PrivacyRequestNotFound
@@ -16,6 +15,7 @@ from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
     PrivacyRequest,
+    RequestTask,
 )
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
@@ -27,9 +27,14 @@ from fides.api.schemas.privacy_request import (
 )
 from fides.api.schemas.redis_cache import Identity
 from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
-from fides.api.tasks import DatabaseTask, celery_app
+from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
-from fides.api.util.cache import FidesopsRedis, celery_tasks_in_flight, get_cache
+from fides.api.util.cache import (
+    FidesopsRedis,
+    celery_tasks_in_flight,
+    get_async_task_tracking_cache_key,
+    get_cache,
+)
 from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS, V1_URL_PREFIX
 from fides.config import CONFIG
 
@@ -210,34 +215,51 @@ def poll_for_exited_privacy_request_tasks(self: DatabaseTask) -> Set[str]:
             .order_by(PrivacyRequest.created_at)
         )
 
-        def some_errored(tasks: Query) -> bool:
-            """All statuses have exited and at least one is errored"""
-            statuses: List[ExecutionLogStatus] = [tsk.status for tsk in tasks]
+        # TODO: With this approach, we're making 3*n queries , where n is the number of in-progress privacy requests.
+        # We could optimize this to just get all the RequestTasks for all in-progress privacy requests in one single
+        # query and then process them in-memory. This could be more efficient.
+        def privacy_request_has_errored_tasks(
+            privacy_request: PrivacyRequest, task_type: ActionType
+        ) -> bool:
+            """Check if a privacy request has exited all its tasks of the given type,
+            and at least one of the tasks has errored.
+            We specifically only query for the RequestTask.status column to avoid
+            querying for the entire RequestTask row.
+            """
+            tasks_statuses_query = db.query(RequestTask.status).filter(
+                RequestTask.privacy_request_id == privacy_request.id,
+                RequestTask.action_type == task_type,
+            )
+
+            statuses = set(status for status, in tasks_statuses_query.all())
             all_exited = all(
                 status in EXITED_EXECUTION_LOG_STATUSES for status in statuses
             )
-            return all_exited and ExecutionLogStatus.error in statuses
+            if all_exited and ExecutionLogStatus.error in statuses:
+                logger.info(
+                    f"Marking {task_type.value} step of {privacy_request.id} as error"
+                )
+                return True
+
+            return False
 
         marked_as_errored: Set[str] = set()
         for pr in in_progress_privacy_requests.all():
             if pr.consent_tasks.count():
                 # Consent propagation tasks - these are not created until access and erasure steps are complete.
-                if some_errored(pr.consent_tasks):
-                    logger.info(f"Marking consent step of {pr.id} as error")
+                if privacy_request_has_errored_tasks(pr, ActionType.consent):
                     pr.error_processing(db)
                     marked_as_errored.add(pr.id)
 
             if pr.erasure_tasks.count():
                 # Erasure tasks are created at the same time as access tasks but if any are errored, this means
                 # we made it to the erasure section
-                if some_errored(pr.erasure_tasks):
-                    logger.info(f"Marking erasure step of {pr.id} as error")
+                if privacy_request_has_errored_tasks(pr, ActionType.erasure):
                     pr.error_processing(db)
                     marked_as_errored.add(pr.id)
 
             if pr.access_tasks.count():
-                if some_errored(pr.access_tasks):
-                    logger.info(f"Marking access step of {pr.id} as error")
+                if privacy_request_has_errored_tasks(pr, ActionType.access):
                     pr.error_processing(db)
                     marked_as_errored.add(pr.id)
 
@@ -347,6 +369,50 @@ def initiate_interrupted_task_requeue_poll() -> None:
     )
 
 
+def get_cached_task_id(entity_id: str) -> Optional[str]:
+    """Gets the cached task ID for a privacy request or request task by ID."""
+    cache: FidesopsRedis = get_cache()
+    task_id = cache.get(get_async_task_tracking_cache_key(entity_id))
+    return task_id
+
+
+REQUEUE_INTERRUPTED_TASKS_LOCK = "requeue_interrupted_tasks_lock"
+
+
+def _get_task_ids_from_dsr_queue(
+    redis_client: FidesopsRedis, chunk_size: int = 100
+) -> Set[str]:
+    """
+    Get all task IDs from a Redis queue in a memory-efficient way.
+
+    Args:
+        redis_client: Redis client
+        chunk_size: Size of chunks to process
+
+    Returns:
+        Set of task IDs
+    """
+    queued_tasks_ids = set()
+    queue_length = redis_client.llen(DSR_QUEUE_NAME)
+
+    # Process the queue in chunks to avoid memory issues with large queues
+    for offset in range(0, queue_length, chunk_size):
+        # Get a chunk of tasks (at most chunk_size elements)
+        tasks_chunk = redis_client.lrange(
+            DSR_QUEUE_NAME, offset, offset + chunk_size - 1
+        )
+
+        # Extract task IDs from the chunk
+        for task in tasks_chunk:
+            try:
+                queued_tasks_ids.add(json.loads(task)["headers"]["id"])
+            except (json.JSONDecodeError, KeyError) as e:
+                logger.warning(f"Failed to parse task from queue: {e}")
+                continue
+
+    return queued_tasks_ids
+
+
 # pylint: disable=too-many-branches
 @celery_app.task(base=DatabaseTask, bind=True)
 def requeue_interrupted_tasks(self: DatabaseTask) -> None:
@@ -364,12 +430,11 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
     3. Checks each privacy request to determine if its tasks are still active.
     4. Requeues the privacy request if any of its tasks are found to be interrupted.
     """
-
     redis_conn: FidesopsRedis = get_cache()
 
     # Create a lock with a timeout of 10 minutes (600 seconds)
     # This ensures the lock is eventually released even if the process crashes
-    lock = redis_conn.lock("requeue_interrupted_tasks_lock", timeout=600)
+    lock = redis_conn.lock(REQUEUE_INTERRUPTED_TASKS_LOCK, timeout=600)
 
     # Try to acquire the lock without blocking
     if not lock.acquire(blocking=False):
@@ -406,30 +471,14 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
             )
 
             # Get task IDs from the queue in a memory-efficient way
-            queued_tasks_ids = set()
-            chunk_size = 100
-            queue_length = redis_conn.llen("fides.dsr")
-
-            # Process the queue in chunks to avoid memory issues with large queues
-            for offset in range(0, queue_length, chunk_size):
-                # Get a chunk of tasks (at most chunk_size elements)
-                tasks_chunk = redis_conn.lrange(
-                    "fides.dsr", offset, offset + chunk_size - 1
-                )
-
-                # Extract task IDs from the chunk
-                for task in tasks_chunk:
-                    try:
-                        queued_tasks_ids.add(json.loads(task)["headers"]["id"])
-                    except (json.JSONDecodeError, KeyError) as e:
-                        logger.warning(f"Failed to parse task from queue: {e}")
-                        continue
+            queued_tasks_ids = _get_task_ids_from_dsr_queue(redis_conn)
 
             # Check each privacy request
             for privacy_request in in_progress_requests:
                 should_requeue = False
                 logger.debug(f"Checking tasks for privacy request {privacy_request.id}")
-                task_id = privacy_request.get_cached_task_id()
+
+                task_id = get_cached_task_id(privacy_request.id)
 
                 # If the task ID is not cached, we can't check if it's running
                 if not task_id:
@@ -439,33 +488,51 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                 if task_id not in queued_tasks_ids and not celery_tasks_in_flight(
                     [task_id]
                 ):
-                    if privacy_request.request_tasks.count() == 0:
+                    request_tasks_count = (
+                        db.query(RequestTask)
+                        .filter(RequestTask.privacy_request_id == privacy_request.id)
+                        .count()
+                    )
+                    if request_tasks_count == 0:
                         logger.warning(
                             f"The task for privacy request {privacy_request.id} was terminated before it could schedule any request tasks, requeueing privacy request"
                         )
                         should_requeue = True
 
+                    request_tasks_in_progress = (
+                        db.query(RequestTask.id)
+                        .filter(RequestTask.privacy_request_id == privacy_request.id)
+                        .filter(
+                            RequestTask.status.in_(
+                                [
+                                    ExecutionLogStatus.in_processing,
+                                    ExecutionLogStatus.pending,
+                                ]
+                            )
+                        )
+                        .all()
+                    )
+                    request_task_ids_in_progress = [
+                        task[0] for task in request_tasks_in_progress
+                    ]
+
                     # Check each individual request task
-                    for request_task in privacy_request.request_tasks:
-                        if request_task.status in [
-                            ExecutionLogStatus.in_processing,
-                            ExecutionLogStatus.pending,
-                        ]:
-                            subtask_id = request_task.get_cached_task_id()
-
-                            # If the task ID is not cached, we can't check if it's running
-                            if not subtask_id:
-                                continue
-
-                            if (
-                                subtask_id not in queued_tasks_ids
-                                and not celery_tasks_in_flight([subtask_id])
-                            ):
-                                logger.warning(
-                                    f"Request task {request_task.id} is not in the queue or running, requeueing privacy request"
-                                )
-                                should_requeue = True
-                                break
+                    for request_task_id in request_task_ids_in_progress:
+                        subtask_id = get_cached_task_id(request_task_id)
+
+                        # If the task ID is not cached, we can't check if it's running
+                        if not subtask_id:
+                            continue
+
+                        if (
+                            subtask_id not in queued_tasks_ids
+                            and not celery_tasks_in_flight([subtask_id])
+                        ):
+                            logger.warning(
+                                f"Request task {request_task_id} is not in the queue or running, requeueing privacy request"
+                            )
+                            should_requeue = True
+                            break
 
                 # Requeue the privacy request if needed
                 if should_requeue: