ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/service/privacy_request/request_runner_service.py

@@ -2,12 +2,15 @@
 import time
 from copy import deepcopy
 from datetime import datetime, timedelta
-from typing import Any, Optional, Tuple
+from typing import Any, Optional
 
-import requests
+import sqlalchemy.exc
 from loguru import logger
+
+# pylint: disable=no-name-in-module
+from psycopg2.errors import InternalError_  # type: ignore[import-untyped]
 from pydantic import ValidationError as PydanticValidationError
-from sqlalchemy.orm import Query, Session
+from sqlalchemy.orm import Query, Session, selectinload
 
 from fides.api import common_exceptions
 from fides.api.common_exceptions import (
@@ -22,7 +25,6 @@ from fides.api.common_exceptions import (
     ValidationError,
 )
 from fides.api.db.session import get_db_session
-from fides.api.graph.config import CollectionAddress
 from fides.api.graph.graph import DatasetGraph
 from fides.api.models.attachment import Attachment, AttachmentReferenceType
 from fides.api.models.audit_log import AuditLog, AuditLogAction
@@ -53,7 +55,7 @@ from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.schemas.redis_cache import Identity
 from fides.api.schemas.storage.storage import StorageType
-from fides.api.service.connectors import FidesConnector, get_connector
+from fides.api.service.connectors import get_connector
 from fides.api.service.connectors.consent_email_connector import (
     CONSENT_EMAIL_CONNECTOR_TYPES,
 )
@@ -69,6 +71,7 @@ from fides.api.service.privacy_request.attachment_handling import (
     get_attachments_content,
     process_attachments_for_upload,
 )
+from fides.api.service.privacy_request.duplication_detection import check_for_duplicates
 from fides.api.service.storage.storage_uploader_service import upload
 from fides.api.task.filter_results import filter_data_categories
 from fides.api.task.graph_runners import access_runner, consent_runner, erasure_runner
@@ -85,11 +88,7 @@ from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
 from fides.api.util.memory_watchdog import memory_limiter
-from fides.common.api.v1.urn_registry import (
-    PRIVACY_CENTER_DSR_PACKAGE,
-    PRIVACY_REQUEST_TRANSFER_TO_PARENT,
-    V1_URL_PREFIX,
-)
+from fides.common.api.v1.urn_registry import PRIVACY_CENTER_DSR_PACKAGE
 from fides.config import CONFIG
 from fides.config.config_proxy import ConfigProxy
 
@@ -261,14 +260,14 @@ def upload_access_results(
             "Error uploading subject access data for rule {} on policy {}: {}",
             rule.key,
             policy.key,
-            Pii(str(exc)),
+            str(exc),
         )
         privacy_request.add_error_execution_log(
             session,
             connection_key=None,
             dataset_name="Access package upload",
             collection_name=None,
-            message="Access package upload failed for privacy request.",
+            message=f"Access package upload failed for privacy request: {str(exc)}",
             action_type=ActionType.access,
         )
         privacy_request.status = PrivacyRequestStatus.error
@@ -276,6 +275,11 @@ def upload_access_results(
     return download_urls
 
 
+@log_context(
+    capture_args={
+        "privacy_request_id": LoggerContextKeys.privacy_request_id,
+    }
+)
 def save_access_results(
     session: Session,
     privacy_request: PrivacyRequest,
@@ -284,12 +288,34 @@ def save_access_results(
 ) -> None:
     """Save the results we uploaded to the user for later retrieval"""
     # Save the results we uploaded to the user for later retrieval
-    privacy_request.save_filtered_access_results(session, rule_filtered_results)
     # Saving access request URL's on the privacy request in case DSR 3.0
     # exits processing before the email is sent
     privacy_request.access_result_urls = {"access_result_urls": download_urls}
     privacy_request.save(session)
 
+    # Try to save the backup results, but don't fail the DSR if this fails
+    try:
+        privacy_request.save_filtered_access_results(session, rule_filtered_results)
+        logger.info("Successfully saved backup filtered access results to database")
+    except (
+        InternalError_,  # invalid memory alloc request size 1073741824
+        sqlalchemy.exc.StatementError,  # SQL statement errors
+        # Python memory errors
+        MemoryError,  # system out of memory
+        OverflowError,  # numeric overflow during serialization
+    ) as exc:
+        logger.warning(
+            "Failed to save backup of DSR results to database after successful S3 upload. "
+            "DSR will continue processing. Error: {}",
+            str(exc),
+        )
+    except Exception as exc:
+        logger.error(
+            "Failed to save backup of DSR results to database after successful S3 upload. "
+            "DSR will continue processing. Unexpected Error: {}",
+            str(exc),
+        )
+
 
 @log_context(
     capture_args={
@@ -307,8 +333,11 @@ def upload_and_save_access_results(  # pylint: disable=R0912
 ) -> list[str]:
     """Process the data uploads after the access portion of the privacy request has completed"""
     download_urls: list[str] = []
-    # Remove manual webhook attachments from the list of attachments
-    # This is done because the manual webhook attachments are already included in the manual_data
+    # Remove manual webhook attachments and request task attachments from the list of attachments
+    # This is done because:
+    # - manual webhook attachments are already included in the manual_data
+    # - manual task submission attachments are already included in the manual_data
+    # - request task attachments (from async polling) are already embedded in the dataset results
     loaded_attachments = [
         attachment
         for attachment in privacy_request.attachments
@@ -317,6 +346,7 @@ def upload_and_save_access_results(  # pylint: disable=R0912
             in [
                 AttachmentReferenceType.access_manual_webhook,
                 AttachmentReferenceType.manual_task_submission,
+                AttachmentReferenceType.request_task,
             ]
             for ref in attachment.references
         )
@@ -397,7 +427,11 @@ def run_privacy_request(
             logger.info("Resuming privacy request from checkpoint: '{}'", from_step)
 
     with self.get_new_session() as session:
-        privacy_request = PrivacyRequest.get(db=session, object_id=privacy_request_id)
+        privacy_request = (
+            PrivacyRequest.query_without_large_columns(session)
+            .filter(PrivacyRequest.id == privacy_request_id)
+            .first()
+        )
         if not privacy_request:
             raise common_exceptions.PrivacyRequestNotFound(
                 f"Privacy request with id {privacy_request_id} not found"
@@ -417,6 +451,10 @@ def run_privacy_request(
                 logger.info("Terminating privacy request: request deleted.")
                 return
 
+            check_for_duplicates(db=session, privacy_request=privacy_request)
+            if privacy_request.status == PrivacyRequestStatus.duplicate:
+                return
+
             logger.info("Dispatching privacy request")
             privacy_request.start_processing(session)
 
@@ -460,7 +498,15 @@ def run_privacy_request(
                 )
 
             try:
-                datasets = DatasetConfig.all(db=session)
+                # Eager load connection_config and ctl_dataset to avoid N+1 queries
+                datasets = (
+                    session.query(DatasetConfig)
+                    .options(
+                        selectinload(DatasetConfig.connection_config),
+                        selectinload(DatasetConfig.ctl_dataset),
+                    )
+                    .all()
+                )
                 dataset_graphs = [
                     dataset_config.get_graph()
                     for dataset_config in datasets
@@ -487,7 +533,12 @@ def run_privacy_request(
                     key: value["value"] if isinstance(value, dict) else value
                     for key, value in privacy_request.get_cached_identity_data().items()
                 }
-                connection_configs = ConnectionConfig.all(db=session)
+                # Eager load datasets relationship to avoid N+1 queries in filter_fides_connector_datasets
+                connection_configs = (
+                    session.query(ConnectionConfig)
+                    .options(selectinload(ConnectionConfig.datasets))
+                    .all()
+                )
                 fides_connector_datasets: set[str] = filter_fides_connector_datasets(
                     connection_configs
                 )
@@ -782,10 +833,28 @@ def run_privacy_request(
                                     privacy_request.property_id,
                                     privacy_request.id,
                                 )
+                                # Add success log for completion email
+                                privacy_request.add_success_execution_log(
+                                    session,
+                                    connection_key=None,
+                                    dataset_name="Privacy request completion email",
+                                    collection_name=None,
+                                    message="Privacy request completion email sent successfully.",
+                                    action_type=privacy_request.policy.get_action_type(),  # type: ignore
+                                )
                             except (
                                 IdentityNotFoundException,
                                 MessageDispatchException,
                             ) as e:
+                                # Add error log for completion email failure
+                                privacy_request.add_error_execution_log(
+                                    session,
+                                    connection_key=None,
+                                    dataset_name="Privacy request completion email",
+                                    collection_name=None,
+                                    message=f"Privacy request completion email failed: {str(e)}",
+                                    action_type=privacy_request.policy.get_action_type(),  # type: ignore
+                                )
                                 privacy_request.error_processing(db=session)
                                 # If dev mode, log traceback
                                 _log_exception(e, CONFIG.dev_mode)
@@ -887,7 +956,11 @@ def mark_paused_privacy_request_as_expired(privacy_request_id: str) -> None:
     """Mark "paused" PrivacyRequest as "errored" after its associated identity data in the redis cache has expired."""
     SessionLocal = get_db_session(CONFIG)
     db = SessionLocal()
-    privacy_request = PrivacyRequest.get(db=db, object_id=privacy_request_id)
+    privacy_request = (
+        PrivacyRequest.query_without_large_columns(db)
+        .filter(PrivacyRequest.id == privacy_request_id)
+        .first()
+    )
     if not privacy_request:
         logger.info(
             "Attempted to mark as expired. No privacy request with id '{}' found.",
@@ -904,78 +977,6 @@ def mark_paused_privacy_request_as_expired(privacy_request_id: str) -> None:
     db.close()
 
 
-def _retrieve_child_results(  # pylint: disable=R0911
-    fides_connector: Tuple[str, ConnectionConfig],
-    rule_key: str,
-    access_result: dict[str, list[Row]],
-) -> Optional[list[dict[str, Optional[list[Row]]]]]:
-    """Get child access request results to add to upload."""
-    try:
-        connector = FidesConnector(fides_connector[1])
-    except Exception as e:
-        logger.error(
-            "Error create client for child server {}: {}", fides_connector[0], e
-        )
-        return None
-
-    results = []
-
-    for key, rows in access_result.items():
-        address = CollectionAddress.from_string(key)
-        privacy_request_id = None
-        if address.dataset == fides_connector[0]:
-            if not rows:
-                logger.info("No rows found for result entry {}", key)
-                continue
-            privacy_request_id = rows[0]["id"]
-
-        if not privacy_request_id:
-            logger.error(
-                "No privacy request found for connector key {}", fides_connector[0]
-            )
-            continue
-
-        try:
-            client = connector.create_client()
-        except requests.exceptions.HTTPError as e:
-            logger.error(
-                "Error logger into to child server for privacy request {}: {}",
-                privacy_request_id,
-                e,
-            )
-            continue
-
-        try:
-            request = client.authenticated_request(
-                method="get",
-                path=f"{V1_URL_PREFIX}{PRIVACY_REQUEST_TRANSFER_TO_PARENT.format(privacy_request_id=privacy_request_id, rule_key=rule_key)}",
-                headers={"Authorization": f"Bearer {client.token}"},
-            )
-            response = client.session.send(request)
-        except requests.exceptions.HTTPError as e:
-            logger.error(
-                "Error retrieving data from child server for privacy request {}: {}",
-                privacy_request_id,
-                e,
-            )
-            continue
-
-        if response.status_code != 200:
-            logger.error(
-                "Error retrieving data from child server for privacy request {}: {}",
-                privacy_request_id,
-                response.json(),
-            )
-            continue
-
-        results.append(response.json())
-
-    if not results:
-        return None
-
-    return results
-
-
 def get_consent_email_connection_configs(db: Session) -> Query:
     """Return enabled consent email connection configs."""
     return db.query(ConnectionConfig).filter(

fides/api/service/privacy_request/request_service.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import json
-from asyncio import sleep
 from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional, Set
 
@@ -11,7 +10,7 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 from sqlalchemy.sql.elements import TextClause
 
-from fides.api.common_exceptions import PrivacyRequestError, PrivacyRequestNotFound
+from fides.api.common_exceptions import PrivacyRequestError
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
     PrivacyRequest,
@@ -21,10 +20,7 @@ from fides.api.models.privacy_request.request_task import AsyncTaskType
 from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.policy import ActionType
-from fides.api.schemas.privacy_request import (
-    PrivacyRequestResponse,
-    PrivacyRequestStatus,
-)
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.schemas.redis_cache import Identity
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
@@ -38,13 +34,17 @@ from fides.api.util.cache import (
     reset_privacy_request_retry_count,
 )
 from fides.api.util.lock import redis_lock
-from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS, V1_URL_PREFIX
 from fides.config import CONFIG
 
 PRIVACY_REQUEST_STATUS_CHANGE_POLL = "privacy_request_status_change_poll"
 DSR_DATA_REMOVAL = "dsr_data_removal"
 INTERRUPTED_TASK_REQUEUE_POLL = "interrupted_task_requeue_poll"
+REQUEUE_INTERRUPTED_TASKS_LOCK = "requeue_interrupted_tasks_lock"
 ASYNC_TASKS_STATUS_POLLING = "async_tasks_status_polling"
+ASYNC_TASKS_STATUS_POLLING_LOCK = "async_tasks_status_polling_lock"
+ASYNC_TASKS_STATUS_POLLING_LOCK_TIMEOUT = (
+    300  # Starting timeout is shorter because the task goes directly to the workers.
+)
 
 
 def build_required_privacy_request_kwargs(
@@ -93,62 +93,6 @@ def get_async_client() -> AsyncClient:
     return AsyncClient()
 
 
-async def poll_server_for_completion(
-    privacy_request_id: str,
-    server_url: str,
-    token: str,
-    *,
-    poll_interval_seconds: int = 30,
-    timeout_seconds: int = 1800,  # 30 minutes
-    client: AsyncClient | None = None,
-) -> PrivacyRequestResponse:
-    """Poll a server for privacy request completion.
-
-    Requests will report complete with if they have a status of canceled, complete,
-    denied, or error. By default the polling will time out if not completed in 30
-    minutes, time can be overridden by setting the timeout_seconds.
-    """
-    url = (
-        f"{server_url}{V1_URL_PREFIX}{PRIVACY_REQUESTS}?request_id={privacy_request_id}"
-    )
-    start_time = datetime.now()
-    elapsed_time = 0.0
-    while elapsed_time < timeout_seconds:
-        if client:
-            response = await client.get(
-                url, headers={"Authorization": f"Bearer {token}"}
-            )
-        else:
-            async_client = get_async_client()
-            response = await async_client.get(
-                url, headers={"Authorization": f"Bearer {token}"}
-            )
-        response.raise_for_status()
-
-        # Privacy requests are returned paginated. Since this is searching for a specific
-        # privacy request there should only be one value present in items.
-        items = response.json()["items"]
-        if not items:
-            raise PrivacyRequestNotFound(
-                f"No privacy request found with id '{privacy_request_id}'"
-            )
-        status = PrivacyRequestResponse(**items[0])
-        if status.status and status.status in (
-            PrivacyRequestStatus.complete,
-            PrivacyRequestStatus.canceled,
-            PrivacyRequestStatus.error,
-            PrivacyRequestStatus.denied,
-        ):
-            return status
-
-        await sleep(poll_interval_seconds)
-        time_delta = datetime.now() - start_time
-        elapsed_time = time_delta.seconds
-    raise TimeoutError(
-        f"Timeout of {timeout_seconds} seconds has been exceeded while waiting for privacy request {privacy_request_id}"
-    )
-
-
 def initiate_poll_for_exited_privacy_request_tasks() -> None:
     """Initiates scheduler to check if a Privacy Request's status needs to be flipped when all
     Request Tasks have had a chance to run"""
@@ -189,8 +133,9 @@ def poll_for_exited_privacy_request_tasks(self: DatabaseTask) -> Set[str]:
         # Privacy Requests that didn't need approval are "In Processing".
         # Privacy Requests in these states should be examined to see if all of its Request Tasks have had a chance
         # to complete.
+        # Use query_without_large_columns to prevent OOM errors when processing many privacy requests
         in_progress_privacy_requests = (
-            db.query(PrivacyRequest)
+            PrivacyRequest.query_without_large_columns(db)
             .filter(
                 PrivacyRequest.status.in_(
                     [
@@ -340,7 +285,7 @@ def remove_saved_dsr_data(self: DatabaseTask) -> None:
 def initiate_interrupted_task_requeue_poll() -> None:
     """Initiates scheduler to check for and requeue interrupted tasks"""
 
-    if CONFIG.test_mode or not CONFIG.execution.use_dsr_3_0:
+    if CONFIG.test_mode:
         return
 
     assert (
@@ -359,7 +304,7 @@ def initiate_interrupted_task_requeue_poll() -> None:
     )
 
 
-def initiate_async_tasks_status_polling() -> None:
+def initiate_polling_task_requeue() -> None:
     """Initiates scheduler to check for and requeue pending polling async tasks"""
     if CONFIG.test_mode:
         return
@@ -370,13 +315,13 @@ def initiate_async_tasks_status_polling() -> None:
 
     logger.info("Initiating scheduler for async tasks status polling")
     scheduler.add_job(
-        func=poll_async_tasks_status,
+        func=requeue_polling_tasks,
         trigger="interval",
         kwargs={},
         id=ASYNC_TASKS_STATUS_POLLING,
         coalesce=True,
         replace_existing=True,
-        seconds=CONFIG.execution.async_tasks_status_polling_interval_seconds,
+        seconds=CONFIG.execution.async_polling_interval_hours * 3600,
     )
 
 
@@ -394,9 +339,6 @@ def get_cached_task_id(entity_id: str) -> Optional[str]:
         raise
 
 
-REQUEUE_INTERRUPTED_TASKS_LOCK = "requeue_interrupted_tasks_lock"
-
-
 def _get_task_ids_from_dsr_queue(
     redis_client: FidesopsRedis, chunk_size: int = 100
 ) -> Set[str]:
@@ -539,6 +481,27 @@ def _get_request_task_ids_in_progress(
     return [task[0] for task in request_tasks_in_progress]
 
 
+def _has_polling_tasks(db: Session, privacy_request_id: str) -> bool:
+    """
+    Check if a privacy request has any polling async tasks.
+
+    Args:
+        db: Database session
+        privacy_request_id: The ID of the privacy request to check
+
+    Returns:
+        bool: True if the privacy request has polling tasks, False otherwise
+    """
+    return db.query(
+        db.query(RequestTask)
+        .filter(
+            RequestTask.privacy_request_id == privacy_request_id,
+            RequestTask.async_type == AsyncTaskType.polling,
+        )
+        .exists()
+    ).scalar()
+
+
 # pylint: disable=too-many-branches
 # pylint: disable=too-many-statements
 @celery_app.task(base=DatabaseTask, bind=True)
@@ -566,8 +529,9 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
             logger.debug("Starting check for interrupted tasks to requeue")
 
             # Get all in-progress privacy requests that haven't been updated in the last 5 minutes
+            # Use query_without_large_columns to prevent OOM errors when processing many privacy requests
             in_progress_requests = (
-                db.query(PrivacyRequest)
+                PrivacyRequest.query_without_large_columns(db)
                 .filter(
                     PrivacyRequest.status.in_(
                         [
@@ -678,6 +642,17 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                                 should_requeue = False
                                 break
 
+                            # Check if the Privacy request has polling tasks
+                            if _has_polling_tasks(db, privacy_request.id):
+                                # If the polling request task has no cached task ID, it's stuck
+                                logger.warning(
+                                    f"No task ID found for request task {request_task_id} "
+                                    f"(privacy request {privacy_request.id}) Contains polling tasks - "
+                                    f"keeping request in current status as it may be waiting for polling task to complete"
+                                )
+                                should_requeue = False
+                                break
+
                             # For other statuses, cancel the entire privacy request
                             _cancel_interrupted_tasks_and_error_privacy_request(
                                 db,
@@ -704,26 +679,35 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
-def poll_async_tasks_status(self: DatabaseTask) -> None:
+def requeue_polling_tasks(self: DatabaseTask) -> None:
     """
     Poll the status of async tasks that are awaiting processing.
     """
-
-    with self.get_new_session() as db:
-        logger.debug("Polling for async tasks status")
-
-        # Get all tasks that are awaiting processing and are from polling async tasks
-        async_tasks = (
-            db.query(RequestTask)
-            .filter(RequestTask.status == ExecutionLogStatus.awaiting_processing)
-            .filter(RequestTask.async_type == AsyncTaskType.polling)
-            .all()
-        )
-        if async_tasks:
-            logger.debug(f"Found {len(async_tasks)} async tasks to poll")
-            from fides.api.service.async_dsr.async_dsr_service import (  # pylint: disable=cyclic-import
-                requeue_polling_request,
+    with redis_lock(
+        ASYNC_TASKS_STATUS_POLLING_LOCK,
+        ASYNC_TASKS_STATUS_POLLING_LOCK_TIMEOUT,
+    ) as lock:
+        if not lock:
+            logger.debug("Async tasks status polling lock not acquired, skipping")
+            return
+        with self.get_new_session() as db:
+            logger.debug("Polling for async tasks status")
+
+            # Get all tasks that are polling and are from polling async tasks
+            async_tasks = (
+                db.query(RequestTask)
+                .filter(RequestTask.status == ExecutionLogStatus.polling)
+                .filter(RequestTask.async_type == AsyncTaskType.polling)
+                .all()
             )
+            logger.info(f"Found {len(async_tasks)} async polling tasks")
 
-            for async_task in async_tasks:
-                requeue_polling_request(db, async_task)
+            if async_tasks:
+                # Avoiding cyclic imports
+                from fides.api.task.execute_request_tasks import queue_request_task
+
+                for async_task in async_tasks:
+                    logger.info(
+                        f"Requeuing polling task {async_task.id} for processing"
+                    )
+                    queue_request_task(async_task, privacy_request_proceed=True)