ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.71.1rc1__py2.py3-none-any.whl

fides/api/service/async_dsr/utils.py

@@ -0,0 +1,130 @@
+"""
+Pure utility functions for async DSR operations.
+
+This module contains utility functions with no business logic dependencies.
+These are helper functions that can be used across the async DSR system.
+"""
+
+from enum import Enum
+
+# Type checking imports
+from typing import TYPE_CHECKING, List, cast
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.common_exceptions import PrivacyRequestError
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.privacy_request.request_task import AsyncTaskType, RequestTask
+from fides.api.schemas.saas.saas_config import SaaSRequest
+
+if TYPE_CHECKING:
+    from fides.api.service.connectors.query_configs.saas_query_config import (
+        SaaSQueryConfig,
+    )
+
+
+def has_async_requests(query_config: "SaaSQueryConfig") -> bool:
+    """Check if any read/update/delete requests have async configuration"""
+    all_requests: List[SaaSRequest] = cast(
+        List[SaaSRequest], query_config.get_read_requests_by_identity()
+    )
+    masking_request = query_config.get_masking_request()
+    if masking_request:
+        all_requests.append(masking_request)
+    return any(request.async_config is not None for request in all_requests)
+
+
+def is_polling_continuation(request_task: RequestTask) -> bool:
+    """Check if this is a polling continuation (not initial request)"""
+    async_type_check = request_task.async_type == AsyncTaskType.polling
+    sub_requests_count = len(request_task.sub_requests)
+
+    logger.warning(
+        f"is_polling_continuation for task {request_task.id}: "
+        f"async_type={request_task.async_type} (polling={async_type_check}), "
+        f"sub_requests_count={sub_requests_count}"
+    )
+
+    return async_type_check and sub_requests_count > 0
+
+
+def is_callback_completion(request_task: RequestTask) -> bool:
+    """Check if this is a completed callback request"""
+    return bool(request_task.callback_succeeded)
+
+
+def is_async_request(
+    request_task: RequestTask, query_config: "SaaSQueryConfig"
+) -> bool:
+    """
+    Check if this is an async request (callback, continuation, or initial async).
+
+    This is the main entry point for async detection.
+    """
+    return (
+        is_callback_completion(request_task)
+        or is_polling_continuation(request_task)
+        or has_async_requests(query_config)
+    )
+
+
+class AsyncPhase(Enum):
+    """Enum representing different phases of async DSR processing."""
+
+    callback_completion = "callback_completion"
+    polling_continuation = "polling_continuation"
+    initial_async = "initial_async"
+    sync = "sync"
+
+
+def get_async_phase(
+    request_task: RequestTask, query_config: "SaaSQueryConfig"
+) -> AsyncPhase:
+    """
+    Classify the phase of async request for routing purposes.
+
+    Returns:
+        AsyncPhase enum value representing the current phase:
+        - callback_completion: Callback has completed
+        - polling_continuation: Continuing an existing polling process
+        - initial_async: Initial async request setup
+        - sync: Not an async request
+    """
+    if is_callback_completion(request_task):
+        return AsyncPhase.callback_completion
+    if is_polling_continuation(request_task):
+        return AsyncPhase.polling_continuation
+    if has_async_requests(query_config):
+        return AsyncPhase.initial_async
+    return AsyncPhase.sync
+
+
+def get_connection_config_from_task(
+    db: Session, request_task: RequestTask
+) -> ConnectionConfig:
+    """
+    Get ConnectionConfig from a RequestTask.
+
+    This utility function retrieves the connection configuration
+    associated with a request task by looking up the dataset configuration.
+    """
+    dataset_config = DatasetConfig.filter(
+        db=db,
+        conditions=(DatasetConfig.fides_key == request_task.dataset_name),
+    ).first()
+    if not dataset_config:
+        raise PrivacyRequestError(
+            f"DatasetConfig with fides_key {request_task.dataset_name} not found."
+        )
+
+    connection_config = ConnectionConfig.get(
+        db=db, object_id=dataset_config.connection_config_id
+    )
+    if not connection_config:
+        raise PrivacyRequestError(
+            f"ConnectionConfig with id {dataset_config.connection_config_id} not found."
+        )
+
+    return connection_config

fides/api/service/connectors/fides/fides_client.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+from asyncio import sleep
+from datetime import datetime
 from typing import Any, Dict, List, Optional
 
 import httpx
 from httpx import AsyncClient, Client, HTTPStatusError, Request, RequestError, Timeout
 from loguru import logger
 
+from fides.api.common_exceptions import PrivacyRequestNotFound
 from fides.api.schemas.privacy_request import (
     PrivacyRequestCreate,
     PrivacyRequestResponse,
@@ -13,11 +16,11 @@ from fides.api.schemas.privacy_request import (
 )
 from fides.api.schemas.redis_cache import Identity
 from fides.api.schemas.user import UserLogin
-from fides.api.service.privacy_request.request_service import poll_server_for_completion
 from fides.api.util.collection_util import Row
 from fides.api.util.errors import FidesError
 from fides.api.util.wrappers import sync
 from fides.common.api.v1 import urn_registry as urls
+from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS
 
 COMPLETION_STATUSES = [
     PrivacyRequestStatus.complete,
@@ -27,6 +30,65 @@ COMPLETION_STATUSES = [
 ]
 
 
+def get_async_client() -> AsyncClient:
+    """Return an async client used to make API requests"""
+    return AsyncClient()
+
+
+async def poll_server_for_completion(
+    privacy_request_id: str,
+    server_url: str,
+    token: str,
+    *,
+    poll_interval_seconds: int = 30,
+    timeout_seconds: int = 1800,  # 30 minutes
+    client: AsyncClient | None = None,
+) -> PrivacyRequestResponse:
+    """Poll a server for privacy request completion.
+
+    Requests will report complete with if they have a status of canceled, complete,
+    denied, or error. By default the polling will time out if not completed in 30
+    minutes, time can be overridden by setting the timeout_seconds.
+    """
+    url = f"{server_url}{urls.V1_URL_PREFIX}{PRIVACY_REQUESTS}?request_id={privacy_request_id}"
+    start_time = datetime.now()
+    elapsed_time = 0.0
+    while elapsed_time < timeout_seconds:
+        if client:
+            response = await client.get(
+                url, headers={"Authorization": f"Bearer {token}"}
+            )
+        else:
+            async_client = get_async_client()
+            response = await async_client.get(
+                url, headers={"Authorization": f"Bearer {token}"}
+            )
+        response.raise_for_status()
+
+        # Privacy requests are returned paginated. Since this is searching for a specific
+        # privacy request there should only be one value present in items.
+        items = response.json()["items"]
+        if not items:
+            raise PrivacyRequestNotFound(
+                f"No privacy request found with id '{privacy_request_id}'"
+            )
+        status = PrivacyRequestResponse(**items[0])
+        if status.status and status.status in (
+            PrivacyRequestStatus.complete,
+            PrivacyRequestStatus.canceled,
+            PrivacyRequestStatus.error,
+            PrivacyRequestStatus.denied,
+        ):
+            return status
+
+        await sleep(poll_interval_seconds)
+        time_delta = datetime.now() - start_time
+        elapsed_time = time_delta.seconds
+    raise TimeoutError(
+        f"Timeout of {timeout_seconds} seconds has been exceeded while waiting for privacy request {privacy_request_id}"
+    )
+
+
 class FidesClient:
     """
     A helper client to broker communications between Fides servers.

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -10,7 +10,6 @@ from uuid import uuid4
 import pydash
 from fideslang.models import FidesDatasetReference
 from loguru import logger
-from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import FidesopsException
 from fides.api.graph.execution import ExecutionNode
@@ -140,7 +139,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             )
         return request
 
-    def get_masking_request(self, db: Session) -> Optional[SaaSRequest]:
+    def get_masking_request(self) -> Optional[SaaSRequest]:
         """
         Returns a tuple of the preferred action and SaaSRequest to use for masking.
         An update request is preferred, but we can use a gdpr delete endpoint or
@@ -353,7 +352,8 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             ]
 
         param_values[UUID] = str(uuid4())
-        param_values[ISO_8601_DATETIME] = datetime.now().date().isoformat()
+        # Use full ISO-8601 timestamp including time component (UTC, seconds precision)
+        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
         param_values[FIELD_LIST] = ",".join(
             [
                 field.name
@@ -384,8 +384,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         The fields in the row are masked according to the policy and added to the request body
         if specified by the body field of the masking request.
         """
-        session = Session.object_session(request)
-        current_request: SaaSRequest = self.get_masking_request(session)  # type: ignore
+        current_request: SaaSRequest = self.get_masking_request()  # type: ignore
         param_values: Dict[str, Any] = self.generate_update_param_values(
             row, policy, request, current_request
         )

fides/api/service/connectors/saas_connector.py

@@ -10,18 +10,16 @@ from requests import Response
 from sqlalchemy.orm import Session
 from starlette.status import HTTP_204_NO_CONTENT
 
+from fides.api.api.deps import get_autoclose_db_session as get_db
 from fides.api.common_exceptions import (
-    AwaitingAsyncTask,
     FidesopsException,
     PostProcessingException,
-    PrivacyRequestError,
     SkippingConsentPropagation,
 )
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
-from fides.api.models.privacy_request.request_task import AsyncTaskType
 from fides.api.schemas.consentable_item import (
     ConsentableItem,
     build_consent_item_hierarchy,
@@ -39,6 +37,10 @@ from fides.api.schemas.saas.shared_schemas import (
     ConsentPropagationStatus,
     SaaSRequestParams,
 )
+from fides.api.service.async_dsr.strategies.async_dsr_strategy import AsyncDSRStrategy
+from fides.api.service.async_dsr.strategies.async_dsr_strategy_factory import (
+    get_strategy,
+)
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.service.connectors.query_configs.saas_query_config import SaaSQueryConfig
 from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
@@ -216,19 +218,29 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         request_task: RequestTask,
         input_data: Dict[str, List[Any]],
     ) -> List[Row]:
-        """Retrieve data from SaaS APIs"""
+        """
+        Retrieve data from SaaS APIs.
+
+        Handles sync requests directly and delegates async requests to external handlers.
+        """
 
         # pylint: disable=too-many-branches
         self.set_privacy_request_state(privacy_request, node, request_task)
-        if request_task.callback_succeeded:
-            # If this is True, we assume we've received results from a third party
-            # asynchronously and we can proceed to the next node.
-            logger.info(
-                "Access callback succeeded for request task '{}'", request_task.id
-            )
-            return request_task.get_access_data()
+
         query_config: SaaSQueryConfig = self.query_config(node)
 
+        # Delegate async requests
+        with get_db() as db:
+            if async_dsr_strategy := _get_async_dsr_strategy(
+                db, request_task, query_config, ActionType.access
+            ):
+                return async_dsr_strategy.async_retrieve_data(
+                    client=self.create_client(),
+                    request_task_id=request_task.id,
+                    query_config=query_config,
+                    input_data=input_data,
+                )
+
         # generate initial set of requests if read request is defined, otherwise raise an exception
         # An endpoint can be defined with multiple 'read' requests if the data for a single
         # collection can be accessed in multiple ways for example:
@@ -265,30 +277,8 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             input_data[CUSTOM_PRIVACY_REQUEST_FIELDS] = [custom_privacy_request_fields]
 
         rows: List[Row] = []
-        awaiting_async_processing: bool = False
-
         for read_request in read_requests:
             self.set_saas_request_state(read_request)
-            if (
-                read_request.async_config is not None
-                and request_task.id  # Only supported in DSR 3.0
-            ):
-                # Asynchronous read request detected. We will exit below and put the
-                # Request Task in an "awaiting_processing" status.
-                awaiting_async_processing = True
-
-                # Validate async strategy with proper enum value checking
-                strategy_value = read_request.async_config.strategy
-                valid_strategies = [task_type.value for task_type in AsyncTaskType]
-
-                if strategy_value not in valid_strategies:
-                    raise PrivacyRequestError(
-                        f"Invalid async type '{strategy_value}' for request task {request_task.id}. "
-                        f"Valid types are: {valid_strategies}"
-                    )
-
-                request_task.async_type = AsyncTaskType(strategy_value)
-
             # check all the values specified by param_values are provided in input_data
             if self._missing_dataset_reference_values(
                 input_data, read_request.param_values
@@ -334,7 +324,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             # This allows us to build an output object even if we didn't generate and execute
             # any HTTP requests. This is useful if we just want to select specific input_data
             # values to provide as row data to the mask_data function
-            elif read_request.output:
+            elif not read_request.path and read_request.output:
                 rows.extend(
                     self._apply_output_template(
                         query_config.generate_param_value_maps(
@@ -345,17 +335,6 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                 )
 
         self.unset_connector_state()
-        if awaiting_async_processing:
-            # If a read request was marked to expect async results, original response data here is ignored.
-            # We'll instead use the data received in the callback URL later.
-            # However for polling async request we want to save the request data for ids that we will use on the pollings status
-            if request_task.async_type == AsyncTaskType.polling:
-                # Saving the request task access data to use it on the polling status request
-                # TODO: Consider if we want to clean up the rows. Currently this is the concern of the GraphTask.
-                request_task.access_data = rows
-            # Raising an AwaitingAsyncTask to put this task in an awaiting_processing state
-            raise AwaitingAsyncTask()
-
         return rows
 
     def _apply_output_template(
@@ -536,22 +515,28 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         rows: List[Row],
         input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
-        """Execute a masking request. Return the number of rows that have been updated."""
-        self.set_privacy_request_state(privacy_request, node, request_task)
-        if request_task.callback_succeeded:
-            # If this is True, we assume the data was masked
-            # asynchronously and we can proceed to the next node.
-            logger.info(
-                "Masking callback succeeded for request task '{}'", request_task.id
-            )
-            # If we've received the callback for this node, return rows_masked directly
-            return request_task.rows_masked or 0
+        """
+        Execute masking requests.
 
+        Handles synchronous requests directly and delegates async requests to external handlers.
+        """
         self.set_privacy_request_state(privacy_request, node, request_task)
+
         query_config = self.query_config(node)
 
-        session = Session.object_session(privacy_request)
-        masking_request = query_config.get_masking_request(session)
+        # Delegate async requests
+        with get_db() as db:
+            if async_dsr_strategy := _get_async_dsr_strategy(
+                db, request_task, query_config, ActionType.erasure
+            ):
+                return async_dsr_strategy.async_mask_data(
+                    client=self.create_client(),
+                    request_task_id=request_task.id,
+                    query_config=query_config,
+                    rows=rows,
+                )
+
+        masking_request = query_config.get_masking_request()
         rows_updated = 0
 
         if not masking_request:
@@ -648,18 +633,6 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                 )
 
         self.unset_connector_state()
-
-        awaiting_async_callback: bool = bool(
-            masking_request.async_config
-            and masking_request.async_config.strategy == AsyncTaskType.callback.value
-        ) and bool(
-            request_task.id
-        )  # Only supported in DSR 3.0
-        if awaiting_async_callback:
-            # Asynchronous masking request detected in saas config.
-            # If the masking request was marked to expect async results, original responses are ignored
-            # and we raise an AwaitingAsyncTask to put this task in an awaiting_processing state.
-            raise AwaitingAsyncTask()
         return rows_updated
 
     @staticmethod
@@ -1112,3 +1085,38 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             return []
 
         return consent_requests.opt_in if opt_in else consent_requests.opt_out  # type: ignore
+
+
+def _get_async_dsr_strategy(
+    session: Session,
+    request_task: RequestTask,
+    query_config: SaaSQueryConfig,
+    action_type: ActionType,
+) -> Optional[AsyncDSRStrategy]:
+    """
+    Returns the async DSR strategy if any of the read or masking requests have an async_config.
+    """
+
+    # Async processing is only supported for DSR 3.0.
+    # A request task with an ID of None is an indicator that the request is not DSR 3.0.
+    if request_task.id is None:
+        return None
+
+    if action_type == ActionType.access:
+        read_requests = query_config.get_read_requests_by_identity()
+        for request in read_requests:
+            if request.async_config is not None:
+                return get_strategy(
+                    request.async_config.strategy,
+                    session,
+                    request.async_config.configuration,
+                )
+    elif action_type == ActionType.erasure:
+        masking_request = query_config.get_masking_request()
+        if masking_request is not None and masking_request.async_config is not None:
+            return get_strategy(
+                masking_request.async_config.strategy,
+                session,
+                masking_request.async_config.configuration,
+            )
+    return None

fides/api/service/privacy_request/attachment_handling.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple
 from loguru import logger
 
 from fides.api.models.attachment import Attachment, AttachmentType
-from fides.api.schemas.storage.storage import StorageDetails
+from fides.api.schemas.storage.storage import StorageDetails, StorageType
 
 
 @dataclass
@@ -84,12 +84,19 @@ def get_attachments_content(
                 continue
 
             processed_count += 1
+
+            # Handle bucket_name differently for different storage types
+            bucket_name = None
+            if attachment.config.type in [StorageType.s3, StorageType.gcs]:
+                bucket_name = attachment.config.details[StorageDetails.BUCKET.value]
+            # For local storage, bucket_name is not needed
+
             yield AttachmentData(
                 file_name=attachment.file_name,
                 file_size=size,
                 download_url=str(url) if url else None,
                 content_type=attachment.content_type,
-                bucket_name=attachment.config.details[StorageDetails.BUCKET.value],
+                bucket_name=bucket_name or "",  # Empty string for local storage
                 file_key=attachment.file_key,
                 storage_key=attachment.storage_key,
             )

fides/api/service/privacy_request/request_runner_service.py

@@ -2,9 +2,8 @@
 import time
 from copy import deepcopy
 from datetime import datetime, timedelta
-from typing import Any, Optional, Tuple
+from typing import Any, Optional
 
-import requests
 from loguru import logger
 from pydantic import ValidationError as PydanticValidationError
 from sqlalchemy.orm import Query, Session
@@ -22,7 +21,6 @@ from fides.api.common_exceptions import (
     ValidationError,
 )
 from fides.api.db.session import get_db_session
-from fides.api.graph.config import CollectionAddress
 from fides.api.graph.graph import DatasetGraph
 from fides.api.models.attachment import Attachment, AttachmentReferenceType
 from fides.api.models.audit_log import AuditLog, AuditLogAction
@@ -53,7 +51,7 @@ from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.schemas.redis_cache import Identity
 from fides.api.schemas.storage.storage import StorageType
-from fides.api.service.connectors import FidesConnector, get_connector
+from fides.api.service.connectors import get_connector
 from fides.api.service.connectors.consent_email_connector import (
     CONSENT_EMAIL_CONNECTOR_TYPES,
 )
@@ -85,11 +83,7 @@ from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
 from fides.api.util.memory_watchdog import memory_limiter
-from fides.common.api.v1.urn_registry import (
-    PRIVACY_CENTER_DSR_PACKAGE,
-    PRIVACY_REQUEST_TRANSFER_TO_PARENT,
-    V1_URL_PREFIX,
-)
+from fides.common.api.v1.urn_registry import PRIVACY_CENTER_DSR_PACKAGE
 from fides.config import CONFIG
 from fides.config.config_proxy import ConfigProxy
 
@@ -307,8 +301,11 @@ def upload_and_save_access_results(  # pylint: disable=R0912
 ) -> list[str]:
     """Process the data uploads after the access portion of the privacy request has completed"""
     download_urls: list[str] = []
-    # Remove manual webhook attachments from the list of attachments
-    # This is done because the manual webhook attachments are already included in the manual_data
+    # Remove manual webhook attachments and request task attachments from the list of attachments
+    # This is done because:
+    # - manual webhook attachments are already included in the manual_data
+    # - manual task submission attachments are already included in the manual_data
+    # - request task attachments (from async polling) are already embedded in the dataset results
     loaded_attachments = [
         attachment
         for attachment in privacy_request.attachments
@@ -317,6 +314,7 @@ def upload_and_save_access_results(  # pylint: disable=R0912
             in [
                 AttachmentReferenceType.access_manual_webhook,
                 AttachmentReferenceType.manual_task_submission,
+                AttachmentReferenceType.request_task,
             ]
             for ref in attachment.references
         )
@@ -922,78 +920,6 @@ def mark_paused_privacy_request_as_expired(privacy_request_id: str) -> None:
     db.close()
 
 
-def _retrieve_child_results(  # pylint: disable=R0911
-    fides_connector: Tuple[str, ConnectionConfig],
-    rule_key: str,
-    access_result: dict[str, list[Row]],
-) -> Optional[list[dict[str, Optional[list[Row]]]]]:
-    """Get child access request results to add to upload."""
-    try:
-        connector = FidesConnector(fides_connector[1])
-    except Exception as e:
-        logger.error(
-            "Error create client for child server {}: {}", fides_connector[0], e
-        )
-        return None
-
-    results = []
-
-    for key, rows in access_result.items():
-        address = CollectionAddress.from_string(key)
-        privacy_request_id = None
-        if address.dataset == fides_connector[0]:
-            if not rows:
-                logger.info("No rows found for result entry {}", key)
-                continue
-            privacy_request_id = rows[0]["id"]
-
-        if not privacy_request_id:
-            logger.error(
-                "No privacy request found for connector key {}", fides_connector[0]
-            )
-            continue
-
-        try:
-            client = connector.create_client()
-        except requests.exceptions.HTTPError as e:
-            logger.error(
-                "Error logger into to child server for privacy request {}: {}",
-                privacy_request_id,
-                e,
-            )
-            continue
-
-        try:
-            request = client.authenticated_request(
-                method="get",
-                path=f"{V1_URL_PREFIX}{PRIVACY_REQUEST_TRANSFER_TO_PARENT.format(privacy_request_id=privacy_request_id, rule_key=rule_key)}",
-                headers={"Authorization": f"Bearer {client.token}"},
-            )
-            response = client.session.send(request)
-        except requests.exceptions.HTTPError as e:
-            logger.error(
-                "Error retrieving data from child server for privacy request {}: {}",
-                privacy_request_id,
-                e,
-            )
-            continue
-
-        if response.status_code != 200:
-            logger.error(
-                "Error retrieving data from child server for privacy request {}: {}",
-                privacy_request_id,
-                response.json(),
-            )
-            continue
-
-        results.append(response.json())
-
-    if not results:
-        return None
-
-    return results
-
-
 def get_consent_email_connection_configs(db: Session) -> Query:
     """Return enabled consent email connection configs."""
     return db.query(ConnectionConfig).filter(