ethyca-fides 2.68.1b2__py2.py3-none-any.whl → 2.68.1b3__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-19T14:51:50-0700",
+ "date": "2025-08-25T16:34:15-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "047dc431c1e669789ec8aa19ededd438909b8c0c",
- "version": "2.68.1b2"
+ "full-revisionid": "05d56cc8111caa00faea2dd8433a36fd4d5b7a8a",
+ "version": "2.68.1b3"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/90502bcda282_update_request_tasks_add_polling_async.py

@@ -0,0 +1,35 @@
+"""Update Request Tasks
+
+Revision ID: 90502bcda282
+Revises: 3baf42d251a6
+Create Date: 2025-08-05 16:31:45.335384
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "90502bcda282"
+down_revision = "3baf42d251a6"
+branch_labels = None
+depends_on = None
+
+async_task_type = sa.Enum("manual", "polling", "callback", name="asynctasktype")
+
+
+def upgrade():
+    async_task_type.create(op.get_bind(), checkfirst=True)
+    op.add_column(
+        "requesttask",
+        sa.Column(
+            "async_type",
+            async_task_type,
+            nullable=True,
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("requesttask", "async_type")
+    async_task_type.drop(op.get_bind(), checkfirst=True)

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -50,6 +50,7 @@ from fides.api.common_exceptions import (
     IdentityVerificationException,
     ManualWebhookFieldsUnset,
     NoCachedManualWebhookEntry,
+    PrivacyRequestError,
     TraversalError,
     ValidationError,
 )
@@ -175,7 +176,6 @@ from fides.service.dataset.dataset_config_service import (
 )
 from fides.service.messaging.messaging_service import MessagingService
 from fides.service.privacy_request.privacy_request_service import (
-    PrivacyRequestError,
     PrivacyRequestService,
     _process_privacy_request_restart,
     _requeue_privacy_request,

fides/api/common_exceptions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import List
+from typing import Dict, List, Optional
 
 from fastapi import HTTPException, status
 from starlette.status import (
@@ -175,8 +175,8 @@ class RequestTaskNotFound(BaseException):
     """Privacy Request Task Not Found"""
 
 
-class AwaitingAsyncTaskCallback(BaseException):
-    """Request Task is Awaiting Processing - Awaiting Async Task Callback"""
+class AwaitingAsyncTask(BaseException):
+    """Request Task is Awaiting Processing - Awaiting Async Task"""
 
 
 class UpstreamTasksNotReady(BaseException):
@@ -373,6 +373,15 @@ class InvalidScopeError(HTTPException):
         )
 
 
+class PrivacyRequestError(Exception):
+    """Base exception for privacy request operations."""
+
+    def __init__(self, message: str, data: Optional[Dict] = None):
+        self.message = message
+        self.data = data
+        super().__init__(message)
+
+
 class KeyOrNameAlreadyExists(Exception):
     """A resource already exists with this key or name."""
 

fides/api/models/detection_discovery/core.py

@@ -432,11 +432,17 @@ class StagedResourceAncestor(Base):
 
                     # Execute batch when it reaches the desired size
                     if len(current_batch) >= batch_size:
+                        logger.debug(
+                            f"Inserting batch of {len(current_batch)} staged resource ancestor links"
+                        )
                         db.execute(stmt_text, current_batch)
                         current_batch = []
 
         # Execute any remaining items in the final batch
         if current_batch:
+            logger.debug(
+                f"Inserting batch of {len(current_batch)} staged resource ancestor links"
+            )
             db.execute(stmt_text, current_batch)
 
 

fides/api/models/privacy_request/request_task.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from enum import Enum as EnumType
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
 from loguru import logger
@@ -15,6 +16,7 @@ from sqlalchemy_utils.types.encrypted.encrypted_type import (
 )
 
 from fides.api.db.base_class import Base, JSONTypeOverride  # type: ignore[attr-defined]
+from fides.api.db.util import EnumColumn
 from fides.api.graph.config import (
     ROOT_COLLECTION_ADDRESS,
     TERMINATOR_ADDRESS,
@@ -67,6 +69,24 @@ class TraversalDetails(FidesSchema):
         )
 
 
+class AsyncTaskType(EnumType):
+    """
+    Enum for asynchronous RequestTask types
+
+    These tasks differ from our regular tasks in that they are not finished on the same process that they are created,
+    but rather are put into a pause state and then triggered to completion by a different process
+
+    manual: A task that is manually triggered by the user
+    polling: A task that is triggered by a polling mechanism
+    callback: A task that is triggered by a callback from an external system
+
+    """
+
+    manual = "manual"
+    polling = "polling"
+    callback = "callback"
+
+
 # TODO: At some point we will refactor this model to store all task types in a common table that links to tables with specific task attributes.
 class RequestTask(WorkerTask, Base):
     """
@@ -145,6 +165,11 @@ class RequestTask(WorkerTask, Base):
 
     # For async tasks awaiting callback
     callback_succeeded = Column(Boolean)
+    # to recognize Polling async task
+    async_type = Column(
+        EnumColumn(AsyncTaskType, native_enum=True),
+        nullable=True,
+    )
 
     # Stores a serialized collection that can be transformed back into a Collection to help
     # execute the current task

fides/api/schemas/privacy_center_config.py

@@ -1,3 +1,4 @@
+from abc import ABC
 from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic import ConfigDict, Field, field_validator, model_validator
@@ -12,22 +13,10 @@ class CustomIdentity(FidesSchema):
     label: str
 
 
-class LocationIdentityField(FidesSchema):
-    """Location field configuration that extends the useful parts of CustomPrivacyRequestField"""
-
-    label: str
-    required: Optional[bool] = True
-    default_value: Optional[str] = None
-    query_param_key: Optional[str] = None
-    ip_geolocation_hint: Optional[bool] = False
-    # Note: We intentionally omit 'hidden' field as it doesn't make sense for location identity input
-
-
 class IdentityInputs(FidesSchema):
     name: Optional[RequiredType] = None
     email: Optional[RequiredType] = None
     phone: Optional[RequiredType] = None
-    location: Optional[Union[RequiredType, LocationIdentityField]] = None
     model_config = ConfigDict(extra="allow")
 
     def __init__(self, **data: Any):
@@ -42,13 +31,12 @@ class IdentityInputs(FidesSchema):
                         f'Custom identity "{field}" must be an instance of CustomIdentity '
                         '(e.g. {"label": "Field label"})'
                     )
-            elif field == "location" and isinstance(value, dict):
-                # Handle location field as LocationIdentityField
-                data[field] = LocationIdentityField(**value)
         super().__init__(**data)
 
 
-class CustomPrivacyRequestField(FidesSchema):
+class BaseCustomPrivacyRequestField(FidesSchema, ABC):
+    """Abstract base class for all custom privacy request fields"""
+
     label: str
     required: Optional[bool] = True
     default_value: Optional[str] = None
@@ -69,6 +57,41 @@ class CustomPrivacyRequestField(FidesSchema):
         return values
 
 
+class CustomPrivacyRequestField(BaseCustomPrivacyRequestField):
+    """Regular custom privacy request field supporting text, select, and multiselect types"""
+
+    field_type: Optional[Literal["text", "select", "multiselect"]] = None
+    options: Optional[List[str]] = None
+
+
+class LocationCustomPrivacyRequestField(BaseCustomPrivacyRequestField):
+    """Location field that doesn't support options and includes IP geolocation hint"""
+
+    field_type: Literal["location"] = "location"
+    ip_geolocation_hint: Optional[bool] = False
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_location_field(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        # Ensure options is not provided for location fields
+        if "options" in values:
+            raise ValueError(
+                "LocationCustomPrivacyRequestField does not support options"
+            )
+
+        # This field cannot be hidden
+        if values.get("hidden"):
+            raise ValueError("Custom location fields cannot be hidden")
+
+        return values
+
+
+# Create a simple union type - Pydantic will use the field_type to determine which model to use
+CustomPrivacyRequestFieldUnion = Union[
+    LocationCustomPrivacyRequestField, CustomPrivacyRequestField
+]
+
+
 class PrivacyRequestOption(FidesSchema):
     locations: Optional[Union[List[PrivacyNoticeRegion], Literal["fallback"]]] = None
     policy_key: Optional[str] = None
@@ -79,7 +102,9 @@ class PrivacyRequestOption(FidesSchema):
     confirm_button_text: Optional[str] = Field(alias="confirmButtonText", default=None)
     cancel_button_text: Optional[str] = Field(alias="cancelButtonText", default=None)
     identity_inputs: Optional[IdentityInputs] = None
-    custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None
+    custom_privacy_request_fields: Optional[
+        Dict[str, CustomPrivacyRequestFieldUnion]
+    ] = None
 
 
 class ConsentConfigButton(FidesSchema):
@@ -89,7 +114,9 @@ class ConsentConfigButton(FidesSchema):
     cancel_button_text: Optional[str] = Field(alias="cancelButtonText", default=None)
     icon_path: str
     identity_inputs: IdentityInputs
-    custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None
+    custom_privacy_request_fields: Optional[
+        Dict[str, CustomPrivacyRequestFieldUnion]
+    ] = None
     title: str
     modal_title: Optional[str] = Field(alias="modalTitle", default=None)
 
@@ -174,7 +201,9 @@ class PartialPrivacyRequestOption(FidesSchema):
     policy_key: str
     title: str
     identity_inputs: Optional[IdentityInputs] = None
-    custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None
+    custom_privacy_request_fields: Optional[
+        Dict[str, CustomPrivacyRequestFieldUnion]
+    ] = None
 
 
 class PartialPrivacyCenterConfig(FidesSchema):

fides/api/schemas/storage/storage.py

@@ -37,6 +37,7 @@ class StorageDetails(Enum):
     NAMING = "naming"
     MAX_RETRIES = "max_retries"
     AUTH_METHOD = "auth_method"
+    ENABLE_STREAMING = "enable_streaming"
     model_config = ConfigDict(extra="forbid")
 
 
@@ -58,6 +59,7 @@ class StorageDetailsS3(FileBasedStorageDetails):
     auth_method: AWSAuthMethod
     bucket: str
     max_retries: Optional[int] = 0
+    enable_streaming: Optional[bool] = False
     model_config = ConfigDict(use_enum_values=True)
 
 

fides/api/service/async_dsr/async_dsr_service.py

@@ -0,0 +1,75 @@
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.common_exceptions import PrivacyRequestError
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
+from fides.api.task.task_resources import TaskResources
+
+
+def requeue_polling_request(
+    db: Session,
+    async_task: RequestTask,
+) -> None:
+    """Re-queue a Privacy request that polls async tasks for a given privacy request"""
+    # Check that the privacy request is approved or in processing
+    privacy_request: PrivacyRequest = async_task.privacy_request
+
+    if privacy_request.status not in [
+        PrivacyRequestStatus.approved,
+        PrivacyRequestStatus.in_processing,
+    ]:
+        raise PrivacyRequestError(
+            f"Cannot re-queue privacy request {privacy_request.id} with status {privacy_request.status.value}"
+        )
+
+    logger.info(
+        "Polling starting for {} task {} {}",
+        async_task.action_type,
+        async_task.collection_address,
+        async_task.id,
+    )
+
+    connection_config = get_connection_config_from_task(db, async_task)
+
+    with TaskResources(
+        privacy_request,
+        privacy_request.policy,
+        [connection_config],
+        async_task,
+        db,
+    ) as resources:
+        # graph_task: GraphTask = create_graph_task(db, async_task, resources)
+        # Currently, upstream tasks and "input keys" (which are built by data dependencies)
+        # are the same, but they may not be the same in the future.
+        # upstream_tasks = async_task.upstream_tasks_objects(db)
+        # upstream_access_data: List[List[Row]] = _build_upstream_access_data(
+        #    graph_task.execution_node.input_keys, upstream_tasks
+        # )
+        # TODO: Implement the polling strategy
+        logger.info(f"found resources: {resources}")
+        return None
+
+
+def get_connection_config_from_task(
+    db: Session, request_task: RequestTask
+) -> ConnectionConfig:
+    dataset_config = DatasetConfig.filter(
+        db=db,
+        conditions=(DatasetConfig.fides_key == request_task.dataset_name),
+    ).first()
+    if not dataset_config:
+        raise PrivacyRequestError(
+            f"DatasetConfig with fides_key {request_task.dataset_name} not found."
+        )
+    connection_config = ConnectionConfig.get(
+        db=db, object_id=dataset_config.connection_config_id
+    )
+    if not connection_config:
+        raise PrivacyRequestError(
+            f"ConnectionConfig with id {dataset_config.connection_config_id} not found."
+        )
+
+    return connection_config

fides/api/service/connectors/saas_connector.py

@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session
 from starlette.status import HTTP_204_NO_CONTENT
 
 from fides.api.common_exceptions import (
-    AwaitingAsyncTaskCallback,
+    AwaitingAsyncTask,
     FidesopsException,
     PostProcessingException,
     SkippingConsentPropagation,
@@ -229,7 +229,6 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         query_config: SaaSQueryConfig = self.query_config(node)
 
         # generate initial set of requests if read request is defined, otherwise raise an exception
-
         # An endpoint can be defined with multiple 'read' requests if the data for a single
         # collection can be accessed in multiple ways for example:
         #
@@ -336,8 +335,8 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         if awaiting_async_callback:
             # If a read request was marked to expect async results, original response data here is ignored.
             # We'll instead use the data received in the callback URL later.
-            # Raising an AwaitingAsyncTaskCallback to put this task in an awaiting_processing state
-            raise AwaitingAsyncTaskCallback()
+            # Raising an AwaitingAsyncTask to put this task in an awaiting_processing state
+            raise AwaitingAsyncTask()
 
         return rows
 
@@ -641,8 +640,8 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         if awaiting_async_callback:
             # Asynchronous masking request detected in saas config.
             # If the masking request was marked to expect async results, original responses are ignored
-            # and we raise an AwaitingAsyncTaskCallback to put this task in an awaiting_processing state.
-            raise AwaitingAsyncTaskCallback()
+            # and we raise an AwaitingAsyncTask to put this task in an awaiting_processing state.
+            raise AwaitingAsyncTask()
         return rows_updated
 
     @staticmethod

fides/api/service/privacy_request/dsr_package/dsr_report_builder.py

@@ -5,13 +5,12 @@ import zipfile
 from collections import defaultdict
 from io import BytesIO
 from pathlib import Path
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 import jinja2
 from jinja2 import Environment, FileSystemLoader
 from loguru import logger
 
-from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
 from fides.api.util.storage_util import StorageJSONEncoder, format_size
 
@@ -21,6 +20,9 @@ TEXT_COLOR = "#4A5568"
 HEADER_COLOR = "#FAFAFA"
 BORDER_COLOR = "#E2E8F0"
 
+if TYPE_CHECKING:
+    from fides.api.models.privacy_request import PrivacyRequest  # pragma: no cover
+
 
 # pylint: disable=too-many-instance-attributes
 class DsrReportBuilder:
@@ -42,7 +44,7 @@ class DsrReportBuilder:
 
     def __init__(
         self,
-        privacy_request: PrivacyRequest,
+        privacy_request: "PrivacyRequest",
         dsr_data: dict[str, Any],
     ):
         """
@@ -387,7 +389,7 @@ class DsrReportBuilder:
         return self.baos
 
 
-def _map_privacy_request(privacy_request: PrivacyRequest) -> dict[str, Any]:
+def _map_privacy_request(privacy_request: "PrivacyRequest") -> dict[str, Any]:
     """Creates a map with a subset of values from the privacy request"""
     request_data: dict[str, Any] = {}
     request_data["id"] = privacy_request.id

fides/api/service/privacy_request/request_service.py

@@ -11,12 +11,13 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 from sqlalchemy.sql.elements import TextClause
 
-from fides.api.common_exceptions import PrivacyRequestNotFound
+from fides.api.common_exceptions import PrivacyRequestError, PrivacyRequestNotFound
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
     PrivacyRequest,
     RequestTask,
 )
+from fides.api.models.privacy_request.request_task import AsyncTaskType
 from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.policy import ActionType
@@ -43,6 +44,7 @@ from fides.config import CONFIG
 PRIVACY_REQUEST_STATUS_CHANGE_POLL = "privacy_request_status_change_poll"
 DSR_DATA_REMOVAL = "dsr_data_removal"
 INTERRUPTED_TASK_REQUEUE_POLL = "interrupted_task_requeue_poll"
+ASYNC_TASKS_STATUS_POLLING = "async_tasks_status_polling"
 
 
 def build_required_privacy_request_kwargs(
@@ -357,6 +359,27 @@ def initiate_interrupted_task_requeue_poll() -> None:
     )
 
 
+def initiate_async_tasks_status_polling() -> None:
+    """Initiates scheduler to check for and requeue pending polling async tasks"""
+    if CONFIG.test_mode:
+        return
+
+    assert (
+        scheduler.running
+    ), "Scheduler is not running! Cannot add async tasks status polling job."
+
+    logger.info("Initiating scheduler for async tasks status polling")
+    scheduler.add_job(
+        func=poll_async_tasks_status,
+        trigger="interval",
+        kwargs={},
+        id=ASYNC_TASKS_STATUS_POLLING,
+        coalesce=True,
+        replace_existing=True,
+        seconds=CONFIG.execution.async_tasks_status_polling_interval_seconds,
+    )
+
+
 def get_cached_task_id(entity_id: str) -> Optional[str]:
     """Gets the cached task ID for a privacy request or request task by ID.
 
@@ -465,7 +488,6 @@ def _handle_privacy_request_requeue(
             )
 
             from fides.service.privacy_request.privacy_request_service import (  # pylint: disable=cyclic-import
-                PrivacyRequestError,
                 _requeue_privacy_request,
             )
 
@@ -662,3 +684,29 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                 # Requeue the privacy request if needed
                 if should_requeue:
                     _handle_privacy_request_requeue(db, privacy_request)
+
+
+@celery_app.task(base=DatabaseTask, bind=True)
+def poll_async_tasks_status(self: DatabaseTask) -> None:
+    """
+    Poll the status of async tasks that are awaiting processing.
+    """
+
+    with self.get_new_session() as db:
+        logger.debug("Polling for async tasks status")
+
+        # Get all tasks that are awaiting processing and are from polling async tasks
+        async_tasks = (
+            db.query(RequestTask)
+            .filter(RequestTask.status == ExecutionLogStatus.awaiting_processing)
+            .filter(RequestTask.async_type == AsyncTaskType.polling)
+            .all()
+        )
+        if async_tasks:
+            logger.debug(f"Found {len(async_tasks)} async tasks to poll")
+            from fides.api.service.async_dsr.async_dsr_service import (  # pylint: disable=cyclic-import
+                requeue_polling_request,
+            )
+
+            for async_task in async_tasks:
+                requeue_polling_request(db, async_task)