ethyca-fides 2.68.1b2__py2.py3-none-any.whl → 2.68.1b4__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-19T14:51:50-0700",
+ "date": "2025-08-27T12:07:27-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "047dc431c1e669789ec8aa19ededd438909b8c0c",
- "version": "2.68.1b2"
+ "full-revisionid": "062904c4ee33fadc4f937ccfd3fd2f2e2a2cbae8",
+ "version": "2.68.1b4"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/90502bcda282_update_request_tasks_add_polling_async.py

@@ -0,0 +1,35 @@
+"""Update Request Tasks
+
+Revision ID: 90502bcda282
+Revises: 3baf42d251a6
+Create Date: 2025-08-05 16:31:45.335384
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "90502bcda282"
+down_revision = "3baf42d251a6"
+branch_labels = None
+depends_on = None
+
+async_task_type = sa.Enum("manual", "polling", "callback", name="asynctasktype")
+
+
+def upgrade():
+    async_task_type.create(op.get_bind(), checkfirst=True)
+    op.add_column(
+        "requesttask",
+        sa.Column(
+            "async_type",
+            async_task_type,
+            nullable=True,
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("requesttask", "async_type")
+    async_task_type.drop(op.get_bind(), checkfirst=True)

fides/api/alembic/migrations/versions/b1a2c3d4e5f6_add_location_to_privacy_request.py

@@ -0,0 +1,26 @@
+"""add location to privacy request
+
+Revision ID: b1a2c3d4e5f6
+Revises: 2f3c1a2d6b10
+Create Date: 2025-08-06 18:15:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b1a2c3d4e5f6"
+down_revision = "90502bcda282"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add location column to privacyrequest table
+    op.add_column("privacyrequest", sa.Column("location", sa.String(), nullable=True))
+
+
+def downgrade():
+    # Remove location column from privacyrequest table
+    op.drop_column("privacyrequest", "location")

fides/api/api/v1/api.py

@@ -5,6 +5,7 @@ from fides.api.api.v1.endpoints import (
     consent_request_endpoints,
     dataset_config_endpoints,
     drp_endpoints,
+    dsr_package_link,
     encryption_endpoints,
     identity_verification_endpoints,
     manual_webhook_endpoints,
@@ -32,6 +33,7 @@ api_router.include_router(connection_endpoints.router)
 api_router.include_router(consent_request_endpoints.router)
 api_router.include_router(dataset_config_endpoints.router)
 api_router.include_router(drp_endpoints.router)
+api_router.include_router(dsr_package_link.router)
 api_router.include_router(encryption_endpoints.router)
 api_router.include_router(masking_endpoints.router)
 api_router.include_router(oauth_endpoints.router)

fides/api/api/v1/endpoints/dsr_package_link.py

@@ -0,0 +1,167 @@
+import uuid
+
+from fastapi import Depends, HTTPException, Request, Response
+from fastapi.responses import RedirectResponse
+from sqlalchemy.orm import Session
+from starlette.status import (
+    HTTP_302_FOUND,
+    HTTP_400_BAD_REQUEST,
+    HTTP_401_UNAUTHORIZED,
+    HTTP_403_FORBIDDEN,
+    HTTP_404_NOT_FOUND,
+    HTTP_422_UNPROCESSABLE_ENTITY,
+)
+
+from fides.api.api.deps import get_db
+from fides.api.common_exceptions import AuthenticationError, AuthorizationError
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.models.storage import get_active_default_storage_config
+from fides.api.oauth.utils import validate_download_token
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
+from fides.api.schemas.storage.storage import StorageType
+from fides.api.service.storage.streaming.s3 import S3StorageClient
+from fides.api.util.api_router import APIRouter
+from fides.api.util.endpoint_utils import fides_limiter
+from fides.common.api.v1.urn_registry import PRIVACY_CENTER_DSR_PACKAGE, V1_URL_PREFIX
+from fides.config import CONFIG
+
+router = APIRouter(tags=["Privacy Center"], prefix=V1_URL_PREFIX)
+
+
+def get_privacy_request_or_error(
+    privacy_request_id: str, db: Session
+) -> PrivacyRequest:
+    """Load the privacy request or throw a 404"""
+    # Note: UUID format validation is now done earlier in the endpoint
+    privacy_request = PrivacyRequest.get(db, object_id=privacy_request_id)
+
+    if not privacy_request:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"No privacy request found with id '{privacy_request_id}'.",
+        )
+
+    if privacy_request.deleted_at is not None:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Privacy request with id {privacy_request_id} has been deleted.",
+        )
+
+    return privacy_request
+
+
+def raise_error(status_code: int, detail: str) -> None:
+    """Raise an HTTPException with the given status code and detail"""
+    raise HTTPException(
+        status_code=status_code,
+        detail=detail,
+    )
+
+
+@router.get(
+    PRIVACY_CENTER_DSR_PACKAGE,
+    status_code=HTTP_302_FOUND,
+)
+@fides_limiter.limit(CONFIG.security.public_request_rate_limit)
+def get_access_results_urls(
+    privacy_request_id: str,
+    token: str,
+    db: Session = Depends(get_db),
+    *,
+    request: Request,  # required for rate limiting
+    response: Response,  # required for rate limiting
+) -> RedirectResponse:
+    """
+    Public endpoint for retrieving access results URLs for a privacy request.
+    This endpoint generates fresh presigned URLs and redirects to the first available result.
+    This endpoint is designed to be accessible via email links sent to end users.
+    No authentication is required, but a valid download token is required for security.
+    Rate limiting is applied for additional security.
+
+    privacy_request_id parameter is required in the URL path.
+    token parameter is required as a query parameter for security.
+    """
+    # --------------Security checks--------------
+    # First validate the privacy request ID format to prevent SSRF attacks
+    # This is a simple string check that doesn't require database access
+    if not privacy_request_id.startswith("pri_"):
+        raise_error(
+            HTTP_400_BAD_REQUEST,
+            f"Invalid privacy request ID format: '{privacy_request_id}'. Must start with 'pri_' followed by a valid UUID v4.",
+        )
+
+    # Extract the UUID part after the prefix
+    uuid_part = privacy_request_id[4:]  # Remove "pri_" prefix
+
+    try:
+        uuid.UUID(uuid_part, version=4)
+    except ValueError:
+        raise_error(
+            HTTP_400_BAD_REQUEST,
+            f"Invalid privacy request ID format: '{privacy_request_id}'. Must start with 'pri_' followed by a valid UUID v4.",
+        )
+
+    try:
+        # Validate the download token before proceeding
+        validate_download_token(token, privacy_request_id)
+    except AuthenticationError as e:
+        raise_error(HTTP_401_UNAUTHORIZED, str(e.detail))
+    except AuthorizationError as e:
+        raise_error(HTTP_403_FORBIDDEN, str(e.detail))
+
+    # --------------Data checks--------------
+    storage_config = get_active_default_storage_config(db)
+    privacy_request = get_privacy_request_or_error(privacy_request_id, db)
+
+    if not storage_config:
+        raise_error(
+            HTTP_400_BAD_REQUEST, "No active default storage configuration found."
+        )
+
+    if privacy_request.status != PrivacyRequestStatus.complete:
+        raise_error(
+            HTTP_400_BAD_REQUEST,
+            f"Access results for privacy request '{privacy_request.id}' are not available because the request is not complete.",
+        )
+
+    if (
+        not privacy_request.access_result_urls
+        or not privacy_request.access_result_urls.get("access_result_urls")
+    ):
+        raise_error(
+            HTTP_404_NOT_FOUND,
+            f"No access results found for privacy request '{privacy_request.id}'.",
+        )
+
+    # --------------Processing--------------
+    file_name = f"{privacy_request.id}.zip"
+
+    # At this point, storage_config is guaranteed to exist due to earlier validation
+    # and we've already checked that it's not None above
+    assert storage_config is not None
+
+    if storage_config.type != StorageType.s3:
+        # Handle all other storage types (transcend, ethyca, local, etc.)
+        raise_error(
+            HTTP_400_BAD_REQUEST,
+            f"Storage type '{storage_config.type}' is not supported for download redirects. "
+            "Only S3 storage is supported for this endpoint.",
+        )
+
+    # Get bucket name from storage config
+    bucket_name = storage_config.details.get("bucket")
+    if not bucket_name:
+        raise_error(HTTP_400_BAD_REQUEST, "S3 bucket name not found in storage config.")
+
+    try:
+        # Use S3StorageClient for cleaner presigned URL generation
+        s3_storage_client = S3StorageClient(storage_config.secrets)
+        result_url = s3_storage_client.generate_presigned_url(
+            bucket=bucket_name,
+            key=file_name,
+        )
+    except Exception as e:
+        raise_error(HTTP_400_BAD_REQUEST, f"Failed to generate presigned URL: {str(e)}")
+
+    # Convert the URL to string for RedirectResponse
+    return RedirectResponse(url=str(result_url), status_code=HTTP_302_FOUND)

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -50,6 +50,7 @@ from fides.api.common_exceptions import (
     IdentityVerificationException,
     ManualWebhookFieldsUnset,
     NoCachedManualWebhookEntry,
+    PrivacyRequestError,
     TraversalError,
     ValidationError,
 )
@@ -123,6 +124,7 @@ from fides.api.util.endpoint_utils import validate_start_and_end_filters
 from fides.api.util.enums import ColumnSort
 from fides.api.util.fuzzy_search_utils import get_decrypted_identities_automaton
 from fides.api.util.storage_util import StorageJSONEncoder
+from fides.api.util.text import normalize_location_code
 from fides.common.api.scope_registry import (
     PRIVACY_REQUEST_CALLBACK_RESUME,
     PRIVACY_REQUEST_CREATE,
@@ -175,7 +177,6 @@ from fides.service.dataset.dataset_config_service import (
 )
 from fides.service.messaging.messaging_service import MessagingService
 from fides.service.privacy_request.privacy_request_service import (
-    PrivacyRequestError,
     PrivacyRequestService,
     _process_privacy_request_restart,
     _requeue_privacy_request,
@@ -403,6 +404,7 @@ def _filter_privacy_request_queryset(
     errored_lt: Optional[datetime] = None,
     errored_gt: Optional[datetime] = None,
     external_id: Optional[str] = None,
+    location: Optional[str] = None,
     action_type: Optional[ActionType] = None,
     include_consent_webhook_requests: Optional[bool] = False,
     include_deleted_requests: Optional[bool] = False,
@@ -539,6 +541,29 @@ def _filter_privacy_request_queryset(
         query = query.filter(PrivacyRequest.id.ilike(f"%{request_id}%"))
     if external_id:
         query = query.filter(PrivacyRequest.external_id.ilike(f"{external_id}%"))
+    if location:
+        # Support filtering by exact location match or country prefix
+        # e.g., "US" matches both "US" and "US-CA", "US-NY", etc.
+        # "US-CA" matches only "US-CA"
+        # Also normalize input to handle underscores and case insensitivity
+
+        try:
+            normalized_location = normalize_location_code(location)
+        except ValueError:
+            # If normalization fails, treat as no results to prevent errors
+            query = query.filter(False)
+        else:
+            if "-" in normalized_location:
+                # Exact match for subdivision codes
+                query = query.filter(PrivacyRequest.location == normalized_location)
+            else:
+                # Country code - match country or any subdivision of that country
+                query = query.filter(
+                    or_(
+                        PrivacyRequest.location == normalized_location,
+                        PrivacyRequest.location.ilike(f"{normalized_location}-%"),
+                    )
+                )
     if status:
         query = query.filter(PrivacyRequest.status.in_(status))
     if created_lt:
@@ -685,6 +710,7 @@ def _shared_privacy_request_search(
     errored_lt: Optional[datetime] = None,
     errored_gt: Optional[datetime] = None,
     external_id: Optional[str] = None,
+    location: Optional[str] = None,
     action_type: Optional[ActionType] = None,
     verbose: Optional[bool] = False,
     include_identities: Optional[bool] = False,
@@ -723,6 +749,7 @@ def _shared_privacy_request_search(
         errored_lt,
         errored_gt,
         external_id,
+        location,
         action_type,
         None,
         include_deleted_requests,
@@ -796,6 +823,7 @@ def get_request_status(
     errored_lt: Optional[datetime] = None,
     errored_gt: Optional[datetime] = None,
     external_id: Optional[str] = None,
+    location: Optional[str] = None,
     action_type: Optional[ActionType] = None,
     verbose: Optional[bool] = False,
     include_identities: Optional[bool] = False,
@@ -836,6 +864,7 @@ def get_request_status(
         errored_lt=errored_lt,
         errored_gt=errored_gt,
         external_id=external_id,
+        location=location,
         action_type=action_type,
         verbose=verbose,
         include_identities=include_identities,
@@ -890,6 +919,7 @@ def privacy_request_search(
         errored_lt=privacy_request_filter.errored_lt,
         errored_gt=privacy_request_filter.errored_gt,
         external_id=privacy_request_filter.external_id,
+        location=privacy_request_filter.location,
         action_type=privacy_request_filter.action_type,
         verbose=privacy_request_filter.verbose,
         include_identities=privacy_request_filter.include_identities,

fides/api/api/v1/endpoints/user_endpoints.py

@@ -250,6 +250,10 @@ def logout_oauth_client(
     if authorization is None:
         raise AuthenticationError(detail="Authentication Failure")
 
+    # Validate that the token looks like a valid JWE token (5 segments separated by dots)
+    if not authorization or authorization.count(".") != 4:
+        return None
+
     try:
         token_data = json.loads(
             extract_payload(authorization, CONFIG.security.app_encryption_key)

fides/api/common_exceptions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import List
+from typing import Dict, List, Optional
 
 from fastapi import HTTPException, status
 from starlette.status import (
@@ -175,8 +175,8 @@ class RequestTaskNotFound(BaseException):
     """Privacy Request Task Not Found"""
 
 
-class AwaitingAsyncTaskCallback(BaseException):
-    """Request Task is Awaiting Processing - Awaiting Async Task Callback"""
+class AwaitingAsyncTask(BaseException):
+    """Request Task is Awaiting Processing - Awaiting Async Task"""
 
 
 class UpstreamTasksNotReady(BaseException):
@@ -373,6 +373,15 @@ class InvalidScopeError(HTTPException):
         )
 
 
+class PrivacyRequestError(Exception):
+    """Base exception for privacy request operations."""
+
+    def __init__(self, message: str, data: Optional[Dict] = None):
+        self.message = message
+        self.data = data
+        super().__init__(message)
+
+
 class KeyOrNameAlreadyExists(Exception):
     """A resource already exists with this key or name."""
 

fides/api/models/detection_discovery/core.py

@@ -432,11 +432,17 @@ class StagedResourceAncestor(Base):
 
                     # Execute batch when it reaches the desired size
                     if len(current_batch) >= batch_size:
+                        logger.debug(
+                            f"Inserting batch of {len(current_batch)} staged resource ancestor links"
+                        )
                         db.execute(stmt_text, current_batch)
                         current_batch = []
 
         # Execute any remaining items in the final batch
         if current_batch:
+            logger.debug(
+                f"Inserting batch of {len(current_batch)} staged resource ancestor links"
+            )
             db.execute(stmt_text, current_batch)
 
 

fides/api/models/privacy_request/privacy_request.py

@@ -222,6 +222,7 @@ class PrivacyRequest(
     canceled_at = Column(DateTime(timezone=True), nullable=True)
     consent_preferences = Column(MutableList.as_mutable(JSONB), nullable=True)
     source = Column(EnumColumn(PrivacyRequestSource), nullable=True)
+    location = Column(String, nullable=True)
 
     # A PrivacyRequest can be soft deleted, so we store when it was deleted
     deleted_at = Column(DateTime(timezone=True), nullable=True)

fides/api/models/privacy_request/request_task.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from enum import Enum as EnumType
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
 from loguru import logger
@@ -15,6 +16,7 @@ from sqlalchemy_utils.types.encrypted.encrypted_type import (
 )
 
 from fides.api.db.base_class import Base, JSONTypeOverride  # type: ignore[attr-defined]
+from fides.api.db.util import EnumColumn
 from fides.api.graph.config import (
     ROOT_COLLECTION_ADDRESS,
     TERMINATOR_ADDRESS,
@@ -67,6 +69,24 @@ class TraversalDetails(FidesSchema):
         )
 
 
+class AsyncTaskType(EnumType):
+    """
+    Enum for asynchronous RequestTask types
+
+    These tasks differ from our regular tasks in that they are not finished on the same process that they are created,
+    but rather are put into a pause state and then triggered to completion by a different process
+
+    manual: A task that is manually triggered by the user
+    polling: A task that is triggered by a polling mechanism
+    callback: A task that is triggered by a callback from an external system
+
+    """
+
+    manual = "manual"
+    polling = "polling"
+    callback = "callback"
+
+
 # TODO: At some point we will refactor this model to store all task types in a common table that links to tables with specific task attributes.
 class RequestTask(WorkerTask, Base):
     """
@@ -145,6 +165,11 @@ class RequestTask(WorkerTask, Base):
 
     # For async tasks awaiting callback
     callback_succeeded = Column(Boolean)
+    # to recognize Polling async task
+    async_type = Column(
+        EnumColumn(AsyncTaskType, native_enum=True),
+        nullable=True,
+    )
 
     # Stores a serialized collection that can be transformed back into a Collection to help
     # execute the current task

fides/api/models/privacy_request/webhook.py

@@ -13,12 +13,17 @@ from fides.api.models.policy import PolicyPreWebhook, WebhookDirection
 from fides.api.models.pre_approval_webhook import PreApprovalWebhook
 from fides.api.models.privacy_request.request_task import RequestTask
 from fides.api.oauth.jwt import generate_jwe
-from fides.api.schemas.external_https import RequestTaskJWE, WebhookJWE
+from fides.api.schemas.external_https import (
+    DownloadTokenJWE,
+    RequestTaskJWE,
+    WebhookJWE,
+)
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.schemas.redis_cache import Identity
 from fides.common.api.scope_registry import (
     PRIVACY_REQUEST_CALLBACK_RESUME,
+    PRIVACY_REQUEST_READ_ACCESS_RESULTS,
     PRIVACY_REQUEST_REVIEW,
 )
 from fides.config import CONFIG
@@ -92,3 +97,30 @@ def generate_request_task_callback_jwe(request_task: RequestTask) -> str:
         json.dumps(jwe.model_dump(mode="json")),
         CONFIG.security.app_encryption_key,
     )
+
+
+def generate_privacy_request_download_token(privacy_request_id: str) -> str:
+    """
+    Generate a JWE token for users to download their privacy request access package.
+    This is currently used for the DSR package link endpoint which provides a redirect
+    to a presigned URL for the access results.
+    This token expires based on the configured TTL for security.
+    """
+    from datetime import timedelta
+
+    now = datetime.now()
+    # Use the configured TTL from security settings
+    expiration = now + timedelta(
+        seconds=CONFIG.security.subject_request_download_link_ttl_seconds
+    )
+
+    jwe = DownloadTokenJWE(
+        privacy_request_id=privacy_request_id,
+        scopes=[PRIVACY_REQUEST_READ_ACCESS_RESULTS],
+        iat=now.isoformat(),
+        exp=expiration.isoformat(),
+    )
+    return generate_jwe(
+        json.dumps(jwe.model_dump(mode="json")),
+        CONFIG.security.app_encryption_key,
+    )