ethyca-fides 2.68.1b3__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

fides/api/service/privacy_request/dsr_package/templates/collection_index.html

@@ -35,7 +35,9 @@
                         {% endif %}
 
                         {% if _is_attachment_block %}
-                           <p class="expiration-notice">Note: All download links will expire in 7 days.</p>
+                           {% if not enable_streaming %}
+                           <p class="expiration-notice">Note: All download links will expire in {{ download_link_ttl_days }} days.</p>
+                           {% endif %}
                            <div class="table table-hover">
                               <div class="table-row">
                                  <div class="table-cell" style="text-align: left;">File Name</div>

fides/api/service/privacy_request/dsr_package/templates/dataset_index.html

@@ -27,4 +27,4 @@
          </div>
       </div>
    </body>
-</html>
+</html>

fides/api/service/privacy_request/request_runner_service.py

@@ -41,6 +41,9 @@ from fides.api.models.privacy_request import (
     ProvidedIdentityType,
     can_run_checkpoint,
 )
+from fides.api.models.privacy_request.webhook import (
+    generate_privacy_request_download_token,
+)
 from fides.api.schemas.base_class import FidesSchema
 from fides.api.schemas.messaging.messaging import (
     AccessRequestCompleteBodyParams,
@@ -49,6 +52,7 @@ from fides.api.schemas.messaging.messaging import (
 from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.schemas.redis_cache import Identity
+from fides.api.schemas.storage.storage import StorageType
 from fides.api.service.connectors import FidesConnector, get_connector
 from fides.api.service.connectors.consent_email_connector import (
     CONSENT_EMAIL_CONNECTOR_TYPES,
@@ -82,6 +86,7 @@ from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
 from fides.api.util.memory_watchdog import memory_limiter
 from fides.common.api.v1.urn_registry import (
+    PRIVACY_CENTER_DSR_PACKAGE,
     PRIVACY_REQUEST_TRANSFER_TO_PARENT,
     V1_URL_PREFIX,
 )
@@ -307,8 +312,14 @@ def upload_and_save_access_results(  # pylint: disable=R0912
     loaded_attachments = [
         attachment
         for attachment in privacy_request.attachments
-        if AttachmentReferenceType.access_manual_webhook
-        not in [ref.reference_type for ref in attachment.references]
+        if not any(
+            ref.reference_type
+            in [
+                AttachmentReferenceType.access_manual_webhook,
+                AttachmentReferenceType.manual_task_submission,
+            ]
+            for ref in attachment.references
+        )
     ]
     attachments = get_attachments_content(loaded_attachments)
     # Process attachments once for both upload and storage
@@ -609,6 +620,9 @@ def run_privacy_request(
                 # The access, consent, and erasure runners for DSR 3.0 throw this exception after its
                 # Request Tasks have been built.  The Privacy Request will be requeued from
                 # the appropriate checkpoint when all the Request Tasks have run.
+                logger.info(
+                    "Privacy Request exited awaiting sub task processing (Request Tasks)"
+                )
                 return
 
             except ValidationError as exc:
@@ -740,14 +754,17 @@ def run_privacy_request(
                             else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
                         )
 
-                        if message_send_enabled(
+                        message_send_result = message_send_enabled(
                             session,
                             privacy_request.property_id,
                             action_type,
                             legacy_request_completion_enabled,
-                        ) and not policy.get_rules_for_action(
+                        )
+                        has_consent_rules = policy.get_rules_for_action(
                             action_type=ActionType.consent
-                        ):
+                        )
+
+                        if message_send_result and not has_consent_rules:
                             if not access_result_urls:
                                 # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
                                 # from the database because the Privacy Request would have exited
@@ -763,6 +780,7 @@ def run_privacy_request(
                                     access_result_urls,
                                     identity_data,
                                     privacy_request.property_id,
+                                    privacy_request.id,
                                 )
                             except (
                                 IdentityNotFoundException,
@@ -780,6 +798,7 @@ def initiate_privacy_request_completion_email(
     access_result_urls: list[str],
     identity_data: dict[str, Any],
     property_id: Optional[str],
+    privacy_request_id: str,
 ) -> None:
     """
     :param session: SQLAlchemy Session
@@ -787,6 +806,7 @@ def initiate_privacy_request_completion_email(
     :param access_result_urls: list of urls generated by access request upload
     :param identity_data: Dict of identity data
     :param property_id: Property id associated with the privacy request
+    :param privacy_request_id: ID of the privacy request for generating DSR package links
     """
     config_proxy = ConfigProxy(session)
     if not (
@@ -801,6 +821,32 @@ def initiate_privacy_request_completion_email(
         phone_number=identity_data.get(ProvidedIdentityType.phone_number.value),
     )
     if policy.get_rules_for_action(action_type=ActionType.access):
+        # Check if any rule has enable_access_package_redirect=True and enable_streaming=True in storage config
+        # This can be extended to other storage types and non streaming access results in the future
+        use_dsr_package_links = False
+        for rule in policy.get_rules_for_action(action_type=ActionType.access):
+            storage_destination = rule.get_storage_destination(session)
+            if (
+                storage_destination.type == StorageType.s3
+                and storage_destination.details.get("enable_access_package_redirect")
+                and storage_destination.details.get("enable_streaming")
+            ):
+                use_dsr_package_links = True
+                break
+
+        # Generate appropriate URLs based on streaming configuration
+        if use_dsr_package_links and config_proxy.privacy_center.url:
+            # Use DSR package links instead of direct storage URLs
+            # Generate the download token for security
+            download_token = generate_privacy_request_download_token(privacy_request_id)
+
+            # Generate DSR package URLs for the messaging template system
+            dsr_package_url = f"{config_proxy.privacy_center.url}/api{PRIVACY_CENTER_DSR_PACKAGE.format(privacy_request_id=privacy_request_id)}?token={download_token}"
+            download_links = [dsr_package_url]
+        else:
+            # Use original direct storage URLs
+            download_links = access_result_urls
+
         # synchronous for now since failure to send complete emails is fatal to request
         dispatch_message(
             db=session,
@@ -808,7 +854,7 @@ def initiate_privacy_request_completion_email(
             to_identity=to_identity,
             service_type=config_proxy.notifications.notification_service_type,
             message_body_params=AccessRequestCompleteBodyParams(
-                download_links=access_result_urls,
+                download_links=download_links,
                 subject_request_download_time_in_days=CONFIG.security.subject_request_download_link_ttl_seconds
                 / 86400,
             ),

fides/api/service/privacy_request/request_service.py

@@ -540,6 +540,7 @@ def _get_request_task_ids_in_progress(
 
 
 # pylint: disable=too-many-branches
+# pylint: disable=too-many-statements
 @celery_app.task(base=DatabaseTask, bind=True)
 def requeue_interrupted_tasks(self: DatabaseTask) -> None:
     """
@@ -660,8 +661,24 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                             break
 
                         # If the task ID is not cached, we can't check if it's running
-                        # This means the subtask is stuck - cancel the entire privacy request
+                        # This means the subtask is stuck - but we need to handle this differently
+                        # based on the privacy request status
                         if not subtask_id:
+                            if (
+                                privacy_request.status
+                                == PrivacyRequestStatus.requires_input
+                            ):
+                                # For requires_input status, don't automatically error the request
+                                # as it's intentionally waiting for user input
+                                logger.warning(
+                                    f"No task ID found for request task {request_task_id} "
+                                    f"(privacy request {privacy_request.id}) in requires_input status - "
+                                    f"keeping request in current status as it may be waiting for manual input"
+                                )
+                                should_requeue = False
+                                break
+
+                            # For other statuses, cancel the entire privacy request
                             _cancel_interrupted_tasks_and_error_privacy_request(
                                 db,
                                 privacy_request,

fides/api/service/storage/storage_uploader_service.py

@@ -51,13 +51,7 @@ def upload(
         config.secrets is not None,
     )
 
-    if config.secrets:
-        logger.debug("Storage config secrets type: {}", type(config.secrets))
-        if isinstance(config.secrets, dict):
-            logger.debug("Storage config secrets keys: {}", list(config.secrets.keys()))
-        else:
-            logger.debug("Storage config secrets is not a dict: {}", config.secrets)
-    else:
+    if not config.secrets:
         logger.warning("Storage config has no secrets!")
 
     uploader: Any = _get_uploader_from_config_type(config.type)  # type: ignore
@@ -130,19 +124,6 @@ def _s3_uploader(
         config.secrets is not None,
     )
 
-    if config.secrets:
-        logger.debug(
-            "Config secrets keys: {}",
-            (
-                list(config.secrets.keys())
-                if isinstance(config.secrets, dict)
-                else "Not a dict"
-            ),
-        )
-        logger.debug("Config secrets type: {}", type(config.secrets))
-    else:
-        logger.warning("Config secrets is None or empty!")
-
     enable_streaming = config.details.get(StorageDetails.ENABLE_STREAMING.value, False)
     file_key: str = _construct_file_key(privacy_request.id, config, enable_streaming)
 
@@ -154,7 +135,6 @@ def _s3_uploader(
         file_key = f"{privacy_request.id}.zip"
         # Use streaming upload for better memory efficiency
         logger.debug("Using streaming S3 upload for {}", file_key)
-        logger.debug("Calling upload_to_s3_streaming with secrets: {}", config.secrets)
         return upload_to_s3_streaming(
             config.secrets,  # type: ignore
             data,

fides/api/service/storage/streaming/dsr_storage.py

@@ -17,7 +17,6 @@ def stream_dsr_buffer_to_storage(
     bucket_name: str,
     file_key: str,
     dsr_buffer: BytesIO,
-    content_type: str = "application/zip",
 ) -> None:
     """Stream DSR buffer to storage using smart-open streaming.
 
@@ -29,15 +28,12 @@ def stream_dsr_buffer_to_storage(
         bucket_name: Storage bucket name
         file_key: File key in storage
         dsr_buffer: Pre-generated DSR report buffer (BytesIO)
-        content_type: MIME type for the uploaded file (defaults to application/zip)
     """
     # Get the content from the buffer
     content = dsr_buffer.getvalue()
     try:
         # Use smart-open's streaming upload for efficient memory usage
-        with storage_client.stream_upload(
-            bucket_name, file_key, content_type=content_type
-        ) as upload_stream:
+        with storage_client.stream_upload(bucket_name, file_key) as upload_stream:
             upload_stream.write(content)
 
     except Exception as e:

fides/api/service/storage/streaming/s3/s3_storage_client.py

@@ -7,7 +7,7 @@ from typing import Any, Optional
 from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
-from fides.api.schemas.storage.storage import AWSAuthMethod, StorageSecrets
+from fides.api.schemas.storage.storage import AWSAuthMethod
 from fides.api.service.storage.s3 import create_presigned_url_for_s3
 from fides.api.service.storage.streaming.base_storage_client import BaseStorageClient
 from fides.api.util.aws_util import get_s3_client
@@ -20,17 +20,19 @@ class S3StorageClient(BaseStorageClient):
     generation for the smart-open storage client.
     """
 
-    def __init__(self, storage_secrets: dict[StorageSecrets, Any]):
+    def __init__(self, auth_method: str, storage_secrets: dict[str, Any]):
         """Initialize the storage client with secrets.
 
         Args:
-            storage_secrets: Provider-specific storage credentials and configuration using StorageSecrets enum keys
+            storage_secrets: Provider-specific storage credentials and configuration using string keys
+                           (e.g., "aws_access_key_id", "region_name") from format_secrets()
         """
         super().__init__(storage_secrets)
-        self.storage_secrets: dict[StorageSecrets, Any] = storage_secrets
+        self.storage_secrets: dict[str, Any] = storage_secrets
+        self.auth_method = auth_method
 
     def build_uri(self, bucket: str, key: str) -> str:
-        """Build the S3 URI for the storage location.
+        """Build S3 URI for the given bucket and key.
 
         Args:
             bucket: S3 bucket name
@@ -45,26 +47,75 @@ class S3StorageClient(BaseStorageClient):
 
     def get_transport_params(self) -> dict[str, Any]:
         """Get S3-specific transport parameters for smart-open.
+        Type annotation: get_s3_client returns a boto3 S3 client object, not a Session
+        This is what smart-open expects for the "client" transport parameter
 
         Returns:
             Dictionary of S3 transport parameters for smart-open
         """
-        params = {}
-
-        if StorageSecrets.AWS_ACCESS_KEY_ID in self.storage_secrets:
-            params["access_key"] = self.storage_secrets[
-                StorageSecrets.AWS_ACCESS_KEY_ID
-            ]
-        if StorageSecrets.AWS_SECRET_ACCESS_KEY in self.storage_secrets:
-            params["secret_key"] = self.storage_secrets[
-                StorageSecrets.AWS_SECRET_ACCESS_KEY
-            ]
-        if StorageSecrets.REGION_NAME in self.storage_secrets:
-            params["region"] = self.storage_secrets[StorageSecrets.REGION_NAME]
-        if StorageSecrets.AWS_ASSUME_ROLE in self.storage_secrets:
-            params["assume_role_arn"] = self.storage_secrets[
-                StorageSecrets.AWS_ASSUME_ROLE
-            ]
+        params: dict[str, Any] = {}
+
+        # Create S3 client for smart-open
+        try:
+            # Determine auth method based on available credentials
+            if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
+
+                # For automatic authentication, check if region is available
+                if not self.storage_secrets.get("region_name", None):
+                    logger.warning(
+                        "No region specified in storage secrets for automatic authentication"
+                        "This may cause credential issues - consider setting a default region"
+                    )
+
+            # Extract assume_role_arn if present
+            assume_role_arn = None
+            if (
+                "assume_role_arn" in self.storage_secrets
+                and self.storage_secrets["assume_role_arn"]
+            ):
+                assume_role_arn = self.storage_secrets["assume_role_arn"]
+                logger.debug(f"Using assume role ARN: {assume_role_arn}")
+
+            # Create S3 client using existing utility
+            # get_s3_client returns a boto3 S3 client, not a Session
+            s3_client: Any = None
+            try:
+                s3_client = get_s3_client(
+                    self.auth_method, self.storage_secrets, assume_role_arn  # type: ignore
+                )
+                logger.debug("Successfully created S3 client")
+            except Exception as e:
+                # For automatic authentication, try to provide more helpful error messages
+                if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
+                    logger.error(
+                        f"Failed to create S3 client with automatic authentication: {e}. "
+                        "This usually means AWS credentials are not available in the environment"
+                        "Please ensure AWS credentials are configured via environment variables, IAM roles, or AWS profiles"
+                    )
+                    raise ValueError(
+                        f"Automatic AWS authentication failed: {e}. Please check your AWS credential configuration."
+                    )
+                raise
+
+            params["client"] = s3_client
+
+        except Exception as e:
+            logger.error(f"Failed to create S3 client for smart-open: {e}")
+            raise
+
+        # Include credentials at top level for compatibility
+        # Note: When using an S3 client, these credential parameters are not needed
+        # and will be ignored by smart-open, causing warnings
+        # Only include them if no S3 client is provided (fallback scenario)
+        if not params.get("client"):
+            for key, transport_key in [
+                ("aws_access_key_id", "access_key"),
+                ("aws_secret_access_key", "secret_key"),
+                ("region_name", "region"),
+                ("assume_role_arn", "assume_role_arn"),
+            ]:
+                if key in self.storage_secrets and self.storage_secrets[key]:
+                    params[transport_key] = self.storage_secrets[key]
 
         return params
 
@@ -85,28 +136,15 @@ class S3StorageClient(BaseStorageClient):
             Exception: If presigned URL generation fails
         """
         try:
-            # Storage secrets are already in the right format for get_s3_client
-            # get_s3_client expects dict[StorageSecrets, Any] with enum keys
-            s3_secrets = self.storage_secrets
-
-            # Determine auth method based on available credentials
-            # If AWS credentials are present, use SECRET_KEYS, otherwise use AUTOMATIC
-            if (
-                StorageSecrets.AWS_ACCESS_KEY_ID in self.storage_secrets
-                and StorageSecrets.AWS_SECRET_ACCESS_KEY in self.storage_secrets
-                and self.storage_secrets[StorageSecrets.AWS_ACCESS_KEY_ID]
-                and self.storage_secrets[StorageSecrets.AWS_SECRET_ACCESS_KEY]
-            ):
-                auth_method = AWSAuthMethod.SECRET_KEYS.value
-            else:
-                auth_method = AWSAuthMethod.AUTOMATIC.value
-
             # Extract assume_role_arn if present for role assumption
             assume_role_arn = None
-            if StorageSecrets.AWS_ASSUME_ROLE in self.storage_secrets:
-                assume_role_arn = self.storage_secrets[StorageSecrets.AWS_ASSUME_ROLE]
+            if "assume_role_arn" in self.storage_secrets:
+                assume_role_arn = self.storage_secrets["assume_role_arn"]
 
-            s3_client = get_s3_client(auth_method, s3_secrets, assume_role_arn)
+            # get_s3_client returns a boto3 S3 client, not a Session
+            s3_client: Any = get_s3_client(
+                self.auth_method, self.storage_secrets, assume_role_arn  # type: ignore
+            )
             return create_presigned_url_for_s3(s3_client, bucket, key, ttl_seconds)
         except Exception as e:
             logger.error(f"Failed to generate S3 presigned URL: {e}")

fides/api/service/storage/streaming/s3/streaming_s3.py

@@ -54,7 +54,8 @@ def upload_to_s3_streaming(
         return response
 
     try:
-        storage_client = SmartOpenStorageClient("s3", formatted_secrets)
+        logger.debug("Creating SmartOpenStorageClient with formatted secrets")
+        storage_client = SmartOpenStorageClient("s3", auth_method, formatted_secrets)
 
         # Create upload config for the streaming interface
         upload_config = StorageUploadConfig(
@@ -92,6 +93,10 @@ def _process_storage_secrets_input(
     """Process input and convert to string-keyed dictionary."""
     final_secrets: dict[str, Any] = {}
 
+    logger.debug(
+        f"Processing storage secrets input of type: {type(storage_secrets).__name__}"
+    )
+
     if isinstance(storage_secrets, StorageSecretsS3):
         # Convert StorageSecretsS3 model directly to string keys
         for key, value in storage_secrets.model_dump().items():
@@ -133,6 +138,7 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
     else:
         # AUTOMATIC authentication - check if region is provided
         has_region = "region_name" in final_secrets and final_secrets["region_name"]
+
         if not has_region:
             raise ValueError(
                 "Missing required region_name for AUTOMATIC authentication. "
@@ -140,13 +146,6 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
             )
 
 
-def _set_default_region(final_secrets: dict[str, Any]) -> None:
-    """Set default region if missing."""
-    if "region_name" not in final_secrets or not final_secrets["region_name"]:
-        logger.debug("Setting default region to 'us-east-1'")
-        final_secrets["region_name"] = "us-east-1"
-
-
 def format_secrets(
     storage_secrets: Union[StorageSecretsS3, dict[StorageSecrets, Any]]  # type: ignore[misc]
 ) -> dict[str, Any]:
@@ -156,8 +155,8 @@ def format_secrets(
     This function handles multiple credential formats and processes them through several stages:
     1. Input processing: Accepts either StorageSecretsS3 models (from API) or raw dicts (from database)
     2. Key normalization: Converts all keys to string format for consistency
-    3. Validation: Ensures required AWS credentials are present based on auth method
-    4. Default setting: Sets default region if missing (after validation)
+    3. Default setting: Sets default region if missing (before validation for better automatic auth support)
+    4. Validation: Ensures required AWS credentials are present based on auth method
     5. Return: Returns string-keyed dict ready for S3StorageClient
 
     Input formats:
@@ -179,18 +178,7 @@ def format_secrets(
     Raises:
         ValueError: If required AWS credentials are missing for the chosen auth method
     """
-    logger.debug("format_secrets called with type: {}", type(storage_secrets).__name__)
-
-    # Stage 1: Process input and create final format directly
     final_secrets = _process_storage_secrets_input(storage_secrets)
-
-    # Stage 2: Validate required credentials BEFORE setting defaults
     _validate_aws_credentials(final_secrets)
 
-    # Stage 3: Set default region if missing (after validation)
-    _set_default_region(final_secrets)
-
-    logger.debug(
-        "format_secrets completed successfully with {} keys", len(final_secrets)
-    )
     return final_secrets

fides/api/service/storage/streaming/smart_open_client.py

@@ -29,7 +29,12 @@ class SmartOpenStorageClient:
 
     min_part_size: int = MIN_PART_SIZE
 
-    def __init__(self, storage_type: str, storage_secrets: Any):
+    def __init__(
+        self,
+        storage_type: str,
+        auth_method: Optional[str],
+        storage_secrets: Any,
+    ):
         """Initialize the smart-open storage client.
 
         Args:
@@ -38,9 +43,10 @@ class SmartOpenStorageClient:
                            Will be passed to the specific storage client implementation.
         """
         self.storage_type = StorageClientFactory._normalize_storage_type(storage_type)
+        self.auth_method = auth_method
         self.storage_secrets = storage_secrets
         self._provider_client = StorageClientFactory.create_client(
-            storage_type, storage_secrets
+            storage_type, auth_method, storage_secrets
         )
 
     def _build_uri(self, bucket: str, key: str) -> str:
@@ -216,7 +222,6 @@ class SmartOpenStorageClient:
         self,
         bucket: str,
         key: str,
-        content_type: Optional[str] = None,
     ) -> Any:
         """Get a writable stream for uploading data.
 
@@ -225,7 +230,6 @@ class SmartOpenStorageClient:
         Args:
             bucket: Storage bucket/container name
             key: Object key/path
-            content_type: MIME type of the object
 
         Returns:
             Writable file-like object
@@ -233,9 +237,6 @@ class SmartOpenStorageClient:
         uri = self._build_uri(bucket, key)
         transport_params = self._get_transport_params()
 
-        if content_type:
-            transport_params["content_type"] = content_type
-
         return smart_open.open(uri, "wb", transport_params=transport_params)
 
     @retry_cloud_storage_operation(