ethyca-fides 2.68.1b4__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

fides/api/service/privacy_request/request_service.py

@@ -540,6 +540,7 @@ def _get_request_task_ids_in_progress(
 
 
 # pylint: disable=too-many-branches
+# pylint: disable=too-many-statements
 @celery_app.task(base=DatabaseTask, bind=True)
 def requeue_interrupted_tasks(self: DatabaseTask) -> None:
     """
@@ -660,8 +661,24 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                             break
 
                         # If the task ID is not cached, we can't check if it's running
-                        # This means the subtask is stuck - cancel the entire privacy request
+                        # This means the subtask is stuck - but we need to handle this differently
+                        # based on the privacy request status
                         if not subtask_id:
+                            if (
+                                privacy_request.status
+                                == PrivacyRequestStatus.requires_input
+                            ):
+                                # For requires_input status, don't automatically error the request
+                                # as it's intentionally waiting for user input
+                                logger.warning(
+                                    f"No task ID found for request task {request_task_id} "
+                                    f"(privacy request {privacy_request.id}) in requires_input status - "
+                                    f"keeping request in current status as it may be waiting for manual input"
+                                )
+                                should_requeue = False
+                                break
+
+                            # For other statuses, cancel the entire privacy request
                             _cancel_interrupted_tasks_and_error_privacy_request(
                                 db,
                                 privacy_request,

fides/api/service/storage/storage_uploader_service.py

@@ -51,13 +51,7 @@ def upload(
         config.secrets is not None,
     )
 
-    if config.secrets:
-        logger.debug("Storage config secrets type: {}", type(config.secrets))
-        if isinstance(config.secrets, dict):
-            logger.debug("Storage config secrets keys: {}", list(config.secrets.keys()))
-        else:
-            logger.debug("Storage config secrets is not a dict: {}", config.secrets)
-    else:
+    if not config.secrets:
         logger.warning("Storage config has no secrets!")
 
     uploader: Any = _get_uploader_from_config_type(config.type)  # type: ignore
@@ -130,19 +124,6 @@ def _s3_uploader(
         config.secrets is not None,
     )
 
-    if config.secrets:
-        logger.debug(
-            "Config secrets keys: {}",
-            (
-                list(config.secrets.keys())
-                if isinstance(config.secrets, dict)
-                else "Not a dict"
-            ),
-        )
-        logger.debug("Config secrets type: {}", type(config.secrets))
-    else:
-        logger.warning("Config secrets is None or empty!")
-
     enable_streaming = config.details.get(StorageDetails.ENABLE_STREAMING.value, False)
     file_key: str = _construct_file_key(privacy_request.id, config, enable_streaming)
 
@@ -154,7 +135,6 @@ def _s3_uploader(
         file_key = f"{privacy_request.id}.zip"
         # Use streaming upload for better memory efficiency
         logger.debug("Using streaming S3 upload for {}", file_key)
-        logger.debug("Calling upload_to_s3_streaming with secrets: {}", config.secrets)
         return upload_to_s3_streaming(
             config.secrets,  # type: ignore
             data,

fides/api/service/storage/streaming/dsr_storage.py

@@ -17,7 +17,6 @@ def stream_dsr_buffer_to_storage(
     bucket_name: str,
     file_key: str,
     dsr_buffer: BytesIO,
-    content_type: str = "application/zip",
 ) -> None:
     """Stream DSR buffer to storage using smart-open streaming.
 
@@ -29,15 +28,12 @@ def stream_dsr_buffer_to_storage(
         bucket_name: Storage bucket name
         file_key: File key in storage
         dsr_buffer: Pre-generated DSR report buffer (BytesIO)
-        content_type: MIME type for the uploaded file (defaults to application/zip)
     """
     # Get the content from the buffer
     content = dsr_buffer.getvalue()
     try:
         # Use smart-open's streaming upload for efficient memory usage
-        with storage_client.stream_upload(
-            bucket_name, file_key, content_type=content_type
-        ) as upload_stream:
+        with storage_client.stream_upload(bucket_name, file_key) as upload_stream:
             upload_stream.write(content)
 
     except Exception as e:

fides/api/service/storage/streaming/s3/s3_storage_client.py

@@ -7,7 +7,7 @@ from typing import Any, Optional
 from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
-from fides.api.schemas.storage.storage import AWSAuthMethod, StorageSecrets
+from fides.api.schemas.storage.storage import AWSAuthMethod
 from fides.api.service.storage.s3 import create_presigned_url_for_s3
 from fides.api.service.storage.streaming.base_storage_client import BaseStorageClient
 from fides.api.util.aws_util import get_s3_client
@@ -20,17 +20,19 @@ class S3StorageClient(BaseStorageClient):
     generation for the smart-open storage client.
     """
 
-    def __init__(self, storage_secrets: dict[StorageSecrets, Any]):
+    def __init__(self, auth_method: str, storage_secrets: dict[str, Any]):
         """Initialize the storage client with secrets.
 
         Args:
-            storage_secrets: Provider-specific storage credentials and configuration using StorageSecrets enum keys
+            storage_secrets: Provider-specific storage credentials and configuration using string keys
+                           (e.g., "aws_access_key_id", "region_name") from format_secrets()
         """
         super().__init__(storage_secrets)
-        self.storage_secrets: dict[StorageSecrets, Any] = storage_secrets
+        self.storage_secrets: dict[str, Any] = storage_secrets
+        self.auth_method = auth_method
 
     def build_uri(self, bucket: str, key: str) -> str:
-        """Build the S3 URI for the storage location.
+        """Build S3 URI for the given bucket and key.
 
         Args:
             bucket: S3 bucket name
@@ -45,26 +47,75 @@ class S3StorageClient(BaseStorageClient):
 
     def get_transport_params(self) -> dict[str, Any]:
         """Get S3-specific transport parameters for smart-open.
+        Type annotation: get_s3_client returns a boto3 S3 client object, not a Session
+        This is what smart-open expects for the "client" transport parameter
 
         Returns:
             Dictionary of S3 transport parameters for smart-open
         """
-        params = {}
-
-        if StorageSecrets.AWS_ACCESS_KEY_ID in self.storage_secrets:
-            params["access_key"] = self.storage_secrets[
-                StorageSecrets.AWS_ACCESS_KEY_ID
-            ]
-        if StorageSecrets.AWS_SECRET_ACCESS_KEY in self.storage_secrets:
-            params["secret_key"] = self.storage_secrets[
-                StorageSecrets.AWS_SECRET_ACCESS_KEY
-            ]
-        if StorageSecrets.REGION_NAME in self.storage_secrets:
-            params["region"] = self.storage_secrets[StorageSecrets.REGION_NAME]
-        if StorageSecrets.AWS_ASSUME_ROLE in self.storage_secrets:
-            params["assume_role_arn"] = self.storage_secrets[
-                StorageSecrets.AWS_ASSUME_ROLE
-            ]
+        params: dict[str, Any] = {}
+
+        # Create S3 client for smart-open
+        try:
+            # Determine auth method based on available credentials
+            if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
+
+                # For automatic authentication, check if region is available
+                if not self.storage_secrets.get("region_name", None):
+                    logger.warning(
+                        "No region specified in storage secrets for automatic authentication"
+                        "This may cause credential issues - consider setting a default region"
+                    )
+
+            # Extract assume_role_arn if present
+            assume_role_arn = None
+            if (
+                "assume_role_arn" in self.storage_secrets
+                and self.storage_secrets["assume_role_arn"]
+            ):
+                assume_role_arn = self.storage_secrets["assume_role_arn"]
+                logger.debug(f"Using assume role ARN: {assume_role_arn}")
+
+            # Create S3 client using existing utility
+            # get_s3_client returns a boto3 S3 client, not a Session
+            s3_client: Any = None
+            try:
+                s3_client = get_s3_client(
+                    self.auth_method, self.storage_secrets, assume_role_arn  # type: ignore
+                )
+                logger.debug("Successfully created S3 client")
+            except Exception as e:
+                # For automatic authentication, try to provide more helpful error messages
+                if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
+                    logger.error(
+                        f"Failed to create S3 client with automatic authentication: {e}. "
+                        "This usually means AWS credentials are not available in the environment"
+                        "Please ensure AWS credentials are configured via environment variables, IAM roles, or AWS profiles"
+                    )
+                    raise ValueError(
+                        f"Automatic AWS authentication failed: {e}. Please check your AWS credential configuration."
+                    )
+                raise
+
+            params["client"] = s3_client
+
+        except Exception as e:
+            logger.error(f"Failed to create S3 client for smart-open: {e}")
+            raise
+
+        # Include credentials at top level for compatibility
+        # Note: When using an S3 client, these credential parameters are not needed
+        # and will be ignored by smart-open, causing warnings
+        # Only include them if no S3 client is provided (fallback scenario)
+        if not params.get("client"):
+            for key, transport_key in [
+                ("aws_access_key_id", "access_key"),
+                ("aws_secret_access_key", "secret_key"),
+                ("region_name", "region"),
+                ("assume_role_arn", "assume_role_arn"),
+            ]:
+                if key in self.storage_secrets and self.storage_secrets[key]:
+                    params[transport_key] = self.storage_secrets[key]
 
         return params
 
@@ -85,28 +136,15 @@ class S3StorageClient(BaseStorageClient):
             Exception: If presigned URL generation fails
         """
         try:
-            # Storage secrets are already in the right format for get_s3_client
-            # get_s3_client expects dict[StorageSecrets, Any] with enum keys
-            s3_secrets = self.storage_secrets
-
-            # Determine auth method based on available credentials
-            # If AWS credentials are present, use SECRET_KEYS, otherwise use AUTOMATIC
-            if (
-                StorageSecrets.AWS_ACCESS_KEY_ID in self.storage_secrets
-                and StorageSecrets.AWS_SECRET_ACCESS_KEY in self.storage_secrets
-                and self.storage_secrets[StorageSecrets.AWS_ACCESS_KEY_ID]
-                and self.storage_secrets[StorageSecrets.AWS_SECRET_ACCESS_KEY]
-            ):
-                auth_method = AWSAuthMethod.SECRET_KEYS.value
-            else:
-                auth_method = AWSAuthMethod.AUTOMATIC.value
-
             # Extract assume_role_arn if present for role assumption
             assume_role_arn = None
-            if StorageSecrets.AWS_ASSUME_ROLE in self.storage_secrets:
-                assume_role_arn = self.storage_secrets[StorageSecrets.AWS_ASSUME_ROLE]
+            if "assume_role_arn" in self.storage_secrets:
+                assume_role_arn = self.storage_secrets["assume_role_arn"]
 
-            s3_client = get_s3_client(auth_method, s3_secrets, assume_role_arn)
+            # get_s3_client returns a boto3 S3 client, not a Session
+            s3_client: Any = get_s3_client(
+                self.auth_method, self.storage_secrets, assume_role_arn  # type: ignore
+            )
             return create_presigned_url_for_s3(s3_client, bucket, key, ttl_seconds)
         except Exception as e:
             logger.error(f"Failed to generate S3 presigned URL: {e}")

fides/api/service/storage/streaming/s3/streaming_s3.py

@@ -54,7 +54,8 @@ def upload_to_s3_streaming(
         return response
 
     try:
-        storage_client = SmartOpenStorageClient("s3", formatted_secrets)
+        logger.debug("Creating SmartOpenStorageClient with formatted secrets")
+        storage_client = SmartOpenStorageClient("s3", auth_method, formatted_secrets)
 
         # Create upload config for the streaming interface
         upload_config = StorageUploadConfig(
@@ -92,6 +93,10 @@ def _process_storage_secrets_input(
     """Process input and convert to string-keyed dictionary."""
     final_secrets: dict[str, Any] = {}
 
+    logger.debug(
+        f"Processing storage secrets input of type: {type(storage_secrets).__name__}"
+    )
+
     if isinstance(storage_secrets, StorageSecretsS3):
         # Convert StorageSecretsS3 model directly to string keys
         for key, value in storage_secrets.model_dump().items():
@@ -133,6 +138,7 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
     else:
         # AUTOMATIC authentication - check if region is provided
         has_region = "region_name" in final_secrets and final_secrets["region_name"]
+
         if not has_region:
             raise ValueError(
                 "Missing required region_name for AUTOMATIC authentication. "
@@ -140,13 +146,6 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
             )
 
 
-def _set_default_region(final_secrets: dict[str, Any]) -> None:
-    """Set default region if missing."""
-    if "region_name" not in final_secrets or not final_secrets["region_name"]:
-        logger.debug("Setting default region to 'us-east-1'")
-        final_secrets["region_name"] = "us-east-1"
-
-
 def format_secrets(
     storage_secrets: Union[StorageSecretsS3, dict[StorageSecrets, Any]]  # type: ignore[misc]
 ) -> dict[str, Any]:
@@ -156,8 +155,8 @@ def format_secrets(
     This function handles multiple credential formats and processes them through several stages:
     1. Input processing: Accepts either StorageSecretsS3 models (from API) or raw dicts (from database)
     2. Key normalization: Converts all keys to string format for consistency
-    3. Validation: Ensures required AWS credentials are present based on auth method
-    4. Default setting: Sets default region if missing (after validation)
+    3. Default setting: Sets default region if missing (before validation for better automatic auth support)
+    4. Validation: Ensures required AWS credentials are present based on auth method
     5. Return: Returns string-keyed dict ready for S3StorageClient
 
     Input formats:
@@ -179,18 +178,7 @@ def format_secrets(
     Raises:
         ValueError: If required AWS credentials are missing for the chosen auth method
     """
-    logger.debug("format_secrets called with type: {}", type(storage_secrets).__name__)
-
-    # Stage 1: Process input and create final format directly
     final_secrets = _process_storage_secrets_input(storage_secrets)
-
-    # Stage 2: Validate required credentials BEFORE setting defaults
     _validate_aws_credentials(final_secrets)
 
-    # Stage 3: Set default region if missing (after validation)
-    _set_default_region(final_secrets)
-
-    logger.debug(
-        "format_secrets completed successfully with {} keys", len(final_secrets)
-    )
     return final_secrets

fides/api/service/storage/streaming/smart_open_client.py

@@ -29,7 +29,12 @@ class SmartOpenStorageClient:
 
     min_part_size: int = MIN_PART_SIZE
 
-    def __init__(self, storage_type: str, storage_secrets: Any):
+    def __init__(
+        self,
+        storage_type: str,
+        auth_method: Optional[str],
+        storage_secrets: Any,
+    ):
         """Initialize the smart-open storage client.
 
         Args:
@@ -38,9 +43,10 @@ class SmartOpenStorageClient:
                            Will be passed to the specific storage client implementation.
         """
         self.storage_type = StorageClientFactory._normalize_storage_type(storage_type)
+        self.auth_method = auth_method
         self.storage_secrets = storage_secrets
         self._provider_client = StorageClientFactory.create_client(
-            storage_type, storage_secrets
+            storage_type, auth_method, storage_secrets
         )
 
     def _build_uri(self, bucket: str, key: str) -> str:
@@ -216,7 +222,6 @@ class SmartOpenStorageClient:
         self,
         bucket: str,
         key: str,
-        content_type: Optional[str] = None,
     ) -> Any:
         """Get a writable stream for uploading data.
 
@@ -225,7 +230,6 @@ class SmartOpenStorageClient:
         Args:
             bucket: Storage bucket/container name
             key: Object key/path
-            content_type: MIME type of the object
 
         Returns:
             Writable file-like object
@@ -233,9 +237,6 @@ class SmartOpenStorageClient:
         uri = self._build_uri(bucket, key)
         transport_params = self._get_transport_params()
 
-        if content_type:
-            transport_params["content_type"] = content_type
-
         return smart_open.open(uri, "wb", transport_params=transport_params)
 
     @retry_cloud_storage_operation(