ethyca-fides 2.69.1b1__py2.py3-none-any.whl → 2.69.1rc0__py2.py3-none-any.whl

fides/api/service/privacy_request/dsr_package/utils.py

@@ -0,0 +1,268 @@
+from typing import Any, List, Optional
+
+from fideslang.models import Dataset, DatasetField
+from loguru import logger
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import ActionType
+
+
+# TODO: keeping this for a bit to help with development and testing
+def get_redaction_entities_map(db: Session) -> set[str]:
+    """
+    Create a set of hierarchical entity keys that should be redacted based on fides_meta.redact: name.
+
+    This utility function reads all enabled dataset configurations from the database
+    and builds a set of hierarchical entity keys (dataset_name, dataset_name.collection_name,
+    dataset_name.collection_name.field_name) that have fides_meta.redact set to "name".
+
+    Supports deeply nested field structures with unlimited nesting depth.
+
+    Args:
+        db: Database session
+
+    Returns:
+        Set of hierarchical entity keys that should be redacted
+    """
+    redaction_entities = set()
+
+    try:
+        dataset_configs = DatasetConfig.all(db=db)
+
+        for dataset_config in dataset_configs:
+            ctl_dataset = dataset_config.ctl_dataset
+            if not ctl_dataset:
+                continue
+
+            dataset = Dataset.model_validate(dataset_config.ctl_dataset)
+            # Intentionally using the fides_key instead of name since it's always provided
+            dataset_name = dataset.fides_key
+
+            # Check dataset level
+            if dataset.fides_meta and dataset.fides_meta.redact == "name":
+                redaction_entities.add(dataset_name)
+
+            # Check collection level
+            for collection_dict in dataset.collections:
+                # Collections are stored as dictionaries in the database
+                collection_name = collection_dict.name
+                if not collection_name:
+                    continue
+
+                collection_path = f"{dataset_name}.{collection_name}"
+                collection_fides_meta = collection_dict.fides_meta
+
+                if collection_fides_meta and collection_fides_meta.redact == "name":
+                    redaction_entities.add(collection_path)
+
+                # Check field level (with recursive nested field support)
+                _traverse_fields_for_redaction(
+                    collection_dict.fields, collection_path, redaction_entities
+                )
+
+    except Exception as exc:
+        # Log error but don't fail, just return empty set
+        logger.warning(f"Error extracting redaction configurations: {exc}")
+
+    return redaction_entities
+
+
+def get_redaction_entities_map_db(db: Session) -> set[str]:
+    """
+    Create a set of hierarchical entity keys that should be redacted based on fides_meta.redact: name.
+
+    This function uses a hybrid approach:
+    1. First identifies datasets that contain ANY redaction metadata at any level
+    2. Then processes only those datasets with redaction metadata
+
+
+    Args:
+        db: Database session
+
+    Returns:
+        Set of hierarchical entity keys that should be redacted
+    """
+    redaction_entities: set[str] = set()
+
+    try:
+        # Step 1: Pre-filter to find datasets with ANY redaction metadata
+        # Simple existence check - no paths needed, just check if redaction exists anywhere
+        pre_filter_query = """
+        SELECT DISTINCT dc.ctl_dataset_id
+        FROM datasetconfig dc
+        JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+        WHERE
+            -- Dataset-level redaction
+            ds.fides_meta->>'redact' = 'name'
+            OR
+            -- Collection-level redaction
+            EXISTS (
+                SELECT 1 FROM jsonb_array_elements(ds.collections::jsonb) AS collection
+                WHERE collection->'fides_meta'->>'redact' = 'name'
+                LIMIT 1
+            )
+            OR
+            -- Field-level redaction using jsonb_path_query
+            EXISTS (
+                SELECT 1
+                FROM jsonb_path_query(ds.collections::jsonb, '$.**.fides_meta') AS fides_meta
+                WHERE fides_meta->>'redact' = 'name'
+                LIMIT 1
+            )
+        """
+
+        candidate_datasets = db.execute(pre_filter_query).fetchall()
+
+        if not candidate_datasets:
+            logger.debug("No datasets found with redaction metadata")
+            return redaction_entities
+
+        logger.debug(
+            f"Pre-filtered to {len(candidate_datasets)} datasets with redaction metadata"
+        )
+
+        # Step 2: Process only the candidate datasets with targeted queries
+        # Convert to a format we can use in SQL ANY clause
+        dataset_ids = [row[0] for row in candidate_datasets]
+
+        # Query for dataset-level redactions (only on candidate datasets)
+        dataset_query = text(
+            """
+        SELECT ds.fides_key as entity_path
+        FROM datasetconfig dc
+        JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+        WHERE ds.id = ANY(:dataset_ids)
+            AND ds.fides_meta->>'redact' = 'name'
+        """
+        )
+
+        dataset_results = db.execute(
+            dataset_query, {"dataset_ids": dataset_ids}
+        ).fetchall()
+        for row in dataset_results:
+            redaction_entities.add(row[0])
+
+        # Query for collection-level redactions (only on candidate datasets)
+        collection_query = text(
+            """
+        SELECT ds.fides_key || '.' || (collection->>'name') as entity_path
+        FROM datasetconfig dc
+        JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+        CROSS JOIN LATERAL jsonb_array_elements(ds.collections::jsonb) AS collection
+        WHERE ds.id = ANY(:dataset_ids)
+            AND collection->'fides_meta'->>'redact' = 'name'
+            AND collection->>'name' IS NOT NULL
+        """
+        )
+
+        collection_results = db.execute(
+            collection_query, {"dataset_ids": dataset_ids}
+        ).fetchall()
+        for row in collection_results:
+            redaction_entities.add(row[0])
+
+        # Query for field-level redactions (including nested fields)
+        # This uses a recursive CTE to handle arbitrary nesting levels
+        field_query = text(
+            """
+        WITH RECURSIVE field_hierarchy AS (
+            -- Base case: top-level fields in collections (only candidate datasets)
+            SELECT
+                ds.fides_key || '.' ||
+                    (collection->>'name') || '.' ||
+                    (field->>'name') as entity_path,
+                field->'fields' as nested_fields,
+                field->'fides_meta'->>'redact' as redact_value
+            FROM datasetconfig dc
+            JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+            CROSS JOIN LATERAL jsonb_array_elements(ds.collections::jsonb) AS collection
+            CROSS JOIN LATERAL jsonb_array_elements(collection->'fields') AS field
+            WHERE ds.id = ANY(:dataset_ids)
+                AND collection->>'name' IS NOT NULL
+                AND field->>'name' IS NOT NULL
+
+            UNION ALL
+
+            -- Recursive case: nested fields
+            SELECT
+                fh.entity_path || '.' || (nested_field->>'name') as entity_path,
+                nested_field->'fields' as nested_fields,
+                nested_field->'fides_meta'->>'redact' as redact_value
+            FROM field_hierarchy fh
+            CROSS JOIN LATERAL jsonb_array_elements(fh.nested_fields) AS nested_field
+            WHERE jsonb_typeof(fh.nested_fields) = 'array'
+                AND nested_field->>'name' IS NOT NULL
+        )
+        SELECT DISTINCT entity_path
+        FROM field_hierarchy
+        WHERE redact_value = 'name'
+        """
+        )
+
+        field_results = db.execute(field_query, {"dataset_ids": dataset_ids}).fetchall()
+        for row in field_results:
+            redaction_entities.add(row[0])
+
+        logger.debug(f"Found {len(redaction_entities)} entities requiring redaction")
+
+    except Exception as exc:
+        # Log error but don't fail, just return empty set
+        logger.warning(
+            f"Error extracting redaction configurations from database: {exc}"
+        )
+
+    return redaction_entities
+
+
+def map_privacy_request(privacy_request: PrivacyRequest) -> dict[str, Any]:
+    """Creates a map with a subset of values from the privacy request"""
+    request_data: dict[str, Any] = {}
+    request_data["id"] = privacy_request.id
+
+    action_type: Optional[ActionType] = privacy_request.policy.get_action_type()
+    if action_type:
+        request_data["type"] = action_type.value
+
+    request_data["identity"] = {
+        key: value
+        for key, value in privacy_request.get_persisted_identity()
+        .labeled_dict(include_default_labels=True)
+        .items()
+        if value["value"] is not None
+    }
+
+    if privacy_request.requested_at:
+        request_data["requested_at"] = privacy_request.requested_at.strftime(
+            "%m/%d/%Y %H:%M %Z"
+        )
+    return request_data
+
+
+def _traverse_fields_for_redaction(
+    fields: List[DatasetField], current_path: str, redaction_entities: set[str]
+) -> None:
+    """
+    Recursively traverse nested fields to find redaction entities.
+
+    Args:
+        fields: List of field dictionaries to traverse
+        current_path: Current hierarchical path (e.g., "dataset.collection")
+        redaction_entities: Set to add redacted field paths to
+    """
+    for field in fields:
+        field_name = field.name
+        if not field_name:
+            continue
+
+        field_path = f"{current_path}.{field_name}"
+        field_fides_meta = field.fides_meta
+
+        if field_fides_meta and field_fides_meta.redact == "name":
+            redaction_entities.add(field_path)
+
+        # Recursively check nested fields
+        if field.fields:
+            _traverse_fields_for_redaction(field.fields, field_path, redaction_entities)

fides/api/service/privacy_request/request_runner_service.py

@@ -312,8 +312,14 @@ def upload_and_save_access_results(  # pylint: disable=R0912
     loaded_attachments = [
         attachment
         for attachment in privacy_request.attachments
-        if AttachmentReferenceType.access_manual_webhook
-        not in [ref.reference_type for ref in attachment.references]
+        if not any(
+            ref.reference_type
+            in [
+                AttachmentReferenceType.access_manual_webhook,
+                AttachmentReferenceType.manual_task_submission,
+            ]
+            for ref in attachment.references
+        )
     ]
     attachments = get_attachments_content(loaded_attachments)
     # Process attachments once for both upload and storage

fides/api/service/privacy_request/request_service.py

@@ -540,6 +540,7 @@ def _get_request_task_ids_in_progress(
 
 
 # pylint: disable=too-many-branches
+# pylint: disable=too-many-statements
 @celery_app.task(base=DatabaseTask, bind=True)
 def requeue_interrupted_tasks(self: DatabaseTask) -> None:
     """
@@ -660,8 +661,24 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                             break
 
                         # If the task ID is not cached, we can't check if it's running
-                        # This means the subtask is stuck - cancel the entire privacy request
+                        # This means the subtask is stuck - but we need to handle this differently
+                        # based on the privacy request status
                         if not subtask_id:
+                            if (
+                                privacy_request.status
+                                == PrivacyRequestStatus.requires_input
+                            ):
+                                # For requires_input status, don't automatically error the request
+                                # as it's intentionally waiting for user input
+                                logger.warning(
+                                    f"No task ID found for request task {request_task_id} "
+                                    f"(privacy request {privacy_request.id}) in requires_input status - "
+                                    f"keeping request in current status as it may be waiting for manual input"
+                                )
+                                should_requeue = False
+                                break
+
+                            # For other statuses, cancel the entire privacy request
                             _cancel_interrupted_tasks_and_error_privacy_request(
                                 db,
                                 privacy_request,

fides/api/service/storage/storage_uploader_service.py

@@ -51,13 +51,7 @@ def upload(
         config.secrets is not None,
     )
 
-    if config.secrets:
-        logger.debug("Storage config secrets type: {}", type(config.secrets))
-        if isinstance(config.secrets, dict):
-            logger.debug("Storage config secrets keys: {}", list(config.secrets.keys()))
-        else:
-            logger.debug("Storage config secrets is not a dict: {}", config.secrets)
-    else:
+    if not config.secrets:
         logger.warning("Storage config has no secrets!")
 
     uploader: Any = _get_uploader_from_config_type(config.type)  # type: ignore
@@ -130,19 +124,6 @@ def _s3_uploader(
         config.secrets is not None,
     )
 
-    if config.secrets:
-        logger.debug(
-            "Config secrets keys: {}",
-            (
-                list(config.secrets.keys())
-                if isinstance(config.secrets, dict)
-                else "Not a dict"
-            ),
-        )
-        logger.debug("Config secrets type: {}", type(config.secrets))
-    else:
-        logger.warning("Config secrets is None or empty!")
-
     enable_streaming = config.details.get(StorageDetails.ENABLE_STREAMING.value, False)
     file_key: str = _construct_file_key(privacy_request.id, config, enable_streaming)
 
@@ -154,7 +135,6 @@ def _s3_uploader(
         file_key = f"{privacy_request.id}.zip"
         # Use streaming upload for better memory efficiency
         logger.debug("Using streaming S3 upload for {}", file_key)
-        logger.debug("Calling upload_to_s3_streaming with secrets: {}", config.secrets)
         return upload_to_s3_streaming(
             config.secrets,  # type: ignore
             data,

fides/api/service/storage/streaming/dsr_storage.py

@@ -17,7 +17,6 @@ def stream_dsr_buffer_to_storage(
     bucket_name: str,
     file_key: str,
     dsr_buffer: BytesIO,
-    content_type: str = "application/zip",
 ) -> None:
     """Stream DSR buffer to storage using smart-open streaming.
 
@@ -29,15 +28,12 @@ def stream_dsr_buffer_to_storage(
         bucket_name: Storage bucket name
         file_key: File key in storage
         dsr_buffer: Pre-generated DSR report buffer (BytesIO)
-        content_type: MIME type for the uploaded file (defaults to application/zip)
     """
     # Get the content from the buffer
     content = dsr_buffer.getvalue()
     try:
         # Use smart-open's streaming upload for efficient memory usage
-        with storage_client.stream_upload(
-            bucket_name, file_key, content_type=content_type
-        ) as upload_stream:
+        with storage_client.stream_upload(bucket_name, file_key) as upload_stream:
             upload_stream.write(content)
 
     except Exception as e:

fides/api/service/storage/streaming/s3/s3_storage_client.py

@@ -7,7 +7,7 @@ from typing import Any, Optional
 from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
-from fides.api.schemas.storage.storage import AWSAuthMethod, StorageSecrets
+from fides.api.schemas.storage.storage import AWSAuthMethod
 from fides.api.service.storage.s3 import create_presigned_url_for_s3
 from fides.api.service.storage.streaming.base_storage_client import BaseStorageClient
 from fides.api.util.aws_util import get_s3_client
@@ -20,17 +20,19 @@ class S3StorageClient(BaseStorageClient):
     generation for the smart-open storage client.
     """
 
-    def __init__(self, storage_secrets: dict[StorageSecrets, Any]):
+    def __init__(self, auth_method: str, storage_secrets: dict[str, Any]):
         """Initialize the storage client with secrets.
 
         Args:
-            storage_secrets: Provider-specific storage credentials and configuration using StorageSecrets enum keys
+            storage_secrets: Provider-specific storage credentials and configuration using string keys
+                           (e.g., "aws_access_key_id", "region_name") from format_secrets()
         """
         super().__init__(storage_secrets)
-        self.storage_secrets: dict[StorageSecrets, Any] = storage_secrets
+        self.storage_secrets: dict[str, Any] = storage_secrets
+        self.auth_method = auth_method
 
     def build_uri(self, bucket: str, key: str) -> str:
-        """Build the S3 URI for the storage location.
+        """Build S3 URI for the given bucket and key.
 
         Args:
             bucket: S3 bucket name
@@ -45,26 +47,75 @@ class S3StorageClient(BaseStorageClient):
 
     def get_transport_params(self) -> dict[str, Any]:
         """Get S3-specific transport parameters for smart-open.
+        Type annotation: get_s3_client returns a boto3 S3 client object, not a Session
+        This is what smart-open expects for the "client" transport parameter
 
         Returns:
             Dictionary of S3 transport parameters for smart-open
         """
-        params = {}
-
-        if StorageSecrets.AWS_ACCESS_KEY_ID in self.storage_secrets:
-            params["access_key"] = self.storage_secrets[
-                StorageSecrets.AWS_ACCESS_KEY_ID
-            ]
-        if StorageSecrets.AWS_SECRET_ACCESS_KEY in self.storage_secrets:
-            params["secret_key"] = self.storage_secrets[
-                StorageSecrets.AWS_SECRET_ACCESS_KEY
-            ]
-        if StorageSecrets.REGION_NAME in self.storage_secrets:
-            params["region"] = self.storage_secrets[StorageSecrets.REGION_NAME]
-        if StorageSecrets.AWS_ASSUME_ROLE in self.storage_secrets:
-            params["assume_role_arn"] = self.storage_secrets[
-                StorageSecrets.AWS_ASSUME_ROLE
-            ]
+        params: dict[str, Any] = {}
+
+        # Create S3 client for smart-open
+        try:
+            # Determine auth method based on available credentials
+            if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
+
+                # For automatic authentication, check if region is available
+                if not self.storage_secrets.get("region_name", None):
+                    logger.warning(
+                        "No region specified in storage secrets for automatic authentication"
+                        "This may cause credential issues - consider setting a default region"
+                    )
+
+            # Extract assume_role_arn if present
+            assume_role_arn = None
+            if (
+                "assume_role_arn" in self.storage_secrets
+                and self.storage_secrets["assume_role_arn"]
+            ):
+                assume_role_arn = self.storage_secrets["assume_role_arn"]
+                logger.debug(f"Using assume role ARN: {assume_role_arn}")
+
+            # Create S3 client using existing utility
+            # get_s3_client returns a boto3 S3 client, not a Session
+            s3_client: Any = None
+            try:
+                s3_client = get_s3_client(
+                    self.auth_method, self.storage_secrets, assume_role_arn  # type: ignore
+                )
+                logger.debug("Successfully created S3 client")
+            except Exception as e:
+                # For automatic authentication, try to provide more helpful error messages
+                if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
+                    logger.error(
+                        f"Failed to create S3 client with automatic authentication: {e}. "
+                        "This usually means AWS credentials are not available in the environment"
+                        "Please ensure AWS credentials are configured via environment variables, IAM roles, or AWS profiles"
+                    )
+                    raise ValueError(
+                        f"Automatic AWS authentication failed: {e}. Please check your AWS credential configuration."
+                    )
+                raise
+
+            params["client"] = s3_client
+
+        except Exception as e:
+            logger.error(f"Failed to create S3 client for smart-open: {e}")
+            raise
+
+        # Include credentials at top level for compatibility
+        # Note: When using an S3 client, these credential parameters are not needed
+        # and will be ignored by smart-open, causing warnings
+        # Only include them if no S3 client is provided (fallback scenario)
+        if not params.get("client"):
+            for key, transport_key in [
+                ("aws_access_key_id", "access_key"),
+                ("aws_secret_access_key", "secret_key"),
+                ("region_name", "region"),
+                ("assume_role_arn", "assume_role_arn"),
+            ]:
+                if key in self.storage_secrets and self.storage_secrets[key]:
+                    params[transport_key] = self.storage_secrets[key]
 
         return params
 
@@ -85,28 +136,15 @@ class S3StorageClient(BaseStorageClient):
             Exception: If presigned URL generation fails
         """
         try:
-            # Storage secrets are already in the right format for get_s3_client
-            # get_s3_client expects dict[StorageSecrets, Any] with enum keys
-            s3_secrets = self.storage_secrets
-
-            # Determine auth method based on available credentials
-            # If AWS credentials are present, use SECRET_KEYS, otherwise use AUTOMATIC
-            if (
-                StorageSecrets.AWS_ACCESS_KEY_ID in self.storage_secrets
-                and StorageSecrets.AWS_SECRET_ACCESS_KEY in self.storage_secrets
-                and self.storage_secrets[StorageSecrets.AWS_ACCESS_KEY_ID]
-                and self.storage_secrets[StorageSecrets.AWS_SECRET_ACCESS_KEY]
-            ):
-                auth_method = AWSAuthMethod.SECRET_KEYS.value
-            else:
-                auth_method = AWSAuthMethod.AUTOMATIC.value
-
             # Extract assume_role_arn if present for role assumption
             assume_role_arn = None
-            if StorageSecrets.AWS_ASSUME_ROLE in self.storage_secrets:
-                assume_role_arn = self.storage_secrets[StorageSecrets.AWS_ASSUME_ROLE]
+            if "assume_role_arn" in self.storage_secrets:
+                assume_role_arn = self.storage_secrets["assume_role_arn"]
 
-            s3_client = get_s3_client(auth_method, s3_secrets, assume_role_arn)
+            # get_s3_client returns a boto3 S3 client, not a Session
+            s3_client: Any = get_s3_client(
+                self.auth_method, self.storage_secrets, assume_role_arn  # type: ignore
+            )
             return create_presigned_url_for_s3(s3_client, bucket, key, ttl_seconds)
         except Exception as e:
             logger.error(f"Failed to generate S3 presigned URL: {e}")

fides/api/service/storage/streaming/s3/streaming_s3.py

@@ -54,7 +54,8 @@ def upload_to_s3_streaming(
         return response
 
     try:
-        storage_client = SmartOpenStorageClient("s3", formatted_secrets)
+        logger.debug("Creating SmartOpenStorageClient with formatted secrets")
+        storage_client = SmartOpenStorageClient("s3", auth_method, formatted_secrets)
 
         # Create upload config for the streaming interface
         upload_config = StorageUploadConfig(
@@ -92,6 +93,10 @@ def _process_storage_secrets_input(
     """Process input and convert to string-keyed dictionary."""
     final_secrets: dict[str, Any] = {}
 
+    logger.debug(
+        f"Processing storage secrets input of type: {type(storage_secrets).__name__}"
+    )
+
     if isinstance(storage_secrets, StorageSecretsS3):
         # Convert StorageSecretsS3 model directly to string keys
         for key, value in storage_secrets.model_dump().items():
@@ -133,6 +138,7 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
     else:
         # AUTOMATIC authentication - check if region is provided
         has_region = "region_name" in final_secrets and final_secrets["region_name"]
+
         if not has_region:
             raise ValueError(
                 "Missing required region_name for AUTOMATIC authentication. "
@@ -140,13 +146,6 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
             )
 
 
-def _set_default_region(final_secrets: dict[str, Any]) -> None:
-    """Set default region if missing."""
-    if "region_name" not in final_secrets or not final_secrets["region_name"]:
-        logger.debug("Setting default region to 'us-east-1'")
-        final_secrets["region_name"] = "us-east-1"
-
-
 def format_secrets(
     storage_secrets: Union[StorageSecretsS3, dict[StorageSecrets, Any]]  # type: ignore[misc]
 ) -> dict[str, Any]:
@@ -156,8 +155,8 @@ def format_secrets(
     This function handles multiple credential formats and processes them through several stages:
     1. Input processing: Accepts either StorageSecretsS3 models (from API) or raw dicts (from database)
     2. Key normalization: Converts all keys to string format for consistency
-    3. Validation: Ensures required AWS credentials are present based on auth method
-    4. Default setting: Sets default region if missing (after validation)
+    3. Default setting: Sets default region if missing (before validation for better automatic auth support)
+    4. Validation: Ensures required AWS credentials are present based on auth method
     5. Return: Returns string-keyed dict ready for S3StorageClient
 
     Input formats:
@@ -179,18 +178,7 @@ def format_secrets(
     Raises:
         ValueError: If required AWS credentials are missing for the chosen auth method
     """
-    logger.debug("format_secrets called with type: {}", type(storage_secrets).__name__)
-
-    # Stage 1: Process input and create final format directly
     final_secrets = _process_storage_secrets_input(storage_secrets)
-
-    # Stage 2: Validate required credentials BEFORE setting defaults
     _validate_aws_credentials(final_secrets)
 
-    # Stage 3: Set default region if missing (after validation)
-    _set_default_region(final_secrets)
-
-    logger.debug(
-        "format_secrets completed successfully with {} keys", len(final_secrets)
-    )
     return final_secrets

fides/api/service/storage/streaming/smart_open_client.py

@@ -29,7 +29,12 @@ class SmartOpenStorageClient:
 
     min_part_size: int = MIN_PART_SIZE
 
-    def __init__(self, storage_type: str, storage_secrets: Any):
+    def __init__(
+        self,
+        storage_type: str,
+        auth_method: Optional[str],
+        storage_secrets: Any,
+    ):
         """Initialize the smart-open storage client.
 
         Args:
@@ -38,9 +43,10 @@ class SmartOpenStorageClient:
                            Will be passed to the specific storage client implementation.
         """
         self.storage_type = StorageClientFactory._normalize_storage_type(storage_type)
+        self.auth_method = auth_method
         self.storage_secrets = storage_secrets
         self._provider_client = StorageClientFactory.create_client(
-            storage_type, storage_secrets
+            storage_type, auth_method, storage_secrets
         )
 
     def _build_uri(self, bucket: str, key: str) -> str:
@@ -216,7 +222,6 @@ class SmartOpenStorageClient:
         self,
         bucket: str,
         key: str,
-        content_type: Optional[str] = None,
     ) -> Any:
         """Get a writable stream for uploading data.
 
@@ -225,7 +230,6 @@ class SmartOpenStorageClient:
         Args:
             bucket: Storage bucket/container name
             key: Object key/path
-            content_type: MIME type of the object
 
         Returns:
             Writable file-like object
@@ -233,9 +237,6 @@ class SmartOpenStorageClient:
         uri = self._build_uri(bucket, key)
         transport_params = self._get_transport_params()
 
-        if content_type:
-            transport_params["content_type"] = content_type
-
         return smart_open.open(uri, "wb", transport_params=transport_params)
 
     @retry_cloud_storage_operation(