ethyca-fides 2.63.3rc0__py2.py3-none-any.whl → 2.64.1b0__py2.py3-none-any.whl

fides/api/service/storage/s3.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import IO, Any, Dict, Tuple, Union
+from io import BytesIO
+from typing import IO, Any, Dict, Optional, Tuple, Union
 
 from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, ParamValidationError
@@ -34,7 +35,7 @@ def maybe_get_s3_client(
 
 
 def create_presigned_url_for_s3(
-    s3_client: Any, bucket_name: str, file_key: str
+    s3_client: Any, bucket_name: str, file_key: str, ttl_seconds: Optional[int] = None
 ) -> AnyHttpUrlString:
     """
     Generates a presigned URL to share an S3 object
@@ -45,10 +46,16 @@ def create_presigned_url_for_s3(
     :return: Presigned URL as string.
     """
     params = {"Bucket": bucket_name, "Key": file_key}
+    if ttl_seconds:
+        if ttl_seconds > 604800:
+            raise ValueError("TTL must be less than 7 days")
+        expires_in = ttl_seconds
+    else:
+        expires_in = CONFIG.security.subject_request_download_link_ttl_seconds
     response = s3_client.generate_presigned_url(
         "get_object",
         Params=params,
-        ExpiresIn=CONFIG.security.subject_request_download_link_ttl_seconds,
+        ExpiresIn=expires_in,
     )
 
     # The response contains the presigned URL
@@ -68,7 +75,7 @@ def generic_upload_to_s3(  # pylint: disable=R0913
     file_key: str,
     auth_method: str,
     document: IO[bytes],
-    size_threshold: int = LARGE_FILE_THRESHOLD,  # 5 MB threshold
+    size_threshold: int = LARGE_FILE_THRESHOLD,  # 25 MB threshold
 ) -> Tuple[int, AnyHttpUrlString]:
     """
     Uploads file like objects to S3.
@@ -130,7 +137,8 @@ def generic_retrieve_from_s3(
     file_key: str,
     auth_method: str,
     get_content: bool = False,
-) -> Tuple[int, Union[str, bytes]]:
+    ttl_seconds: Optional[int] = None,
+) -> Tuple[int, Union[str, IO[bytes]]]:
     """
     Retrieves a file from S3 and returns its size and either a presigned URL or the actual content.
 
@@ -148,17 +156,23 @@ def generic_retrieve_from_s3(
     s3_client = get_s3_client(auth_method, storage_secrets)
 
     try:
-        if get_content:
-            # Get the actual content
-            response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
-            content = response["Body"].read()
-            return response["ContentLength"], content
+        # Get file size using head_object
+        size_response = s3_client.head_object(Bucket=bucket_name, Key=file_key)
+        # If the file is less than 25MB, we can get the content otherwise return the presigned URL
+        if get_content and size_response["ContentLength"] <= LARGE_FILE_THRESHOLD:
+            # Get the actual content using download_fileobj
+            file_obj = BytesIO()
+            s3_client.download_fileobj(
+                Bucket=bucket_name, Key=file_key, Fileobj=file_obj
+            )
+            file_obj.seek(0)  # Reset file pointer to beginning
+            return int(size_response["ContentLength"]), file_obj
 
         # Get presigned URL
-        presigned_url = create_presigned_url_for_s3(s3_client, bucket_name, file_key)
-        # Get file size
-        response = s3_client.head_object(Bucket=bucket_name, Key=file_key)
-        return int(response["ContentLength"]), str(presigned_url)
+        presigned_url = create_presigned_url_for_s3(
+            s3_client, bucket_name, file_key, ttl_seconds
+        )
+        return int(size_response["ContentLength"]), str(presigned_url)
     except ClientError as e:
         logger.error(f"Error retrieving file from S3: {e}")
         raise e

fides/api/service/storage/util.py

@@ -3,6 +3,10 @@ from enum import Enum as EnumType
 
 from loguru import logger
 
+# This is the max file size for downloading the content of an attachment.
+# This is an industry standard used by companies like Google and Microsoft.
+LARGE_FILE_THRESHOLD = 25 * 1024 * 1024  # 25 MB
+
 
 class AllowedFileType(EnumType):
     """
@@ -24,15 +28,49 @@ class AllowedFileType(EnumType):
 
 LOCAL_FIDES_UPLOAD_DIRECTORY = "fides_uploads"
 
-# Default to 10MB if not specified in environment
-LARGE_FILE_THRESHOLD = 10 * 1024 * 1024  # 10 MB threshold
-
 
 def get_local_filename(file_key: str) -> str:
-    """Verifies that the local storage directory exists and returns the local filepath"""
-    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
-        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
-    return f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"
+    """Verifies that the local storage directory exists and returns the local filepath.
+
+    This extra security checks are to prevent directory traversal attacks and "complete business and technical destruction".
+    Thanks Claude.
+
+    Args:
+        file_key: The key/path for the file
+
+    Returns:
+        The full local filepath
+
+    Raises:
+        ValueError: If the file_key is invalid or would result in a path outside the upload directory
+    """
+    # Basic validation
+    if not file_key:
+        raise ValueError("File key cannot be empty")
+
+    # Security checks before normalization
+    if file_key.startswith("/"):
+        raise ValueError("Invalid file key: cannot start with '/'")
+
+    # Normalize the path to handle any path separators consistently
+    # First normalize using os.path.normpath to handle any redundant separators
+    normalized_key = os.path.normpath(file_key)
+    # Then convert all separators to forward slashes for consistency
+    normalized_key = normalized_key.replace("\\", "/")
+
+    # Additional security: ensure the final path is within the upload directory
+    final_path = os.path.join(LOCAL_FIDES_UPLOAD_DIRECTORY, normalized_key)
+    if not os.path.abspath(final_path).startswith(
+        os.path.abspath(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    ):
+        raise ValueError(
+            "Invalid file key: would result in path outside upload directory"
+        )
+
+    # Create all necessary directories
+    os.makedirs(os.path.dirname(final_path), exist_ok=True)
+
+    return final_path
 
 
 def get_allowed_file_type_or_raise(file_key: str) -> str:

fides/api/tasks/csv_utils.py

@@ -0,0 +1,170 @@
+import zipfile
+from io import BytesIO
+from typing import Any, Optional
+
+import pandas as pd
+
+from fides.api.tasks.encryption_utils import encrypt_access_request_results
+from fides.config import CONFIG
+
+
+def create_csv_from_dataframe(df: pd.DataFrame) -> BytesIO:
+    """Create a CSV file from a pandas DataFrame.
+
+    Args:
+        df: The DataFrame to convert to CSV
+
+    Returns:
+        BytesIO: A file-like object containing the CSV data
+    """
+    buffer = BytesIO()
+    df.to_csv(buffer, index=False, encoding=CONFIG.security.encoding)
+    buffer.seek(0)
+    return buffer
+
+
+def create_attachment_csv(attachments: list[dict[str, Any]]) -> Optional[BytesIO]:
+    """Create a CSV file containing attachment metadata.
+
+    Args:
+        attachments: List of attachment dictionaries
+        privacy_request_id: The ID of the privacy request for encryption
+
+    Returns:
+        Optional[BytesIO]: A file-like object containing the CSV data, or None if no attachments
+    """
+    if not attachments:
+        return None
+
+    # Filter out invalid attachments and create a list of valid ones
+    valid_attachments = []
+    for a in attachments:
+        if not isinstance(a, dict):
+            continue
+
+        # Check if the attachment has at least one of the required fields
+        if not any(
+            key in a
+            for key in ["file_name", "file_size", "content_type", "download_url"]
+        ):
+            continue
+
+        valid_attachments.append(
+            {
+                "file_name": a.get("file_name", ""),
+                "file_size": a.get("file_size", 0),
+                "content_type": a.get("content_type", "application/octet-stream"),
+                "download_url": a.get("download_url", ""),
+            }
+        )
+
+    # Return None if there are no valid attachments
+    if not valid_attachments:
+        return None
+
+    df = pd.DataFrame(valid_attachments)
+
+    if df.empty:
+        return None
+
+    return create_csv_from_dataframe(df)
+
+
+def _write_attachment_csv(
+    zip_file: zipfile.ZipFile,
+    key: str,
+    idx: int,
+    attachments: list[dict[str, Any]],
+    privacy_request_id: str,
+) -> None:
+    """Write attachment data to a CSV file in the zip archive.
+
+    Args:
+        zip_file: The zip file to write to
+        key: The key for the data
+        idx: The index of the item in the list
+        attachments: List of attachment dictionaries
+        privacy_request_id: The ID of the privacy request for encryption
+    """
+    buffer = create_attachment_csv(attachments)
+    if buffer:
+        zip_file.writestr(
+            f"{key}/{idx + 1}/attachments.csv",
+            encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
+        )
+
+
+def _write_item_csv(
+    zip_file: zipfile.ZipFile,
+    key: str,
+    items: list[dict[str, Any]],
+    privacy_request_id: str,
+) -> None:
+    """Write item data to a CSV file in the zip archive.
+
+    Args:
+        zip_file: The zip file to write to
+        key: The key for the data
+        items: List of items to write
+        privacy_request_id: The ID of the privacy request for encryption
+    """
+    if items:
+        df = pd.DataFrame(items)
+        buffer = create_csv_from_dataframe(df)
+        zip_file.writestr(
+            f"{key}.csv",
+            encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
+        )
+
+
+def _write_simple_csv(
+    zip_file: zipfile.ZipFile,
+    key: str,
+    value: Any,
+    privacy_request_id: str,
+) -> None:
+    """Write simple key-value data to a CSV file in the zip archive.
+
+    Args:
+        zip_file: The zip file to write to
+        key: The key for the data
+        value: The value to write
+        privacy_request_id: The ID of the privacy request for encryption
+    """
+    df = pd.json_normalize({key: value})
+    buffer = create_csv_from_dataframe(df)
+    zip_file.writestr(
+        f"{key}.csv",
+        encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
+    )
+
+
+def write_csv_to_zip(
+    zip_file: zipfile.ZipFile, data: dict[str, Any], privacy_request_id: str
+) -> None:
+    """Write data to a zip file in CSV format.
+
+    Args:
+        zip_file: The zip file to write to
+        data: The data to convert to CSV
+        privacy_request_id: The ID of the privacy request for encryption
+    """
+    for key, value in data.items():
+        if (
+            isinstance(value, list)
+            and value
+            and all(isinstance(item, dict) for item in value)
+        ):
+            # Handle lists of dictionaries
+            items: list[dict[str, Any]] = []
+            for item in value:
+                # Extract attachments if they exist
+                attachments = item.pop("attachments", [])
+                if attachments:
+                    _write_attachment_csv(
+                        zip_file, key, len(items), attachments, privacy_request_id
+                    )
+                items.append(item)
+            _write_item_csv(zip_file, key, items, privacy_request_id)
+        else:
+            _write_simple_csv(zip_file, key, value, privacy_request_id)

fides/api/tasks/encryption_utils.py

@@ -0,0 +1,42 @@
+import secrets
+from typing import Optional, Union
+
+from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
+from fides.api.util.cache import get_cache, get_encryption_cache_key
+from fides.api.util.encryption.aes_gcm_encryption_scheme import (
+    encrypt_to_bytes_verify_secrets_length,
+)
+from fides.config import CONFIG
+
+
+def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> str:
+    """Encrypt data with encryption key if provided, otherwise return unencrypted data.
+
+    Args:
+        data: The data to encrypt
+        request_id: The ID of the privacy request for encryption key lookup
+
+    Returns:
+        str: The encrypted data as a string
+    """
+    cache = get_cache()
+    encryption_cache_key = get_encryption_cache_key(
+        privacy_request_id=request_id,
+        encryption_attr="key",
+    )
+    if isinstance(data, bytes):
+        data = data.decode(CONFIG.security.encoding)
+
+    encryption_key: Optional[str] = cache.get(encryption_cache_key)
+    if not encryption_key:
+        return data
+
+    bytes_encryption_key: bytes = encryption_key.encode(
+        encoding=CONFIG.security.encoding
+    )
+    nonce: bytes = secrets.token_bytes(CONFIG.security.aes_gcm_nonce_length)
+    # b64encode the entire nonce and the encrypted message together
+    return bytes_to_b64_str(
+        nonce
+        + encrypt_to_bytes_verify_secrets_length(data, bytes_encryption_key, nonce)
+    )

fides/api/tasks/storage.py

@@ -1,23 +1,20 @@
 from __future__ import annotations
 
 import json
-import secrets
 import zipfile
 from io import BytesIO
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional
 
-import pandas as pd
 from botocore.exceptions import ClientError, ParamValidationError
 from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
 from fides.api.common_exceptions import StorageUploadError
-from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
 from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
 from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
     DsrReportBuilder,
 )
-from fides.api.service.storage.gcs import get_gcs_client
+from fides.api.service.storage.gcs import get_gcs_blob
 from fides.api.service.storage.s3 import (
     create_presigned_url_for_s3,
     generic_upload_to_s3,
@@ -26,11 +23,9 @@ from fides.api.service.storage.util import (
     LOCAL_FIDES_UPLOAD_DIRECTORY,
     get_local_filename,
 )
+from fides.api.tasks.csv_utils import write_csv_to_zip
+from fides.api.tasks.encryption_utils import encrypt_access_request_results
 from fides.api.util.aws_util import get_s3_client
-from fides.api.util.cache import get_cache, get_encryption_cache_key
-from fides.api.util.encryption.aes_gcm_encryption_scheme import (
-    encrypt_to_bytes_verify_secrets_length,
-)
 from fides.api.util.storage_util import StorageJSONEncoder
 from fides.config import CONFIG
 
@@ -38,33 +33,8 @@ if TYPE_CHECKING:
     from fides.api.models.privacy_request import PrivacyRequest
 
 
-def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> str:
-    """Encrypt data with encryption key if provided, otherwise return unencrypted data"""
-    cache = get_cache()
-    encryption_cache_key = get_encryption_cache_key(
-        privacy_request_id=request_id,
-        encryption_attr="key",
-    )
-    if isinstance(data, bytes):
-        data = data.decode(CONFIG.security.encoding)
-
-    encryption_key: str | None = cache.get(encryption_cache_key)
-    if not encryption_key:
-        return data
-
-    bytes_encryption_key: bytes = encryption_key.encode(
-        encoding=CONFIG.security.encoding
-    )
-    nonce: bytes = secrets.token_bytes(CONFIG.security.aes_gcm_nonce_length)
-    # b64encode the entire nonce and the encrypted message together
-    return bytes_to_b64_str(
-        nonce
-        + encrypt_to_bytes_verify_secrets_length(data, bytes_encryption_key, nonce)
-    )
-
-
 def write_to_in_memory_buffer(
-    resp_format: str, data: Dict[str, Any], privacy_request: PrivacyRequest
+    resp_format: str, data: dict[str, Any], privacy_request: PrivacyRequest
 ) -> BytesIO:
     """Write JSON/CSV data to in-memory file-like object to be passed to S3 or GCS. Encrypt data if encryption key/nonce
     has been cached for the given privacy request id
@@ -73,46 +43,62 @@ def write_to_in_memory_buffer(
     :param data: Dict
     :param request_id: str, The privacy request id
     """
+
     logger.debug("Writing data to in-memory buffer")
+    try:
+        if resp_format == ResponseFormat.html.value:
+            return DsrReportBuilder(
+                privacy_request=privacy_request,
+                dsr_data=data,
+            ).generate()
+
+        if resp_format == ResponseFormat.json.value:
+            return convert_dict_to_encrypted_json(data, privacy_request.id)
+
+        if resp_format == ResponseFormat.csv.value:
+            zipped_csvs = BytesIO()
+            with zipfile.ZipFile(zipped_csvs, "w") as f:
+                write_csv_to_zip(f, data, privacy_request.id)
+            zipped_csvs.seek(0)
+            return zipped_csvs
+    except Exception as e:
+        logger.error(f"Error writing data to in-memory buffer: {str(e)}")
+        raise e
+
+    raise NotImplementedError(f"No handling for response format {resp_format}.")
+
+
+def convert_dict_to_encrypted_json(
+    data: dict[str, Any], privacy_request_id: str
+) -> BytesIO:
+    """Convert data to JSON and encrypt it.
+
+    Args:
+        data: The data to convert and encrypt
+        privacy_request_id: The ID of the privacy request for encryption
 
-    if resp_format == ResponseFormat.json.value:
+    Returns:
+        BytesIO: A file-like object containing the encrypted JSON data
+
+    Raises:
+        Exception: If JSON conversion fails
+    """
+    try:
         json_str = json.dumps(data, indent=2, default=StorageJSONEncoder().default)
         return BytesIO(
-            encrypt_access_request_results(json_str, privacy_request.id).encode(
+            encrypt_access_request_results(json_str, privacy_request_id).encode(
                 CONFIG.security.encoding
             )
         )
-
-    if resp_format == ResponseFormat.csv.value:
-        zipped_csvs = BytesIO()
-        with zipfile.ZipFile(zipped_csvs, "w") as f:
-            for key in data:
-                df = pd.json_normalize(data[key])
-                buffer = BytesIO()
-                df.to_csv(buffer, index=False, encoding=CONFIG.security.encoding)
-                buffer.seek(0)
-                f.writestr(
-                    f"{key}.csv",
-                    encrypt_access_request_results(
-                        buffer.getvalue(), privacy_request.id
-                    ),
-                )
-
-        zipped_csvs.seek(0)
-        return zipped_csvs
-
-    if resp_format == ResponseFormat.html.value:
-        return DsrReportBuilder(
-            privacy_request=privacy_request,
-            dsr_data=data,
-        ).generate()
-
-    raise NotImplementedError(f"No handling for response format {resp_format}.")
+    except Exception as e:
+        logger.error(f"Error converting data to JSON: {str(e)}")
+        logger.error(f"Data that failed to convert: {data}")
+        raise
 
 
 def upload_to_s3(  # pylint: disable=R0913
-    storage_secrets: Dict[StorageSecrets, Any],
-    data: Dict,
+    storage_secrets: dict[StorageSecrets, Any],
+    data: dict,
     bucket_name: str,
     file_key: str,
     resp_format: str,
@@ -140,18 +126,22 @@ def upload_to_s3(  # pylint: disable=R0913
                 "storage", {}
             ).get("aws_s3_assume_role_arn"),
         )
+    except (ClientError, ParamValidationError) as e:
+        logger.error(f"Error getting s3 client: {str(e)}")
+        raise StorageUploadError(f"Error getting s3 client: {str(e)}")
 
-        # handles file chunking
-        try:
-            s3_client.upload_fileobj(
-                Fileobj=write_to_in_memory_buffer(resp_format, data, privacy_request),
-                Bucket=bucket_name,
-                Key=file_key,
-            )
-        except Exception as e:
-            logger.error("Encountered error while uploading s3 object: {}", e)
-            raise e
+    # handles file chunking
+    try:
+        s3_client.upload_fileobj(
+            Fileobj=write_to_in_memory_buffer(resp_format, data, privacy_request),
+            Bucket=bucket_name,
+            Key=file_key,
+        )
+    except ClientError as e:
+        logger.error("Encountered error while uploading s3 object: {}", e)
+        raise StorageUploadError(f"Error uploading to S3: {e}")
 
+    try:
         presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
             s3_client, bucket_name, file_key
         )
@@ -162,13 +152,11 @@ def upload_to_s3(  # pylint: disable=R0913
             "Encountered error while uploading and generating link for s3 object: {}", e
         )
         raise StorageUploadError(f"Error uploading to S3: {e}")
-    except ParamValidationError as e:
-        raise StorageUploadError(f"The parameters you provided are incorrect: {e}")
 
 
 def upload_to_gcs(
-    storage_secrets: Dict,
-    data: Dict,
+    storage_secrets: dict,
+    data: dict,
     bucket_name: str,
     file_key: str,
     resp_format: str,
@@ -177,24 +165,30 @@ def upload_to_gcs(
 ) -> str:
     """Uploads access request data to a Google Cloud Storage bucket"""
     logger.info("Starting Google Cloud Storage upload of {}", file_key)
+    content_type = {
+        ResponseFormat.json.value: "application/json",
+        ResponseFormat.csv.value: "application/zip",
+        ResponseFormat.html.value: "application/zip",
+    }
+
+    blob = get_gcs_blob(auth_method, storage_secrets, bucket_name, file_key)
+    in_memory_file = write_to_in_memory_buffer(resp_format, data, privacy_request)
 
     try:
-        storage_client = get_gcs_client(auth_method, storage_secrets)
-        bucket = storage_client.bucket(bucket_name)
-
-        blob = bucket.blob(file_key)
-        in_memory_file = write_to_in_memory_buffer(resp_format, data, privacy_request)
-        content_type = {
-            ResponseFormat.json.value: "application/json",
-            ResponseFormat.csv.value: "application/zip",
-            ResponseFormat.html.value: "application/zip",
-        }
         blob.upload_from_string(
             in_memory_file.getvalue(), content_type=content_type[resp_format]
         )
+    except Exception as e:
+        logger.error("Error uploading to GCS: {}", str(e))
+        logger.error(
+            "Encountered error while uploading and generating link for Google Cloud Storage object: {}",
+            e,
+        )
+        raise
 
-        logger.info("File {} uploaded to {}", file_key, blob.public_url)
+    logger.info("File {} uploaded to {}", file_key, blob.public_url)
 
+    try:
         presigned_url = blob.generate_signed_url(
             version="v4",
             expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
@@ -210,7 +204,7 @@ def upload_to_gcs(
 
 
 def upload_to_local(
-    data: Dict,
+    data: dict,
     file_key: str,
     privacy_request: PrivacyRequest,
     resp_format: str = ResponseFormat.json.value,

fides/api/util/cache.py

@@ -174,6 +174,7 @@ def _determine_redis_db_index(
        - If *not* running under xdist, always use DB 1.
 
     2. Non-test mode: return the value already present in `CONFIG.redis.db_index`
+    (or CONFIG.redis.read_only_db_index` if read_only is True)
     """
 
     # 1. Test mode logic
@@ -261,18 +262,20 @@ def get_read_only_cache() -> FidesopsRedis:
             ssl_ca_certs=CONFIG.redis.read_only_ssl_ca_certs,
             ssl_cert_reqs=CONFIG.redis.read_only_ssl_cert_reqs,
         )
-        logger.debug("New read-only Redis connection created.")
 
     try:
+        # Test the connection by attempting to ping the Redis server
         connected = _read_only_connection.ping()
-        logger.debug("Read-only Redis connection succeeded.")
-    except ConnectionErrorFromRedis:
+        logger.debug("Read-only Redis connection established successfully.")
+    except Exception as e:
+        logger.error(f"Failed to connect to read-only Redis: {e}")
         connected = False
 
     if not connected:
         logger.error(
             "Unable to establish read-only Redis connection. Returning writeable cache connection instead."
         )
+        # If we can't connect to the read-only cache, fall back to the regular cache
         return get_cache()
 
     return _read_only_connection

fides/api/util/saas_util.py

@@ -28,6 +28,7 @@ ALL_OBJECT_FIELDS = "all_object_fields"
 CUSTOM_PRIVACY_REQUEST_FIELDS = "custom_privacy_request_fields"
 UUID = "uuid"
 ISO_8601_DATETIME = "iso_8601_datetime"
+FIELD_LIST = "field_list"
 REPLY_TO = "reply_to"
 REPLY_TO_TOKEN = "reply_to_token"