ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.69.2__py2.py3-none-any.whl

fides/api/service/storage/streaming/smart_open_streaming_storage.py

@@ -3,11 +3,12 @@ from __future__ import annotations
 
 import csv
 import json
+import time
 from datetime import datetime
 from io import BytesIO, StringIO
 from itertools import chain
 from typing import Any, Generator, Iterable, Optional, Tuple
-from urllib.parse import urlparse
+from urllib.parse import unquote, urlparse
 
 from fideslang.validation import AnyHttpUrlString
 from loguru import logger
@@ -25,10 +26,11 @@ from fides.api.service.storage.streaming.dsr_storage import (
 )
 from fides.api.service.storage.streaming.retry import retry_cloud_storage_operation
 from fides.api.service.storage.streaming.schemas import (
-    CHUNK_SIZE_THRESHOLD,
+    DEFAULT_CHUNK_SIZE,
+    MAX_FILE_SIZE,
     AttachmentInfo,
     AttachmentProcessingInfo,
-    PackageSplitConfig,
+    SmartOpenStreamingStorageConfig,
     StorageUploadConfig,
     StreamingBufferConfig,
 )
@@ -66,16 +68,22 @@ class SmartOpenStreamingStorage:
     def __init__(
         self,
         storage_client: SmartOpenStorageClient,
-        chunk_size: int = CHUNK_SIZE_THRESHOLD,
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
     ):
         """Initialize with a smart-open storage client.
 
         Args:
             storage_client: Smart-open based storage client
-            chunk_size: Size of chunks for streaming attachments (default: 8KB)
+            chunk_size: Size of chunks for streaming attachments (default: 5MB)
+
+        Raises:
+            ValidationError: If chunk_size is outside valid range (1KB - 2GB)
         """
+        # Validate parameters using Pydantic schema
+        config = SmartOpenStreamingStorageConfig(chunk_size=chunk_size)
+
         self.storage_client = storage_client
-        self.chunk_size = chunk_size
+        self.chunk_size = config.chunk_size
         # Track used filenames per dataset to match DSR report builder behavior
         # Maps dataset_name -> set of used filenames
         self.used_filenames_per_dataset: dict[str, set[str]] = {}
@@ -98,10 +106,15 @@ class SmartOpenStreamingStorage:
         Raises:
             ValueError: If URL cannot be parsed
         """
+        if storage_key is None or storage_key == "":
+            logger.error(f"Storage key cannot be empty: {storage_key}")
+            raise ValueError("Storage key cannot be empty")
+
         if storage_key.startswith("s3://"):
             # Extract bucket from S3 URL: s3://bucket/path
             parts = storage_key.split("/")
             if len(parts) < 4:
+                logger.error(f"Invalid S3 URL format: {storage_key}")
                 raise ValueError(f"Invalid S3 URL format: {storage_key}")
             return parts[2], "/".join(parts[3:])
 
@@ -111,16 +124,16 @@ class SmartOpenStreamingStorage:
             parts = clean_url.split(S3_AMAZONAWS_COM_DOMAIN)
             if len(parts) == 2:
                 bucket = parts[0].replace("https://", "").replace("http://", "")
-                key = parts[1].lstrip(
-                    "/"
-                )  # Strip leading forward slash for S3 compatibility
+                key = unquote(
+                    parts[1].lstrip("/")
+                )  # URL-decode and strip leading slash for S3 compatibility
                 return bucket, key
 
         # Handle generic HTTP(S) URLs
         if storage_key.startswith(("http://", "https://")):
             parsed = urlparse(storage_key)
             bucket = parsed.netloc
-            key = parsed.path.lstrip("/")
+            key = unquote(parsed.path.lstrip("/"))  # URL-decode the path
             return bucket, key
 
         raise ValueError(f"Could not parse storage URL: {storage_key}")
@@ -143,103 +156,6 @@ class SmartOpenStreamingStorage:
                 [content_bytes]
             )
 
-    def build_attachments_list(
-        self, data: dict, config: PackageSplitConfig
-    ) -> list[tuple[str, dict, int]]:
-        """
-        Build a list of attachments from the data.
-
-        Args:
-            data: The data to build the attachments list from
-            config: The configuration for package splitting
-
-        Returns:
-            A list of AttachmentInfo objects
-        """
-        attachments_list = []
-        for key, value in data.items():
-            if not isinstance(value, list):
-                continue
-
-            for item in value:
-                attachments = item.get("attachments", [])
-                if not isinstance(attachments, list):
-                    attachments = []
-
-                attachment_count = len(attachments)
-
-                # Only include items that have attachments
-                if attachment_count > 0:
-                    # If a single item has more attachments than the limit, we need to split it
-                    if attachment_count > config.max_attachments:
-                        # Split the item into multiple sub-items
-                        for i in range(0, attachment_count, config.max_attachments):
-                            sub_attachments = attachments[
-                                i : i + config.max_attachments
-                            ]
-                            sub_item = item.copy()
-                            sub_item["attachments"] = sub_attachments
-                            attachments_list.append(
-                                (key, sub_item, len(sub_attachments))
-                            )
-                    else:
-                        attachments_list.append((key, item, attachment_count))
-
-        return attachments_list
-
-    def split_data_into_packages(
-        self, data: dict, config: Optional[PackageSplitConfig] = None
-    ) -> list[dict]:
-        """Split large datasets into multiple smaller packages.
-
-        Uses a best-fit decreasing algorithm to optimize package distribution:
-        1. Sort items by attachment count (largest first)
-        2. Try to fit each item in the package with the most remaining space
-        3. Create new packages only when necessary
-        4. Handle items that exceed the max_attachments limit by splitting them
-
-        Args:
-            data: The data to split
-            config: Configuration for package splitting (defaults to PackageSplitConfig())
-
-        Returns:
-            List of data packages
-        """
-        # Use default config if none provided
-        if config is None:
-            config = PackageSplitConfig()
-
-        # Collect all items with their attachment counts
-        all_items = self.build_attachments_list(data, config)
-
-        # Sort by attachment count (largest first) for better space utilization
-        all_items.sort(key=lambda x: x[2], reverse=True)
-
-        packages: list[dict[str, Any]] = []
-        package_attachment_counts: list[int] = []
-
-        for key, item, attachment_count in all_items:
-            # Try to find a package with enough space
-            package_found = False
-
-            for i, current_count in enumerate(package_attachment_counts):
-                if current_count + attachment_count <= config.max_attachments:
-                    # Add to existing package
-                    if key not in packages[i]:
-                        packages[i][key] = []
-                    packages[i][key].append(item)
-                    package_attachment_counts[i] += attachment_count
-                    package_found = True
-                    break
-
-            if not package_found:
-                # Create new package - this item cannot fit in any existing package
-                new_package = {key: [item]}
-                packages.append(new_package)
-                package_attachment_counts.append(attachment_count)
-
-        return packages
-
     def _validate_attachment(
         self, attachment: dict
     ) -> Optional[AttachmentProcessingInfo]:
@@ -294,23 +210,75 @@ class SmartOpenStreamingStorage:
         Returns:
             Iterator that yields chunks of the attachment content
         """
+        for arg in [bucket, key, storage_key]:
+            if arg is None or arg == "":
+                logger.error(f"{arg} cannot be empty: {arg}")
+                raise ValueError(f"{arg} cannot be empty")
+
         try:
             with self.storage_client.stream_read(bucket, key) as content_stream:
                 # Stream in chunks instead of reading entire file
-                chunk_count = 0
-                total_bytes = 0
-                while True:
-                    chunk = content_stream.read(self.chunk_size)
+                chunk_count = total_bytes = 0
+                max_chunks = (
+                    MAX_FILE_SIZE // self.chunk_size + 1
+                )  # Safety limit to prevent infinite loops
+
+                size_based_timeout = MAX_FILE_SIZE // (10 * 1024 * 1024)  # 1s per 10MB
+                timeout = 300 + size_based_timeout  # 5 minutes base + 1s per 10MB
+                start_time = time.time()
+
+                # Log the calculated timeout for debugging
+                logger.debug(
+                    f"Starting stream for {storage_key} with timeout: {timeout}s "
+                    f"(base: 300s + size-based: {size_based_timeout}s)"
+                )
+
+                while chunk_count < max_chunks and total_bytes < MAX_FILE_SIZE:
+                    elapsed_time = time.time() - start_time
+                    if elapsed_time >= timeout:
+                        raise TimeoutError(
+                            f"Timeout reached ({timeout}s) while streaming attachment {storage_key}."
+                        )
+
+                    try:
+                        chunk = content_stream.read(self.chunk_size)
+                    except Exception as read_error:
+                        logger.error(
+                            f"Error reading chunk from stream for {storage_key}: {read_error}"
+                        )
+                        raise StorageUploadError(
+                            f"Stream read error for {storage_key}: {read_error}"
+                        ) from read_error
+
                     if not chunk:
+                        # End of stream reached normally
+                        logger.debug(
+                            f"Successfully streamed attachment {storage_key}: "
+                            f"{total_bytes} bytes in {chunk_count} chunks"
+                        )
                         break
+
                     chunk_count += 1
                     total_bytes += len(chunk)
                     yield chunk
 
+                # Log if we hit limits
+                if chunk_count >= max_chunks:
+                    logger.warning(
+                        f"Maximum chunk count ({max_chunks}) reached for attachment {storage_key}. "
+                        f"Streamed {total_bytes} bytes. Stream may be incomplete."
+                    )
+                elif total_bytes >= MAX_FILE_SIZE:
+                    logger.warning(
+                        f"Maximum file size ({MAX_FILE_SIZE} bytes) reached for attachment {storage_key}. "
+                        f"Streamed {total_bytes} bytes in {chunk_count} chunks. Stream may be incomplete."
+                    )
+
         except Exception as e:
-            logger.warning(f"Failed to stream attachment {storage_key}: {e}")
-            # Yield empty content on failure
-            yield b""
+            logger.error(f"Failed to stream attachment {storage_key}: {e}")
+            raise StorageUploadError(
+                f"Failed to stream attachment {storage_key}: {e}"
+            ) from e
 
     def _collect_and_validate_attachments(
         self, data: dict
@@ -332,6 +300,8 @@ class SmartOpenStreamingStorage:
         processed_attachments: dict[tuple[str, str], str] = {}
 
         # Use the shared contextual processing function
+        # Note: This method should only be used when DSR report builder is not available
+        # For HTML format, use _collect_and_validate_attachments_from_dsr_builder instead
         processed_attachments_list = process_attachments_contextually(
             data,
             used_filenames_data,
@@ -557,8 +527,9 @@ class SmartOpenStreamingStorage:
             raise StorageUploadError(f"Failed to generate DSR report: {e}") from e
 
         # Use the DSR report builder's processed attachments to avoid duplicates
+        # Use the redacted data from the DSR report builder instead of the original data
         all_attachments = self._collect_and_validate_attachments_from_dsr_builder(
-            data, dsr_builder
+            dsr_builder.dsr_data, dsr_builder
         )
 
         if not all_attachments:
@@ -595,7 +566,9 @@ class SmartOpenStreamingStorage:
         )
 
         # Create ZIP generator with attachment files
-        attachment_files_generator = self._create_attachment_files(all_attachments)
+        attachment_files_generator = self._create_attachment_files(
+            all_attachments, buffer_config
+        )
 
         # Combine both generators and stream the complete ZIP to storage
         combined_entries = chain(attachment_files_generator, dsr_files_generator)
@@ -673,6 +646,7 @@ class SmartOpenStreamingStorage:
             max_workers,
             batch_size,
             resp_format,
+            buffer_config,
         )
 
         # Use smart-open's streaming upload capability
@@ -718,6 +692,7 @@ class SmartOpenStreamingStorage:
         max_workers: int,
         batch_size: int,
         resp_format: str,
+        buffer_config: Optional[StreamingBufferConfig] = None,
     ) -> Generator[Tuple[str, datetime, int, Any, Iterable[bytes]], None, None]:
         """Create a generator for ZIP file contents including data and attachments.
 
@@ -743,7 +718,9 @@ class SmartOpenStreamingStorage:
             yield from self._convert_to_stream_zip_format(data_files_generator)
 
         # Then, yield attachment files (already in stream_zip format, stream directly)
-        attachment_files_generator = self._create_attachment_files(all_attachments)
+        attachment_files_generator = self._create_attachment_files(
+            all_attachments, buffer_config
+        )
         yield from attachment_files_generator
 
     def _create_data_files(
@@ -785,9 +762,62 @@ class SmartOpenStreamingStorage:
                     data_content = json.dumps(value, default=str).encode("utf-8")
                     yield f"{key}.json", BytesIO(data_content), {}
 
+    def _handle_attachment_error(
+        self,
+        all_attachments: list[AttachmentProcessingInfo],
+        failed_attachments: list[dict[str, Optional[str]]],
+    ) -> Generator[Tuple[str, datetime, int, Any, Iterable[bytes]], None, None]:
+        """Handle attachment errors and create a summary file."""
+
+        try:
+            # Calculate success rate with division by zero protection
+            total_attempted = len(all_attachments)
+            total_failed = len(failed_attachments)
+            success_rate = "N/A"
+
+            if total_attempted > 0:
+                success_rate = (
+                    f"{((total_attempted - total_failed) / total_attempted * 100):.1f}%"
+                )
+
+            error_summary = {
+                "failed_attachments": failed_attachments,
+                "total_failed": total_failed,
+                "total_attempted": total_attempted,
+                "success_rate": success_rate,
+                "timestamp": datetime.now().isoformat(),
+            }
+
+            error_summary_content = json.dumps(error_summary, indent=2).encode("utf-8")
+            yield (
+                "errors/attachment_failures_summary.json",
+                datetime.now(),
+                DEFAULT_FILE_MODE,
+                _ZIP_32_TYPE(),
+                iter([error_summary_content]),
+            )
+        except Exception as summary_error:
+            logger.error(f"Failed to create error summary: {summary_error}")
+            # Create a minimal error summary as fallback
+            fallback_summary = {
+                "error": "Failed to generate detailed error summary",
+                "total_failed": len(failed_attachments),
+                "total_attempted": len(all_attachments),
+                "timestamp": datetime.now().isoformat(),
+            }
+            fallback_content = json.dumps(fallback_summary, indent=2).encode("utf-8")
+            yield (
+                "errors/attachment_failures_summary.json",
+                datetime.now(),
+                DEFAULT_FILE_MODE,
+                _ZIP_32_TYPE(),
+                iter([fallback_content]),
+            )
+
     def _create_attachment_files(
         self,
         all_attachments: list[AttachmentProcessingInfo],
+        buffer_config: Optional[StreamingBufferConfig] = None,
     ) -> Generator[Tuple[str, datetime, int, Any, Iterable[bytes]], None, None]:
         """Create attachment files for the ZIP using true cloud-to-cloud streaming.
 
@@ -797,13 +827,66 @@ class SmartOpenStreamingStorage:
 
         Args:
             all_attachments: List of validated attachments
+            buffer_config: Configuration for error handling behavior
 
         Returns:
             Generator yielding attachment file entries in stream_zip format
         """
+        if buffer_config is None:
+            buffer_config = StreamingBufferConfig()
+
+        failed_attachments = []
+
         for attachment_info in all_attachments:
-            result = self._process_attachment_safely(attachment_info)
-            yield result
+            try:
+                result = self._process_attachment_safely(attachment_info)
+                yield result
+            except StorageUploadError as e:
+                # Log the failure
+                failed_attachments.append(
+                    {
+                        "attachment": attachment_info.attachment.file_name,
+                        "storage_key": attachment_info.attachment.storage_key,
+                        "error": str(e),
+                    }
+                )
+                logger.error(
+                    f"Failed to process attachment {attachment_info.attachment.file_name} "
+                    f"({attachment_info.attachment.storage_key}): {e}"
+                )
+
+                # If fail_fast is enabled, re-raise the exception
+                if buffer_config.fail_fast_on_attachment_errors:
+                    raise
+
+                # Create a placeholder file with error information if error details are enabled
+                if buffer_config.include_error_details:
+                    error_content = (
+                        f"Error: Failed to retrieve attachment - {e}".encode("utf-8")
+                    )
+                    error_filename = (
+                        f"ERROR_{attachment_info.attachment.file_name or 'unknown'}"
+                    )
+                    yield (
+                        f"errors/{error_filename}",
+                        datetime.now(),
+                        DEFAULT_FILE_MODE,
+                        _ZIP_32_TYPE(),
+                        iter([error_content]),
+                    )
+
+        # Log summary of failed attachments
+        if failed_attachments:
+            logger.warning(
+                f"Failed to process {len(failed_attachments)} attachments: "
+                f"{[att['attachment'] for att in failed_attachments]}"
+            )
+
+            # Create a summary error file with details about all failures if error details are enabled
+            if buffer_config.include_error_details:
+                yield from self._handle_attachment_error(
+                    all_attachments, failed_attachments
+                )
 
     def _transform_data_for_access_package(
         self, data: dict[str, Any], all_attachments: list[AttachmentProcessingInfo]

fides/api/service/storage/util.py

@@ -10,7 +10,7 @@ from fides.api.util.storage_util import format_size
 
 # This is the max file size for downloading the content of an attachment.
 # This is an industry standard used by companies like Google and Microsoft.
-LARGE_FILE_THRESHOLD = 25 * 1024 * 1024  # 25 MB
+LARGE_FILE_THRESHOLD = 2 * 1024 * 1024 * 1024  # 2 GB
 
 
 class AllowedFileType(EnumType):
@@ -172,6 +172,7 @@ def generate_attachment_url_from_storage_path(
     1. Using resolve_attachment_storage_path() to calculate the actual storage path
     2. Handling different directory structures (attachments vs data/dataset/collection)
     3. Generating proper relative paths from HTML template locations to attachment files
+    4. URL-encoding filenames for proper HTML link functionality
 
     Used by:
     - _process_attachment_list() in this file
@@ -191,17 +192,21 @@ def generate_attachment_url_from_storage_path(
         # Calculate the actual storage path
         storage_path = resolve_attachment_storage_path(unique_filename, base_path)
 
+        # URL-encode the filename for proper HTML link functionality
+        # Always encode when streaming is enabled to ensure consistency
+        encoded_filename = quote(unique_filename, safe="")
+
         # Generate relative path from HTML template directory to storage path
         if html_directory == "attachments" and base_path == "attachments":
             # From attachments/index.html to attachments/filename.pdf (same directory)
-            return unique_filename
+            return encoded_filename
         if html_directory.startswith("data/") and base_path.startswith("data/"):
             # From data/dataset/collection/index.html to data/dataset/collection/attachments/filename.pdf
             # Both are in data/ structure, so go to attachments subdirectory
-            return f"attachments/{unique_filename}"
+            return f"attachments/{encoded_filename}"
         # For other cases, calculate relative path
         # This is a simplified approach - in practice, you might need more sophisticated path resolution
-        return f"../{storage_path}"
+        return f"../{storage_path.replace(unique_filename, encoded_filename)}"
     return download_url
 
 

fides/api/util/rate_limit.py

@@ -180,15 +180,33 @@ is_rate_limit_enabled = (
     CONFIG.security.rate_limit_client_ip_header is not None
     and CONFIG.security.rate_limit_client_ip_header != ""
 )
-fides_limiter = Limiter(
-    storage_uri=CONFIG.redis.connection_url_unencoded,
-    application_limits=[
-        CONFIG.security.request_rate_limit
-    ],  # Creates ONE shared bucket for all endpoints
+
+disabled_limiter = Limiter(
+    default_limits=[CONFIG.security.request_rate_limit],
     headers_enabled=True,
     key_prefix=CONFIG.security.rate_limit_prefix,
     key_func=safe_rate_limit_key,
     retry_after="http-date",
-    in_memory_fallback_enabled=False,  # Fall back to no rate limiting if Redis unavailable
-    enabled=is_rate_limit_enabled,
+    enabled=False,
 )
+
+try:
+    if is_rate_limit_enabled:
+        fides_limiter = Limiter(
+            storage_uri=CONFIG.redis.connection_url_unencoded,
+            application_limits=[
+                CONFIG.security.request_rate_limit
+            ],  # Creates ONE shared bucket for all endpoints
+            headers_enabled=True,
+            key_prefix=CONFIG.security.rate_limit_prefix,
+            key_func=safe_rate_limit_key,
+            retry_after="http-date",
+            in_memory_fallback_enabled=False,  # Fall back to no rate limiting if Redis unavailable
+            enabled=is_rate_limit_enabled,
+        )
+    else:
+        fides_limiter = disabled_limiter
+except Exception as e:
+    logger.exception("Error instantiating rate limiter: {}", e)
+    if is_rate_limit_enabled:
+        raise e

fides/config/security_settings.py

@@ -222,10 +222,13 @@ class SecuritySettings(FidesSettings):
     def validate_rate_limit_client_ip_header(
         cls,
         v: str,
-    ) -> str:
+    ) -> Optional[str]:
         """Validate supported `rate_limit_client_ip_header`"""
         insecure_headers = ["x-forwarded-for"]
 
+        if not v:
+            return None
+
         if v.lower() in insecure_headers:
             raise ValueError(
                 "The rate_limit_client_ip_header cannot be set to a header that is not secure."

fides/ui-build/static/admin/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/650df9c348000a26.css" as="style"/><link rel="stylesheet" href="/_next/static/css/650df9c348000a26.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-678e89d68dbcd94f.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-fcdad91f6f66292b.js" defer=""></script><script src="/_next/static/chunks/pages/404-471a6b18e712f050.js" defer=""></script><script src="/_next/static/OmXHlY9MvjoZH9jDkAytl/_buildManifest.js" defer=""></script><script src="/_next/static/OmXHlY9MvjoZH9jDkAytl/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"OmXHlY9MvjoZH9jDkAytl","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/650df9c348000a26.css" as="style"/><link rel="stylesheet" href="/_next/static/css/650df9c348000a26.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-678e89d68dbcd94f.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-fcdad91f6f66292b.js" defer=""></script><script src="/_next/static/chunks/pages/404-471a6b18e712f050.js" defer=""></script><script src="/_next/static/Mzh6ue6wVfRTXIvDbuNvr/_buildManifest.js" defer=""></script><script src="/_next/static/Mzh6ue6wVfRTXIvDbuNvr/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"Mzh6ue6wVfRTXIvDbuNvr","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>