ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/service/saas_request/saas_request_override_factory.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from inspect import Signature, signature
-from typing import Callable, Dict, List, Union
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union
 
 from loguru import logger
 
@@ -12,6 +12,9 @@ from fides.api.schemas.consentable_item import ConsentableItem, ConsentWebhookRe
 from fides.api.schemas.saas.shared_schemas import ConsentPropagationStatus
 from fides.api.util.collection_util import Row
 
+if TYPE_CHECKING:
+    from fides.api.schemas.saas.async_polling_configuration import PollingResult
+
 
 # at some point this should likely be formalized more centrally...
 class SaaSRequestType(Enum):
@@ -30,6 +33,10 @@ class SaaSRequestType(Enum):
     UPDATE_CONSENT = "update_consent"
     PROCESS_CONSENT_WEBHOOK = "process_consent_webhook"
 
+    # Async polling request types
+    POLLING_STATUS = "polling_status"
+    POLLING_RESULT = "polling_result"
+
 
 RequestOverrideFunction = Callable[
     ...,
@@ -39,6 +46,8 @@ RequestOverrideFunction = Callable[
         List[Row],
         ConsentPropagationStatus,
         int,
+        bool,  # For polling status overrides
+        "PollingResult",  # For polling result overrides - string literal to avoid circular import
         None,
     ],
 ]
@@ -105,6 +114,10 @@ class SaaSRequestOverrideFactory:
                     validate_update_consent_function(override_function)
                 elif request_type == SaaSRequestType.PROCESS_CONSENT_WEBHOOK:
                     validate_process_consent_webhook_function(override_function)
+                elif request_type == SaaSRequestType.POLLING_STATUS:
+                    validate_polling_status_override_function(override_function)
+                elif request_type == SaaSRequestType.POLLING_RESULT:
+                    validate_polling_result_override_function(override_function)
                 else:
                     raise ValueError(
                         f"Invalid SaaSRequestType '{request_type}' provided for SaaS request override function"
@@ -264,5 +277,62 @@ def validate_process_consent_webhook_function(f: Callable) -> None:
         )
 
 
+def validate_polling_status_override_function(f: Callable) -> None:
+    """
+    Perform some basic checks on the user-provided SaaS request override function
+    that will be used for polling status checks.
+
+    The validation is not overly strict to allow for some flexibility in
+    the functions that are used for overrides, but we check to ensure that
+    the function meets the framework's basic expectations.
+
+    Specifically, the validation checks that function's return type is `bool`
+    and that it declares at least 4 parameters.
+    """
+    sig: Signature = signature(f)
+    if sig.return_annotation is not bool:
+        raise InvalidSaaSRequestOverrideException(
+            "Polling status override function must return a bool"
+        )
+    if len(sig.parameters) < 4:
+        raise InvalidSaaSRequestOverrideException(
+            "Polling status override function must declare at least 4 parameters"
+        )
+
+
+def validate_polling_result_override_function(f: Callable) -> None:
+    """
+    Perform some basic checks on the user-provided SaaS request override function
+    that will be used for polling result retrieval.
+
+    The validation is not overly strict to allow for some flexibility in
+    the functions that are used for overrides, but we check to ensure that
+    the function meets the framework's basic expectations.
+
+    Specifically, the validation checks that function's return type is `PollingResult`
+    and that it declares at least 4 parameters.
+    """
+    sig: Signature = signature(f)
+
+    # Import PollingResult here to avoid circular imports at module level
+    from fides.api.schemas.saas.async_polling_configuration import PollingResult
+
+    # Check return type annotation - handle both direct class and string literals
+    return_annotation = sig.return_annotation
+    if return_annotation not in (
+        PollingResult,
+        "PollingResult",
+        Optional[PollingResult],
+        "Optional[PollingResult]",
+    ):
+        raise InvalidSaaSRequestOverrideException(
+            "Polling result override function must return a PollingResult"
+        )
+    if len(sig.parameters) < 4:
+        raise InvalidSaaSRequestOverrideException(
+            "Polling result override function must declare at least 4 parameters"
+        )
+
+
 # TODO: Avoid running this on import?
 register = SaaSRequestOverrideFactory.register

fides/api/service/storage/streaming/schemas.py

@@ -2,11 +2,15 @@ from typing import Annotated, Any, Optional
 
 from pydantic import BaseModel, Field, field_validator
 
-# multipart upload requirements
-MIN_PART_SIZE = 5 * 1024 * 1024  # 5MB minimum part size for AWS S3 uploads
+# Storage size constants
+S3_MIN_PART_SIZE = 5 * 1024 * 1024  # 5MB minimum part size for AWS S3 multipart uploads
 ZIP_BUFFER_THRESHOLD = 5 * 1024 * 1024  # 5MB threshold for zip buffer uploads
-CHUNK_SIZE_THRESHOLD = 5 * 1024 * 1024  # 5MB threshold for chunk-based uploads
-LARGE_FILE_THRESHOLD = 25 * 1024 * 1024  # 25MB threshold for large file handling
+DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB default chunk size for streaming operations
+MAX_FILE_SIZE = 2 * 1024 * 1024 * 1024  # 2GB threshold for large file handling
+
+# Backward compatibility aliases
+MIN_PART_SIZE = S3_MIN_PART_SIZE
+CHUNK_SIZE_THRESHOLD = DEFAULT_CHUNK_SIZE
 
 
 class AttachmentInfo(BaseModel):
@@ -52,21 +56,6 @@ class StorageUploadConfig(BaseModel):
         return v.strip()
 
 
-class PackageSplitConfig(BaseModel):
-    """Configuration for package splitting operations"""
-
-    max_attachments: int = Field(
-        default=100, ge=1, le=1000, description="Max attachments per package"
-    )
-
-    @field_validator("max_attachments")
-    @classmethod
-    def validate_max_attachments(cls, v: int) -> int:
-        if v < 1:
-            raise ValueError("max_attachments must be at least 1")
-        return v
-
-
 class StreamingBufferConfig(BaseModel):
     """Configuration for streaming buffer operations"""
 
@@ -85,6 +74,25 @@ class StreamingBufferConfig(BaseModel):
         ge=1024 * 1024,
         description="Chunk size threshold in bytes (default: 1MB)",
     )
+    fail_fast_on_attachment_errors: bool = Field(
+        default=True,
+        description="If True, stop processing on first attachment error. If False, continue with error placeholders.",
+    )
+    include_error_details: bool = Field(
+        default=True,
+        description="If True, include detailed error information in the output ZIP.",
+    )
+
+
+class SmartOpenStreamingStorageConfig(BaseModel):
+    """Configuration for SmartOpenStreamingStorage initialization"""
+
+    chunk_size: int = Field(
+        default=DEFAULT_CHUNK_SIZE,
+        ge=1024,  # 1KB minimum
+        le=MAX_FILE_SIZE,  # 2GB maximum
+        description="Size of chunks for streaming attachments (default: 5MB)",
+    )
 
 
 class AttachmentProcessingInfo(BaseModel):

fides/api/service/storage/streaming/smart_open_client.py

@@ -10,7 +10,7 @@ from loguru import logger
 
 from fides.api.service.storage.streaming.retry import retry_cloud_storage_operation
 from fides.api.service.storage.streaming.schemas import (
-    MIN_PART_SIZE,
+    S3_MIN_PART_SIZE,
     MultipartUploadResponse,
     UploadPartResponse,
 )
@@ -27,7 +27,7 @@ class SmartOpenStorageClient:
     logic to specialized client implementations.
     """
 
-    min_part_size: int = MIN_PART_SIZE
+    min_part_size: int = S3_MIN_PART_SIZE
 
     def __init__(
         self,