ethyca-fides 2.63.0rc2__py2.py3-none-any.whl → 2.63.1__py2.py3-none-any.whl

fides/api/service/external_data_storage.py

@@ -0,0 +1,371 @@
+"""
+Service for handling external storage of large encrypted data.
+
+This service provides a generic interface for storing large data that would
+otherwise exceed database column size limits or impact performance.
+"""
+
+import os
+from io import BytesIO
+from typing import Any, Optional
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.models.storage import StorageConfig, get_active_default_storage_config
+from fides.api.schemas.external_storage import ExternalStorageMetadata
+from fides.api.schemas.storage.storage import StorageDetails, StorageType
+from fides.api.service.storage.gcs import get_gcs_client
+from fides.api.service.storage.s3 import generic_delete_from_s3, generic_upload_to_s3
+from fides.api.service.storage.util import get_local_filename
+from fides.api.util.aws_util import get_s3_client
+from fides.api.util.encryption.aes_gcm_encryption_util import decrypt_data, encrypt_data
+
+
+class ExternalDataStorageError(Exception):
+    """Raised when external data storage operations fail."""
+
+
+class ExternalDataStorageService:
+    """
+    Service for storing large encrypted data externally.
+
+    Handles:
+    - Automatic encryption/decryption
+    - Multiple storage backends (S3, local, GCS, etc.)
+    - Consistent file organization
+    - Cleanup operations
+    """
+
+    @staticmethod
+    def _get_storage_config(db: Session, storage_key: Optional[str]) -> "StorageConfig":
+        """Resolve and return the StorageConfig to use.
+
+        Preference order:
+
+        1. If *storage_key* is provided, fetch that specific configuration.
+        2. Otherwise, fall back to the *active* default storage configuration.
+
+        Raises ExternalDataStorageError when no suitable configuration is found.
+        """
+
+        if storage_key:
+            storage_config = (
+                db.query(StorageConfig).filter(StorageConfig.key == storage_key).first()
+            )
+            if not storage_config:
+                msg = f"Storage configuration with key '{storage_key}' not found"
+                logger.error(msg)
+                raise ExternalDataStorageError(msg)
+            return storage_config
+
+        # No explicit key – use the active default
+        storage_config = get_active_default_storage_config(db)
+        if not storage_config:
+            msg = "No active default storage configuration available for large data"
+            logger.error(msg)
+            raise ExternalDataStorageError(msg)
+
+        return storage_config
+
+    @staticmethod
+    def store_data(
+        db: Session,
+        storage_path: str,
+        data: Any,
+        storage_key: Optional[str] = None,
+    ) -> ExternalStorageMetadata:
+        """
+        Store data in external storage with encryption.
+
+        Args:
+            db: Database session
+            storage_path: Path where data should be stored (e.g., "model/id/field/timestamp")
+            data: The data to store (will be serialized and encrypted)
+            storage_key: Optional specific storage config key to use
+
+        Returns:
+            ExternalStorageMetadata with storage details
+
+        Raises:
+            ExternalDataStorageError: If storage operation fails
+        """
+        try:
+            storage_config = ExternalDataStorageService._get_storage_config(
+                db, storage_key
+            )
+
+            # Serialize and encrypt the data
+            encrypted_data = encrypt_data(data)
+            file_size = len(encrypted_data)
+
+            # Store to external storage based on type
+            if storage_config.type == StorageType.s3:
+                ExternalDataStorageService._store_to_s3(
+                    storage_config, storage_path, encrypted_data
+                )
+            elif storage_config.type == StorageType.gcs:
+                ExternalDataStorageService._store_to_gcs(
+                    storage_config, storage_path, encrypted_data
+                )
+            elif storage_config.type == StorageType.local:
+                ExternalDataStorageService._store_to_local(storage_path, encrypted_data)
+            else:
+                raise ExternalDataStorageError(
+                    f"Unsupported storage type: {storage_config.type}"
+                )
+
+            # Create and return metadata
+            metadata = ExternalStorageMetadata(
+                storage_type=StorageType(storage_config.type.value),
+                file_key=storage_path,
+                filesize=file_size,
+                storage_key=storage_config.key,
+            )
+
+            logger.info(
+                f"Stored {file_size:,} bytes to {storage_config.type} storage "
+                f"at path: {storage_path}"
+            )
+
+            return metadata
+
+        except Exception as e:
+            logger.error(f"Failed to store data externally: {str(e)}")
+            raise ExternalDataStorageError(f"Failed to store data: {str(e)}") from e
+
+    @staticmethod
+    def retrieve_data(
+        db: Session,
+        metadata: ExternalStorageMetadata,
+    ) -> Any:
+        """
+        Retrieve and decrypt data from external storage.
+
+        Args:
+            db: Database session
+            metadata: Storage metadata containing location and details
+
+        Returns:
+            Decrypted and deserialized data
+
+        Raises:
+            ExternalDataStorageError: If retrieval operation fails
+        """
+        try:
+            storage_config = ExternalDataStorageService._get_storage_config(
+                db, metadata.storage_key
+            )
+
+            # Retrieve encrypted data based on storage type
+            storage_type_value = (
+                metadata.storage_type.value
+                if isinstance(metadata.storage_type, StorageType)
+                else metadata.storage_type
+            )
+
+            if storage_type_value == StorageType.s3.value:
+                encrypted_data = ExternalDataStorageService._retrieve_from_s3(
+                    storage_config, metadata
+                )
+            elif storage_type_value == StorageType.gcs.value:
+                encrypted_data = ExternalDataStorageService._retrieve_from_gcs(
+                    storage_config, metadata
+                )
+            elif storage_type_value == StorageType.local.value:
+                encrypted_data = ExternalDataStorageService._retrieve_from_local(
+                    metadata
+                )
+            else:
+                raise ExternalDataStorageError(
+                    f"Unsupported storage type: {storage_type_value}"
+                )
+
+            # Handle case where download returns None
+            if encrypted_data is None:
+                raise ExternalDataStorageError(
+                    f"No data found at path: {metadata.file_key}"
+                )
+
+            # Decrypt and deserialize
+            data = decrypt_data(encrypted_data)
+
+            logger.info(
+                f"Retrieved {metadata.filesize:,} bytes from {storage_type_value} storage "
+                f"at path: {metadata.file_key}"
+            )
+
+            return data
+
+        except ExternalDataStorageError:
+            raise
+        except Exception as e:
+            logger.error(f"Failed to retrieve data from external storage: {str(e)}")
+            raise ExternalDataStorageError(f"Failed to retrieve data: {str(e)}") from e
+
+    @staticmethod
+    def delete_data(
+        db: Session,
+        metadata: ExternalStorageMetadata,
+    ) -> None:
+        """
+        Delete data from external storage.
+
+        Args:
+            db: Database session
+            metadata: Storage metadata containing location
+
+        Note:
+            This operation is best-effort and will log warnings on failure
+            rather than raising exceptions, to support cleanup scenarios.
+        """
+        try:
+            storage_config = ExternalDataStorageService._get_storage_config(
+                db, metadata.storage_key
+            )
+
+            # Delete from external storage based on type
+            storage_type_value = (
+                metadata.storage_type.value
+                if isinstance(metadata.storage_type, StorageType)
+                else metadata.storage_type
+            )
+
+            if storage_type_value == StorageType.s3.value:
+                ExternalDataStorageService._delete_from_s3(storage_config, metadata)
+            elif storage_type_value == StorageType.gcs.value:
+                ExternalDataStorageService._delete_from_gcs(storage_config, metadata)
+            elif storage_type_value == StorageType.local.value:
+                ExternalDataStorageService._delete_from_local(metadata)
+            else:
+                logger.warning(
+                    f"Unsupported storage type for cleanup: {storage_type_value}"
+                )
+                return
+
+            logger.info(
+                f"Deleted external storage file from {storage_type_value} storage "
+                f"at path: {metadata.file_key}"
+            )
+
+        except Exception as e:
+            # Log but don't raise - cleanup should be best effort
+            logger.warning(
+                f"Failed to delete external storage file at {metadata.file_key}: {str(e)}"
+            )
+
+    # Private helper methods for each storage type
+
+    @staticmethod
+    def _store_to_s3(config: StorageConfig, file_key: str, data: bytes) -> None:
+        """Store data to S3 using existing generic_upload_to_s3"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        document = BytesIO(data)
+        generic_upload_to_s3(
+            storage_secrets=config.secrets,
+            bucket_name=bucket_name,
+            file_key=file_key,
+            auth_method=auth_method,
+            document=document,
+        )
+
+    @staticmethod
+    def _store_to_gcs(config: StorageConfig, file_key: str, data: bytes) -> None:
+        """Store data to GCS using existing get_gcs_client"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        storage_client = get_gcs_client(auth_method, config.secrets)
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_key)
+
+        blob.upload_from_string(data, content_type="application/octet-stream")
+
+    @staticmethod
+    def _store_to_local(file_key: str, data: bytes) -> None:
+        """Store data to local filesystem using existing get_local_filename"""
+        file_path = get_local_filename(file_key)
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, "wb") as f:
+            f.write(data)
+
+    @staticmethod
+    def _retrieve_from_s3(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> bytes:
+        """Retrieve data from S3 directly, bypassing file size limits"""
+
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        # Get S3 client directly and download content regardless of file size
+        s3_client = get_s3_client(auth_method, config.secrets)
+
+        try:
+            # Download content directly to BytesIO buffer
+            file_obj = BytesIO()
+            s3_client.download_fileobj(
+                Bucket=bucket_name, Key=metadata.file_key, Fileobj=file_obj
+            )
+            file_obj.seek(0)  # Reset file pointer to beginning
+            return file_obj.read()
+        except Exception as e:
+            logger.error(f"Error retrieving file from S3: {e}")
+            raise e
+
+    @staticmethod
+    def _retrieve_from_gcs(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> bytes:
+        """Retrieve data from GCS using existing get_gcs_client"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        storage_client = get_gcs_client(auth_method, config.secrets)
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(metadata.file_key)
+        return blob.download_as_bytes()
+
+    @staticmethod
+    def _retrieve_from_local(metadata: ExternalStorageMetadata) -> bytes:
+        """Retrieve data from local filesystem"""
+        file_path = get_local_filename(metadata.file_key)
+        with open(file_path, "rb") as f:
+            return f.read()
+
+    @staticmethod
+    def _delete_from_s3(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> None:
+        """Delete data from S3 using existing generic_delete_from_s3"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        generic_delete_from_s3(
+            storage_secrets=config.secrets,
+            bucket_name=bucket_name,
+            file_key=metadata.file_key,
+            auth_method=auth_method,
+        )
+
+    @staticmethod
+    def _delete_from_gcs(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> None:
+        """Delete data from GCS using existing get_gcs_client"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        storage_client = get_gcs_client(auth_method, config.secrets)
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(metadata.file_key)
+        blob.delete()
+
+    @staticmethod
+    def _delete_from_local(metadata: ExternalStorageMetadata) -> None:
+        """Delete data from local filesystem"""
+        file_path = get_local_filename(metadata.file_key)
+        if os.path.exists(file_path):
+            os.remove(file_path)

fides/api/service/privacy_request/request_runner_service.py

@@ -262,14 +262,14 @@ def upload_access_results(  # pylint: disable=R0912
                 privacy_request.add_success_execution_log(
                     session,
                     connection_key=None,
-                    dataset_name="Access Package Upload",
+                    dataset_name="Access package upload",
                     collection_name=None,
-                    message="Access Package Upload successful for privacy request.",
+                    message="Access package upload successful for privacy request.",
                     action_type=ActionType.access,
                 )
             logger.bind(
                 time_taken=time.time() - start_time,
-            ).info("Access Package Upload successful for privacy request.")
+            ).info("Access package upload successful for privacy request.")
         except common_exceptions.StorageUploadError as exc:
             logger.bind(
                 policy_key=policy.key,
@@ -279,9 +279,9 @@ def upload_access_results(  # pylint: disable=R0912
             privacy_request.add_error_execution_log(
                 session,
                 connection_key=None,
-                dataset_name="Access Package Upload",
+                dataset_name="Access package upload",
                 collection_name=None,
-                message="Access Package Upload failed for privacy request.",
+                message="Access package upload failed for privacy request.",
                 action_type=ActionType.access,
             )
             privacy_request.status = PrivacyRequestStatus.error

fides/api/service/privacy_request/request_service.py

@@ -17,11 +17,11 @@ from fides.api.models.privacy_request import (
     PrivacyRequest,
     RequestTask,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import (
-    ExecutionLogStatus,
     PrivacyRequestResponse,
     PrivacyRequestStatus,
 )

fides/api/task/create_request_tasks.py

@@ -29,8 +29,8 @@ from fides.api.models.privacy_request import (
     RequestTask,
     TraversalDetails,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
 from fides.api.util.logger_context_utils import log_context

fides/api/task/execute_request_tasks.py

@@ -22,8 +22,9 @@ from fides.api.common_exceptions import (
 from fides.api.graph.config import TERMINATOR_ADDRESS, CollectionAddress
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest, RequestTask
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType, CurrentStep
-from fides.api.schemas.privacy_request import ExecutionLogStatus, PrivacyRequestStatus
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
@@ -145,7 +146,7 @@ def can_run_task_body(
     if request_task.is_terminator_task:
         logger.info(
             "Terminator {} task reached.",
-            request_task.action_type.value,
+            request_task.action_type,
         )
         return False
     if request_task.is_root_task:
@@ -154,7 +155,7 @@ def can_run_task_body(
     if request_task.status != ExecutionLogStatus.pending:
         logger_method(request_task)(
             "Skipping {} task {} with status {}.",
-            request_task.action_type.value,
+            request_task.action_type,
             request_task.collection_address,
             request_task.status.value,
         )
@@ -449,7 +450,7 @@ def log_task_complete(request_task: RequestTask) -> None:
     """Convenience method for logging task completion"""
     logger.info(
         "{} task {} is {}.",
-        request_task.action_type.value.capitalize(),
+        request_task.action_type.capitalize(),
         request_task.collection_address,
         request_task.status.value,
     )
@@ -478,9 +479,9 @@ def _order_tasks_by_input_key(
 
 
 mapping = {
-    ActionType.access: run_access_node,
-    ActionType.erasure: run_erasure_node,
-    ActionType.consent: run_consent_node,
+    ActionType.access.value: run_access_node,
+    ActionType.erasure.value: run_erasure_node,
+    ActionType.consent.value: run_consent_node,
 }
 
 
@@ -504,7 +505,7 @@ def log_task_queued(request_task: RequestTask, location: str) -> None:
     """Helper for logging that tasks are queued"""
     logger_method(request_task)(
         "Queuing {} task {} from {}.",
-        request_task.action_type.value,
+        request_task.action_type,
         request_task.collection_address,
         location,
     )

fides/api/task/graph_task.py

@@ -39,8 +39,8 @@ from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.models.policy import Policy, Rule
 from fides.api.models.privacy_preference import PrivacyPreferenceHistory
 from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest, RequestTask
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType, CurrentStep
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.task.consolidate_query_matches import consolidate_query_matches
 from fides.api.task.filter_element_match import filter_element_match
@@ -503,12 +503,20 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             self.post_process_input_data(formatted_input_data)
         )
 
-        # For erasures: cache results with non-matching array elements *replaced* with placeholder text
-        placeholder_output: List[Row] = copy.deepcopy(output)
-        for row in placeholder_output:
+        # For erasures: build placeholder version incrementally to avoid holding two full
+        # copies of the data in memory simultaneously.
+        placeholder_output: List[Row] = []
+        for original_row in output:
+            # Create a deep copy of the *single* row, transform it, then append to
+            # the placeholder list.  Peak memory at any point is one extra row rather
+            # than an entire dataset.
+            row_copy = copy.deepcopy(original_row)
             filter_element_match(
-                row, query_paths=post_processed_node_input_data, delete_elements=False
+                row_copy,
+                query_paths=post_processed_node_input_data,
+                delete_elements=False,
             )
+            placeholder_output.append(row_copy)
 
         # For DSR 3.0, save data to build masking requests directly
         # on the Request Task.
@@ -519,11 +527,14 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         # TODO Remove when we stop support for DSR 2.0
         # Save data to build masking requests for DSR 2.0 in Redis.
         # Results saved with matching array elements preserved
-        self.resources.cache_results_with_placeholders(
-            f"access_request__{self.key}", placeholder_output
-        )
+        if not CONFIG.execution.use_dsr_3_0:
+            self.resources.cache_results_with_placeholders(
+                f"access_request__{self.key}", placeholder_output
+            )
 
-        # For access request results, cache results with non-matching array elements *removed*
+        # For access request results, mutate rows in-place to remove non-matching
+        # array elements.  We already iterated over `output` above, so reuse the same
+        # loop structure to keep cache locality.
         for row in output:
             logger.info(
                 "Filtering row in {} for matching array elements.",
@@ -537,7 +548,8 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
 
         # TODO Remove when we stop support for DSR 2.0
         # Saves intermediate access results for DSR 2.0 in Redis
-        self.resources.cache_object(f"access_request__{self.key}", output)
+        if not CONFIG.execution.use_dsr_3_0:
+            self.resources.cache_object(f"access_request__{self.key}", output)
 
         # Return filtered rows with non-matched array data removed.
         return output

fides/api/util/consent_util.py

@@ -18,7 +18,7 @@ from fides.api.models.privacy_request import (
 )
 from fides.api.models.sql_models import System  # type: ignore[attr-defined]
 from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride
-from fides.api.schemas.privacy_request import ExecutionLogStatus
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.redis_cache import Identity
 
 

fides/api/util/data_size.py

@@ -0,0 +1,102 @@
+"""
+Helpers for estimating the size of large collections of access data.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import List, Optional
+
+from loguru import logger
+
+from fides.api.util.collection_util import Row
+from fides.api.util.custom_json_encoder import CustomJSONEncoder
+
+# 640MB threshold for external storage
+# We only generate an estimated size for large datasets so we want to be conservative
+# and fallback to external storage even if we haven't hit the 1GB max limit.
+# We also want to pad for encryption and base64 encoding.
+LARGE_DATA_THRESHOLD_BYTES = 640 * 1024 * 1024  # 640MB
+
+
+def calculate_data_size(data: List[Row]) -> int:  # noqa: D401 – utility function
+    """Return an approximate JSON-serialized size (in bytes) for a list of *Row*.
+
+    The implementation purposefully avoids serializing the entire payload when
+    *data* is large.  For collections >1000 rows we sample a subset, measure the
+    encoded size, then extrapolate.  This keeps memory usage bounded while still
+    giving us an order-of-magnitude estimate suitable for "should I stream this
+    out to S3?" decisions.
+    """
+
+    if not data:
+        return 0
+
+    try:
+        data_count = len(data)
+
+        # For very large datasets, estimate size from a sample to avoid memory issues
+        if data_count > 1000:
+            logger.debug(
+                f"Calculating size for large dataset ({data_count} rows) using sampling"
+            )
+
+            sample_size = min(500, max(100, data_count // 20))  # 5 % capped at 500
+
+            # stratified sampling – take items spaced across the set when possible
+            if data_count > sample_size * 3:
+                step = data_count // sample_size
+                sample_indices = list(range(0, data_count, step))[:sample_size]
+                sample = [data[i] for i in sample_indices]
+            else:
+                sample = data[:sample_size]
+
+            sample_json = json.dumps(
+                sample, cls=CustomJSONEncoder, separators=(",", ":")
+            )
+            sample_bytes = len(sample_json.encode("utf-8"))
+
+            avg_record_size = sample_bytes / sample_size
+            content_size = int(avg_record_size * data_count)
+
+            # overhead: 2 bytes for [] plus a comma between every record plus 1 % slack
+            structure_overhead = 2 + (data_count - 1) + int(content_size * 0.01)
+            return content_size + structure_overhead
+
+        # small datasets – just measure
+        json_str = json.dumps(data, cls=CustomJSONEncoder, separators=(",", ":"))
+        return len(json_str.encode("utf-8"))
+
+    except (TypeError, ValueError) as exc:
+        logger.warning(
+            f"Failed to calculate JSON size, falling back to sys.getsizeof: {exc}"
+        )
+        return sys.getsizeof(data)
+
+
+def is_large_data(
+    data: List[Row], threshold_bytes: Optional[int] = None
+) -> bool:  # noqa: D401
+    """Return *True* if *data* is likely to exceed *threshold_bytes* when serialized."""
+
+    if not data:
+        return False
+
+    threshold = (
+        threshold_bytes if threshold_bytes is not None else LARGE_DATA_THRESHOLD_BYTES
+    )
+    size = calculate_data_size(data)
+    if size > threshold:
+        logger.info(
+            f"Data size ({size:,} bytes) exceeds threshold ({threshold:,} bytes) – using external storage"
+        )
+        return True
+    return False
+
+
+__all__ = [
+    "calculate_data_size",
+    "is_large_data",
+    "LARGE_DATA_THRESHOLD_BYTES",
+]