ethyca-fides 2.63.0rc3__py2.py3-none-any.whl → 2.63.1__py2.py3-none-any.whl

fides/api/service/privacy_request/request_runner_service.py

@@ -262,14 +262,14 @@ def upload_access_results(  # pylint: disable=R0912
                 privacy_request.add_success_execution_log(
                     session,
                     connection_key=None,
-                    dataset_name="Access Package Upload",
+                    dataset_name="Access package upload",
                     collection_name=None,
-                    message="Access Package Upload successful for privacy request.",
+                    message="Access package upload successful for privacy request.",
                     action_type=ActionType.access,
                 )
             logger.bind(
                 time_taken=time.time() - start_time,
-            ).info("Access Package Upload successful for privacy request.")
+            ).info("Access package upload successful for privacy request.")
         except common_exceptions.StorageUploadError as exc:
             logger.bind(
                 policy_key=policy.key,
@@ -279,9 +279,9 @@ def upload_access_results(  # pylint: disable=R0912
             privacy_request.add_error_execution_log(
                 session,
                 connection_key=None,
-                dataset_name="Access Package Upload",
+                dataset_name="Access package upload",
                 collection_name=None,
-                message="Access Package Upload failed for privacy request.",
+                message="Access package upload failed for privacy request.",
                 action_type=ActionType.access,
             )
             privacy_request.status = PrivacyRequestStatus.error

fides/api/service/privacy_request/request_service.py

@@ -17,11 +17,11 @@ from fides.api.models.privacy_request import (
     PrivacyRequest,
     RequestTask,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import (
-    ExecutionLogStatus,
     PrivacyRequestResponse,
     PrivacyRequestStatus,
 )

fides/api/task/create_request_tasks.py

@@ -29,8 +29,8 @@ from fides.api.models.privacy_request import (
     RequestTask,
     TraversalDetails,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
 from fides.api.util.logger_context_utils import log_context

fides/api/task/execute_request_tasks.py

@@ -22,8 +22,9 @@ from fides.api.common_exceptions import (
 from fides.api.graph.config import TERMINATOR_ADDRESS, CollectionAddress
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest, RequestTask
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType, CurrentStep
-from fides.api.schemas.privacy_request import ExecutionLogStatus, PrivacyRequestStatus
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
@@ -145,7 +146,7 @@ def can_run_task_body(
     if request_task.is_terminator_task:
         logger.info(
             "Terminator {} task reached.",
-            request_task.action_type.value,
+            request_task.action_type,
         )
         return False
     if request_task.is_root_task:
@@ -154,7 +155,7 @@ def can_run_task_body(
     if request_task.status != ExecutionLogStatus.pending:
         logger_method(request_task)(
             "Skipping {} task {} with status {}.",
-            request_task.action_type.value,
+            request_task.action_type,
             request_task.collection_address,
             request_task.status.value,
         )
@@ -449,7 +450,7 @@ def log_task_complete(request_task: RequestTask) -> None:
     """Convenience method for logging task completion"""
     logger.info(
         "{} task {} is {}.",
-        request_task.action_type.value.capitalize(),
+        request_task.action_type.capitalize(),
         request_task.collection_address,
         request_task.status.value,
     )
@@ -478,9 +479,9 @@ def _order_tasks_by_input_key(
 
 
 mapping = {
-    ActionType.access: run_access_node,
-    ActionType.erasure: run_erasure_node,
-    ActionType.consent: run_consent_node,
+    ActionType.access.value: run_access_node,
+    ActionType.erasure.value: run_erasure_node,
+    ActionType.consent.value: run_consent_node,
 }
 
 
@@ -504,7 +505,7 @@ def log_task_queued(request_task: RequestTask, location: str) -> None:
     """Helper for logging that tasks are queued"""
     logger_method(request_task)(
         "Queuing {} task {} from {}.",
-        request_task.action_type.value,
+        request_task.action_type,
         request_task.collection_address,
         location,
     )

fides/api/task/graph_task.py

@@ -39,8 +39,8 @@ from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.models.policy import Policy, Rule
 from fides.api.models.privacy_preference import PrivacyPreferenceHistory
 from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest, RequestTask
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType, CurrentStep
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.task.consolidate_query_matches import consolidate_query_matches
 from fides.api.task.filter_element_match import filter_element_match
@@ -503,12 +503,20 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             self.post_process_input_data(formatted_input_data)
         )
 
-        # For erasures: cache results with non-matching array elements *replaced* with placeholder text
-        placeholder_output: List[Row] = copy.deepcopy(output)
-        for row in placeholder_output:
+        # For erasures: build placeholder version incrementally to avoid holding two full
+        # copies of the data in memory simultaneously.
+        placeholder_output: List[Row] = []
+        for original_row in output:
+            # Create a deep copy of the *single* row, transform it, then append to
+            # the placeholder list.  Peak memory at any point is one extra row rather
+            # than an entire dataset.
+            row_copy = copy.deepcopy(original_row)
             filter_element_match(
-                row, query_paths=post_processed_node_input_data, delete_elements=False
+                row_copy,
+                query_paths=post_processed_node_input_data,
+                delete_elements=False,
             )
+            placeholder_output.append(row_copy)
 
         # For DSR 3.0, save data to build masking requests directly
         # on the Request Task.
@@ -519,11 +527,14 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         # TODO Remove when we stop support for DSR 2.0
         # Save data to build masking requests for DSR 2.0 in Redis.
         # Results saved with matching array elements preserved
-        self.resources.cache_results_with_placeholders(
-            f"access_request__{self.key}", placeholder_output
-        )
+        if not CONFIG.execution.use_dsr_3_0:
+            self.resources.cache_results_with_placeholders(
+                f"access_request__{self.key}", placeholder_output
+            )
 
-        # For access request results, cache results with non-matching array elements *removed*
+        # For access request results, mutate rows in-place to remove non-matching
+        # array elements.  We already iterated over `output` above, so reuse the same
+        # loop structure to keep cache locality.
         for row in output:
             logger.info(
                 "Filtering row in {} for matching array elements.",
@@ -537,7 +548,8 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
 
         # TODO Remove when we stop support for DSR 2.0
         # Saves intermediate access results for DSR 2.0 in Redis
-        self.resources.cache_object(f"access_request__{self.key}", output)
+        if not CONFIG.execution.use_dsr_3_0:
+            self.resources.cache_object(f"access_request__{self.key}", output)
 
         # Return filtered rows with non-matched array data removed.
         return output

fides/api/util/consent_util.py

@@ -18,7 +18,7 @@ from fides.api.models.privacy_request import (
 )
 from fides.api.models.sql_models import System  # type: ignore[attr-defined]
 from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride
-from fides.api.schemas.privacy_request import ExecutionLogStatus
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.redis_cache import Identity
 
 

fides/api/util/data_size.py

@@ -0,0 +1,102 @@
+"""
+Helpers for estimating the size of large collections of access data.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import List, Optional
+
+from loguru import logger
+
+from fides.api.util.collection_util import Row
+from fides.api.util.custom_json_encoder import CustomJSONEncoder
+
+# 640MB threshold for external storage
+# We only generate an estimated size for large datasets so we want to be conservative
+# and fallback to external storage even if we haven't hit the 1GB max limit.
+# We also want to pad for encryption and base64 encoding.
+LARGE_DATA_THRESHOLD_BYTES = 640 * 1024 * 1024  # 640MB
+
+
+def calculate_data_size(data: List[Row]) -> int:  # noqa: D401 – utility function
+    """Return an approximate JSON-serialized size (in bytes) for a list of *Row*.
+
+    The implementation purposefully avoids serializing the entire payload when
+    *data* is large.  For collections >1000 rows we sample a subset, measure the
+    encoded size, then extrapolate.  This keeps memory usage bounded while still
+    giving us an order-of-magnitude estimate suitable for "should I stream this
+    out to S3?" decisions.
+    """
+
+    if not data:
+        return 0
+
+    try:
+        data_count = len(data)
+
+        # For very large datasets, estimate size from a sample to avoid memory issues
+        if data_count > 1000:
+            logger.debug(
+                f"Calculating size for large dataset ({data_count} rows) using sampling"
+            )
+
+            sample_size = min(500, max(100, data_count // 20))  # 5 % capped at 500
+
+            # stratified sampling – take items spaced across the set when possible
+            if data_count > sample_size * 3:
+                step = data_count // sample_size
+                sample_indices = list(range(0, data_count, step))[:sample_size]
+                sample = [data[i] for i in sample_indices]
+            else:
+                sample = data[:sample_size]
+
+            sample_json = json.dumps(
+                sample, cls=CustomJSONEncoder, separators=(",", ":")
+            )
+            sample_bytes = len(sample_json.encode("utf-8"))
+
+            avg_record_size = sample_bytes / sample_size
+            content_size = int(avg_record_size * data_count)
+
+            # overhead: 2 bytes for [] plus a comma between every record plus 1 % slack
+            structure_overhead = 2 + (data_count - 1) + int(content_size * 0.01)
+            return content_size + structure_overhead
+
+        # small datasets – just measure
+        json_str = json.dumps(data, cls=CustomJSONEncoder, separators=(",", ":"))
+        return len(json_str.encode("utf-8"))
+
+    except (TypeError, ValueError) as exc:
+        logger.warning(
+            f"Failed to calculate JSON size, falling back to sys.getsizeof: {exc}"
+        )
+        return sys.getsizeof(data)
+
+
+def is_large_data(
+    data: List[Row], threshold_bytes: Optional[int] = None
+) -> bool:  # noqa: D401
+    """Return *True* if *data* is likely to exceed *threshold_bytes* when serialized."""
+
+    if not data:
+        return False
+
+    threshold = (
+        threshold_bytes if threshold_bytes is not None else LARGE_DATA_THRESHOLD_BYTES
+    )
+    size = calculate_data_size(data)
+    if size > threshold:
+        logger.info(
+            f"Data size ({size:,} bytes) exceeds threshold ({threshold:,} bytes) – using external storage"
+        )
+        return True
+    return False
+
+
+__all__ = [
+    "calculate_data_size",
+    "is_large_data",
+    "LARGE_DATA_THRESHOLD_BYTES",
+]

fides/api/util/encryption/aes_gcm_encryption_util.py

@@ -0,0 +1,271 @@
+"""
+AES GCM encryption utilities with SQLAlchemy-Utils and cryptography library implementations.
+
+This module provides simplified encrypt/decrypt functions using two approaches:
+1. SQLAlchemy-Utils AesGcmEngine (compatible with existing database encryption)
+2. Cryptography library with chunked processing (better performance, standard library)
+"""
+
+import base64
+import hashlib
+import json
+import os
+from typing import Any, List, Optional, Union
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from loguru import logger
+from sqlalchemy_utils.types.encrypted.encrypted_type import AesGcmEngine
+
+from fides.api.util.collection_util import Row
+from fides.api.util.custom_json_encoder import CustomJSONEncoder, _custom_decoder
+from fides.config import CONFIG
+
+
+class EncryptionError(Exception):
+    """Raised when encryption/decryption operations fail"""
+
+
+# SQLAlchemy-Utils Implementation (for compatibility with existing database encryption)
+def encrypt_with_sqlalchemy_utils(data: List[Row]) -> bytes:
+    """
+    Serialize and encrypt data using CustomJSONEncoder and SQLAlchemy-Utils AesGcmEngine.
+
+    This approach is compatible with existing database encryption but has lower performance.
+
+    Args:
+        data: Raw data to serialize and encrypt
+
+    Returns:
+        Encrypted bytes
+
+    Raises:
+        EncryptionError: If serialization or encryption fails
+    """
+    try:
+        # Serialize using CustomJSONEncoder for consistent ObjectId handling
+        serialized_data = json.dumps(data, cls=CustomJSONEncoder, separators=(",", ":"))
+        data_bytes = serialized_data.encode("utf-8")
+
+        # Encrypt using SQLAlchemy-Utils AesGcmEngine
+        engine = AesGcmEngine()
+        key = CONFIG.security.app_encryption_key
+        engine._update_key(key)  # pylint: disable=protected-access
+
+        # AesGcmEngine expects string input
+        data_str = data_bytes.decode("utf-8")
+        encrypted_data = engine.encrypt(data_str)
+        encrypted_bytes = encrypted_data.encode("utf-8")
+
+        logger.debug(
+            f"SQLAlchemy-Utils: Encrypted {len(data_bytes)} bytes to {len(encrypted_bytes)} bytes"
+        )
+        return encrypted_bytes
+
+    except Exception as e:
+        logger.error(f"SQLAlchemy-Utils encryption failed: {e}")
+        raise EncryptionError(f"SQLAlchemy-Utils encryption failed: {str(e)}")
+
+
+def decrypt_with_sqlalchemy_utils(encrypted_bytes: bytes) -> List[Row]:
+    """
+    Decrypt and deserialize data using SQLAlchemy-Utils AesGcmEngine and _custom_decoder.
+
+    Args:
+        encrypted_bytes: Encrypted data bytes to decrypt
+
+    Returns:
+        Deserialized data
+
+    Raises:
+        EncryptionError: If decryption or deserialization fails
+    """
+    try:
+        # Decrypt using SQLAlchemy-Utils AesGcmEngine
+        engine = AesGcmEngine()
+        key = CONFIG.security.app_encryption_key
+        engine._update_key(key)  # pylint: disable=protected-access
+
+        # AesGcmEngine expects string input
+        encrypted_str = encrypted_bytes.decode("utf-8")
+        decrypted_data = engine.decrypt(encrypted_str)
+
+        # Deserialize using _custom_decoder for consistent ObjectId handling
+        data = json.loads(decrypted_data, object_hook=_custom_decoder)
+
+        logger.debug(
+            f"SQLAlchemy-Utils: Decrypted {len(encrypted_bytes)} bytes to {len(data)} records"
+        )
+        return data
+
+    except Exception as e:
+        logger.error(f"SQLAlchemy-Utils decryption failed: {e}")
+        raise EncryptionError(f"SQLAlchemy-Utils decryption failed: {str(e)}")
+
+
+# Cryptography Library Implementation (standard, chunked processing)
+def encrypt_with_cryptography(
+    data: Union[List[Row], Any], chunk_size: Optional[int] = None
+) -> bytes:
+    """
+    Serialize and encrypt data using the standard cryptography library with chunked processing.
+
+    This provides fast performance and memory efficiency for large datasets.
+
+    Args:
+        data: Raw data to serialize and encrypt
+        chunk_size: Size of chunks for processing (default 4MB)
+
+    Returns:
+        Encrypted bytes (base64-encoded string as bytes)
+
+    Raises:
+        EncryptionError: If serialization or encryption fails
+    """
+    try:
+        # Set default chunk size
+        if chunk_size is None:
+            chunk_size = 4 * 1024 * 1024  # 4MB chunks
+
+        # Serialize using CustomJSONEncoder for consistent handling
+        serialized_data = json.dumps(data, cls=CustomJSONEncoder, separators=(",", ":"))
+        plaintext = serialized_data.encode("utf-8")
+
+        data_size_mb = len(plaintext) / (1024 * 1024)
+        chunk_size_mb = chunk_size / (1024 * 1024)
+        estimated_chunks = len(plaintext) // chunk_size + (
+            1 if len(plaintext) % chunk_size else 0
+        )
+        record_count = len(data) if isinstance(data, list) else "N/A"
+
+        logger.info(
+            f"Cryptography: Encrypting {record_count} records ({data_size_mb:.1f} MB) "
+            f"using {chunk_size_mb:.0f}MB chunks (~{estimated_chunks} chunks)"
+        )
+
+        # Use SQLAlchemy-Utils compatible key (SHA256 hash of app key)
+        key = _get_sqlalchemy_compatible_key()
+        nonce = os.urandom(12)  # 96-bit nonce for AES-GCM
+
+        # Create cipher
+        cipher = Cipher(
+            algorithms.AES(key), modes.GCM(nonce), backend=default_backend()
+        )
+        encryptor = cipher.encryptor()
+
+        # Process in chunks for memory efficiency
+        ciphertext_chunks = []
+        for i in range(0, len(plaintext), chunk_size):
+            chunk = plaintext[i : i + chunk_size]
+            ciphertext_chunks.append(encryptor.update(chunk))
+
+        # Finalize and get tag
+        encryptor.finalize()
+        tag = encryptor.tag
+
+        # Combine in same format as SQLAlchemy-Utils: [nonce/iv][tag][ciphertext]
+        ciphertext = b"".join(ciphertext_chunks)
+        binary_result = nonce + tag + ciphertext
+
+        # Base64 encode to match SQLAlchemy-Utils format
+        base64_result = base64.b64encode(binary_result).decode("utf-8")
+        result_bytes = base64_result.encode("utf-8")
+
+        encrypted_size_mb = len(result_bytes) / (1024 * 1024)
+        logger.info(
+            f"Cryptography: Encrypted successfully - "
+            f"{len(ciphertext_chunks)} chunks, {encrypted_size_mb:.1f} MB output (base64)"
+        )
+
+        return result_bytes
+
+    except Exception as e:
+        logger.error(f"Cryptography encryption failed: {e}")
+        raise EncryptionError(f"Cryptography encryption failed: {str(e)}")
+
+
+def decrypt_with_cryptography(
+    encrypted_bytes: bytes, chunk_size: Optional[int] = None
+) -> Union[List[Row], Any]:
+    """
+    Decrypt and deserialize data using the cryptography library with chunked processing.
+
+    Args:
+        encrypted_bytes: Encrypted data (base64-encoded string as bytes)
+        chunk_size: Size of chunks for processing (default 4MB)
+
+    Returns:
+        Deserialized data
+
+    Raises:
+        EncryptionError: If decryption or deserialization fails
+    """
+    try:
+        # Set default chunk size
+        if chunk_size is None:
+            chunk_size = 4 * 1024 * 1024  # 4MB chunks
+
+        # Decode from base64
+        encrypted_str = encrypted_bytes.decode("utf-8")
+        binary_data = base64.b64decode(encrypted_str)
+
+        # Extract components in SQLAlchemy-Utils format: [nonce/iv][tag][ciphertext]
+        if len(binary_data) < 28:  # 12 (nonce) + 16 (tag)
+            raise ValueError("Encrypted data too short")
+
+        nonce = binary_data[:12]  # First 12 bytes: nonce/IV
+        tag = binary_data[12:28]  # Next 16 bytes: tag
+        ciphertext = binary_data[28:]  # Remaining bytes: ciphertext
+
+        encrypted_size_mb = len(encrypted_bytes) / (1024 * 1024)
+        chunk_size_mb = chunk_size / (1024 * 1024)
+        estimated_chunks = len(ciphertext) // chunk_size + (
+            1 if len(ciphertext) % chunk_size else 0
+        )
+
+        logger.info(
+            f"Cryptography: Decrypting {encrypted_size_mb:.1f} MB "
+            f"using {chunk_size_mb:.0f}MB chunks (~{estimated_chunks} chunks)"
+        )
+
+        # Use SQLAlchemy-Utils compatible key
+        key = _get_sqlalchemy_compatible_key()
+        cipher = Cipher(
+            algorithms.AES(key), modes.GCM(nonce, tag), backend=default_backend()
+        )
+        decryptor = cipher.decryptor()
+
+        # Process in chunks for memory efficiency
+        plaintext_chunks = []
+        for i in range(0, len(ciphertext), chunk_size):
+            chunk = ciphertext[i : i + chunk_size]
+            plaintext_chunks.append(decryptor.update(chunk))
+
+        # Finalize
+        decryptor.finalize()
+
+        # Combine and deserialize
+        plaintext = b"".join(plaintext_chunks)
+        decrypted_json = plaintext.decode("utf-8")
+        data = json.loads(decrypted_json, object_hook=_custom_decoder)
+
+        record_count = len(data) if isinstance(data, list) else "N/A"
+        logger.info(f"Cryptography: Successfully decrypted {record_count} records")
+
+        return data
+
+    except Exception as e:
+        logger.error(f"Cryptography decryption failed: {e}")
+        raise EncryptionError(f"Cryptography decryption failed: {str(e)}")
+
+
+def _get_sqlalchemy_compatible_key() -> bytes:
+    """Get 32-byte encryption key compatible with SQLAlchemy-Utils AesGcmEngine."""
+    app_key = CONFIG.security.app_encryption_key.encode(CONFIG.security.encoding)
+    # SQLAlchemy-Utils always uses SHA256 hash of the key
+    return hashlib.sha256(app_key).digest()
+
+
+# Public API - Use cryptography by default for new operations
+encrypt_data = encrypt_with_cryptography
+decrypt_data = decrypt_with_cryptography