ethyca-fides 2.63.0rc3__py2.py3-none-any.whl → 2.63.1__py2.py3-none-any.whl

fides/api/schemas/manual_tasks/manual_task_schemas.py

@@ -0,0 +1,79 @@
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Any, Optional
+
+from pydantic import ConfigDict, Field
+
+from fides.api.schemas.base_class import FidesSchema
+
+
+class ManualTaskType(str, Enum):
+    """Enum for manual task types."""
+
+    privacy_request = "privacy_request"
+    # Add more task types as needed
+
+
+class ManualTaskParentEntityType(str, Enum):
+    """Enum for manual task parent entity types."""
+
+    connection_config = (
+        "connection_config"  # used for access and erasure privacy requests
+    )
+    # Add more parent entity types as needed
+
+
+class ManualTaskReferenceType(str, Enum):
+    """Enum for manual task reference types."""
+
+    privacy_request = "privacy_request"
+    connection_config = "connection_config"
+    manual_task_config = "manual_task_config"
+    assigned_user = "assigned_user"  # Reference to the user assigned to the task
+    # Add more reference types as needed
+
+
+class ManualTaskLogStatus(str, Enum):
+    """Enum for manual task log status."""
+
+    created = "created"
+    updated = "updated"
+    in_processing = "in_processing"
+    complete = "complete"
+    error = "error"
+    retrying = "retrying"
+    paused = "paused"
+    awaiting_input = "awaiting_input"
+
+
+class ManualTaskLogCreate(FidesSchema):
+    """Schema for creating a manual task log entry."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    task_id: Annotated[str, Field(..., description="ID of the task")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+
+
+class ManualTaskLogResponse(FidesSchema):
+    """Schema for manual task log response."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: Annotated[str, Field(..., description="Log ID")]
+    task_id: Annotated[str, Field(..., description="Task ID")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+    created_at: Annotated[datetime, Field(..., description="Creation timestamp")]
+    updated_at: Annotated[datetime, Field(..., description="Last update timestamp")]

fides/api/schemas/manual_tasks/manual_task_status.py

@@ -0,0 +1,151 @@
+from datetime import datetime, timezone
+from enum import Enum as EnumType
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+
+class StatusTransitionNotAllowed(Exception):
+    """Exception raised when a status transition is not allowed."""
+
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class StatusType(str, EnumType):
+    """Enum for manual task status."""
+
+    pending = "pending"
+    in_progress = "in_progress"
+    completed = "completed"
+    failed = "failed"
+
+    @classmethod
+    def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
+        """Get valid transitions from the current status.
+
+        Args:
+            current_status: The current status
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        if current_status == cls.pending:
+            return [cls.in_progress, cls.failed]
+        if current_status == cls.in_progress:
+            return [cls.completed, cls.failed]
+        if current_status == cls.completed:
+            return []
+        if current_status == cls.failed:
+            return [cls.pending, cls.in_progress]
+        return []
+
+
+class StatusTransitionMixin:
+    """Mixin for handling status transitions.
+
+    This mixin provides methods for managing status transitions and completion tracking.
+    It can be used by any model that needs status management.
+    """
+
+    # These should be overridden by the implementing class
+    status: StatusType
+    completed_at: Optional[datetime]
+    completed_by_id: Optional[str]
+
+    def _get_valid_transitions(self) -> list[StatusType]:
+        """Get valid transitions from the current status.
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        return StatusType.get_valid_transitions(self.status)
+
+    def _validate_status_transition(self, new_status: StatusType) -> None:
+        """Validate that a status transition is allowed.
+
+        Args:
+            new_status: The new status to transition to
+
+        Raises:
+            StatusTransitionNotAllowed: If the transition is not allowed
+        """
+        # Don't allow transitions to the same status
+        if new_status == self.status:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition: already in status {new_status}"
+            )
+
+        # Get valid transitions for current status
+        valid_transitions = self._get_valid_transitions()
+        if new_status not in valid_transitions:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition from {self.status} to {new_status}. "
+                f"Valid transitions are: {valid_transitions}"
+            )
+
+    def update_status(
+        self, db: Session, new_status: StatusType, user_id: Optional[str] = None
+    ) -> None:
+        """Update the status with validation and completion handling.
+
+        Args:
+            db: Database session
+            new_status: New status to set
+            user_id: Optional user ID who is making the change
+        """
+        self._validate_status_transition(new_status)
+
+        if new_status == StatusType.completed:
+            self.completed_at = datetime.now(timezone.utc)
+            self.completed_by_id = user_id
+        elif new_status == StatusType.pending:
+            # Reset completion fields if going back to pending
+            self.completed_at = None
+            self.completed_by_id = None
+
+        self.status = new_status
+        db.add(self)
+        db.commit()
+
+    def mark_completed(self, db: Session, user_id: str) -> None:
+        """Mark as completed.
+
+        Args:
+            db: Database session
+            user_id: user ID who completed the task
+        """
+        self.update_status(db, StatusType.completed, user_id)
+
+    def mark_failed(self, db: Session) -> None:
+        """Mark as failed."""
+        self.update_status(db, StatusType.failed)
+
+    def start_progress(self, db: Session) -> None:
+        """Mark as in progress."""
+        self.update_status(db, StatusType.in_progress)
+
+    def reset_to_pending(self, db: Session) -> None:
+        """Reset to pending status."""
+        self.update_status(db, StatusType.pending)
+
+    @property
+    def is_completed(self) -> bool:
+        """Check if completed."""
+        return self.status == StatusType.completed
+
+    @property
+    def is_failed(self) -> bool:
+        """Check if failed."""
+        return self.status == StatusType.failed
+
+    @property
+    def is_in_progress(self) -> bool:
+        """Check if in progress."""
+        return self.status == StatusType.in_progress
+
+    @property
+    def is_pending(self) -> bool:
+        """Check if pending."""
+        return self.status == StatusType.pending

fides/api/schemas/privacy_request.py

@@ -8,6 +8,7 @@ from pydantic import ConfigDict, Field, field_serializer, field_validator
 from fides.api.custom_types import SafeStr
 from fides.api.graph.config import CollectionAddress
 from fides.api.models.audit_log import AuditLogAction
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.api import BulkResponse, BulkUpdateFailed
 from fides.api.schemas.base_class import FidesSchema
 from fides.api.schemas.policy import ActionType, CurrentStep
@@ -141,18 +142,6 @@ class FieldsAffectedResponse(FidesSchema):
     model_config = ConfigDict(from_attributes=True, use_enum_values=True)
 
 
-class ExecutionLogStatus(EnumType):
-    """Enum for execution log statuses, reflecting where they are in their workflow"""
-
-    in_processing = "in_processing"
-    pending = "pending"
-    complete = "complete"
-    error = "error"
-    awaiting_processing = "paused"  # "paused" in the database to avoid a migration, but use "awaiting_processing" in the app
-    retrying = "retrying"
-    skipped = "skipped"
-
-
 class ExecutionLogStatusSerializeOverride(FidesSchema):
     """Override to serialize "paused" Execution Logs as awaiting_processing instead"""
 

fides/api/service/connectors/base_erasure_email_connector.py

@@ -5,6 +5,7 @@ from sqlalchemy.orm import Session
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.models.policy import Rule
 from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.connection_configuration.connection_secrets_email import (
     AdvancedSettings,
     BaseEmailSchema,
@@ -15,7 +16,6 @@ from fides.api.schemas.messaging.messaging import (
     MessagingActionType,
 )
 from fides.api.schemas.policy import ActionType
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.schemas.redis_cache import Identity
 from fides.api.service.connectors.base_email_connector import (
     BaseEmailConnector,

fides/api/service/connectors/consent_email_connector.py

@@ -16,6 +16,7 @@ from fides.api.models.privacy_notice import (
 )
 from fides.api.models.privacy_preference import PrivacyPreferenceHistory
 from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.connection_configuration.connection_secrets_email import (
     AdvancedSettingsWithExtendedIdentityTypes,
     ExtendedEmailSchema,
@@ -29,7 +30,7 @@ from fides.api.schemas.messaging.messaging import (
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_notice import PrivacyNoticeHistorySchema
 from fides.api.schemas.privacy_preference import MinimalPrivacyPreferenceHistorySchema
-from fides.api.schemas.privacy_request import Consent, ExecutionLogStatus
+from fides.api.schemas.privacy_request import Consent
 from fides.api.schemas.redis_cache import Identity
 from fides.api.service.connectors.base_email_connector import (
     BaseEmailConnector,

fides/api/service/connectors/dynamic_erasure_email_connector.py

@@ -17,11 +17,12 @@ from fides.api.models.privacy_request import (
     RequestTask,
     TraversalDetails,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.connection_configuration.connection_secrets_dynamic_erasure_email import (
     DynamicErasureEmailSchema,
 )
 from fides.api.schemas.policy import ActionType
-from fides.api.schemas.privacy_request import ExecutionLogStatus, PrivacyRequestStatus
+from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.service.connectors.base_erasure_email_connector import (
     BaseErasureEmailConnector,

fides/api/service/connectors/erasure_email_connector.py

@@ -10,9 +10,9 @@ from fides.api.models.connectionconfig import (
     ConnectionType,
 )
 from fides.api.models.privacy_request import ExecutionLog
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.connection_configuration import EmailSchema
 from fides.api.schemas.policy import ActionType
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.service.connectors.base_erasure_email_connector import (
     BaseErasureEmailConnector,
     filter_user_identities_for_connector,

fides/api/service/external_data_storage.py

@@ -0,0 +1,371 @@
+"""
+Service for handling external storage of large encrypted data.
+
+This service provides a generic interface for storing large data that would
+otherwise exceed database column size limits or impact performance.
+"""
+
+import os
+from io import BytesIO
+from typing import Any, Optional
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.models.storage import StorageConfig, get_active_default_storage_config
+from fides.api.schemas.external_storage import ExternalStorageMetadata
+from fides.api.schemas.storage.storage import StorageDetails, StorageType
+from fides.api.service.storage.gcs import get_gcs_client
+from fides.api.service.storage.s3 import generic_delete_from_s3, generic_upload_to_s3
+from fides.api.service.storage.util import get_local_filename
+from fides.api.util.aws_util import get_s3_client
+from fides.api.util.encryption.aes_gcm_encryption_util import decrypt_data, encrypt_data
+
+
+class ExternalDataStorageError(Exception):
+    """Raised when external data storage operations fail."""
+
+
+class ExternalDataStorageService:
+    """
+    Service for storing large encrypted data externally.
+
+    Handles:
+    - Automatic encryption/decryption
+    - Multiple storage backends (S3, local, GCS, etc.)
+    - Consistent file organization
+    - Cleanup operations
+    """
+
+    @staticmethod
+    def _get_storage_config(db: Session, storage_key: Optional[str]) -> "StorageConfig":
+        """Resolve and return the StorageConfig to use.
+
+        Preference order:
+
+        1. If *storage_key* is provided, fetch that specific configuration.
+        2. Otherwise, fall back to the *active* default storage configuration.
+
+        Raises ExternalDataStorageError when no suitable configuration is found.
+        """
+
+        if storage_key:
+            storage_config = (
+                db.query(StorageConfig).filter(StorageConfig.key == storage_key).first()
+            )
+            if not storage_config:
+                msg = f"Storage configuration with key '{storage_key}' not found"
+                logger.error(msg)
+                raise ExternalDataStorageError(msg)
+            return storage_config
+
+        # No explicit key – use the active default
+        storage_config = get_active_default_storage_config(db)
+        if not storage_config:
+            msg = "No active default storage configuration available for large data"
+            logger.error(msg)
+            raise ExternalDataStorageError(msg)
+
+        return storage_config
+
+    @staticmethod
+    def store_data(
+        db: Session,
+        storage_path: str,
+        data: Any,
+        storage_key: Optional[str] = None,
+    ) -> ExternalStorageMetadata:
+        """
+        Store data in external storage with encryption.
+
+        Args:
+            db: Database session
+            storage_path: Path where data should be stored (e.g., "model/id/field/timestamp")
+            data: The data to store (will be serialized and encrypted)
+            storage_key: Optional specific storage config key to use
+
+        Returns:
+            ExternalStorageMetadata with storage details
+
+        Raises:
+            ExternalDataStorageError: If storage operation fails
+        """
+        try:
+            storage_config = ExternalDataStorageService._get_storage_config(
+                db, storage_key
+            )
+
+            # Serialize and encrypt the data
+            encrypted_data = encrypt_data(data)
+            file_size = len(encrypted_data)
+
+            # Store to external storage based on type
+            if storage_config.type == StorageType.s3:
+                ExternalDataStorageService._store_to_s3(
+                    storage_config, storage_path, encrypted_data
+                )
+            elif storage_config.type == StorageType.gcs:
+                ExternalDataStorageService._store_to_gcs(
+                    storage_config, storage_path, encrypted_data
+                )
+            elif storage_config.type == StorageType.local:
+                ExternalDataStorageService._store_to_local(storage_path, encrypted_data)
+            else:
+                raise ExternalDataStorageError(
+                    f"Unsupported storage type: {storage_config.type}"
+                )
+
+            # Create and return metadata
+            metadata = ExternalStorageMetadata(
+                storage_type=StorageType(storage_config.type.value),
+                file_key=storage_path,
+                filesize=file_size,
+                storage_key=storage_config.key,
+            )
+
+            logger.info(
+                f"Stored {file_size:,} bytes to {storage_config.type} storage "
+                f"at path: {storage_path}"
+            )
+
+            return metadata
+
+        except Exception as e:
+            logger.error(f"Failed to store data externally: {str(e)}")
+            raise ExternalDataStorageError(f"Failed to store data: {str(e)}") from e
+
+    @staticmethod
+    def retrieve_data(
+        db: Session,
+        metadata: ExternalStorageMetadata,
+    ) -> Any:
+        """
+        Retrieve and decrypt data from external storage.
+
+        Args:
+            db: Database session
+            metadata: Storage metadata containing location and details
+
+        Returns:
+            Decrypted and deserialized data
+
+        Raises:
+            ExternalDataStorageError: If retrieval operation fails
+        """
+        try:
+            storage_config = ExternalDataStorageService._get_storage_config(
+                db, metadata.storage_key
+            )
+
+            # Retrieve encrypted data based on storage type
+            storage_type_value = (
+                metadata.storage_type.value
+                if isinstance(metadata.storage_type, StorageType)
+                else metadata.storage_type
+            )
+
+            if storage_type_value == StorageType.s3.value:
+                encrypted_data = ExternalDataStorageService._retrieve_from_s3(
+                    storage_config, metadata
+                )
+            elif storage_type_value == StorageType.gcs.value:
+                encrypted_data = ExternalDataStorageService._retrieve_from_gcs(
+                    storage_config, metadata
+                )
+            elif storage_type_value == StorageType.local.value:
+                encrypted_data = ExternalDataStorageService._retrieve_from_local(
+                    metadata
+                )
+            else:
+                raise ExternalDataStorageError(
+                    f"Unsupported storage type: {storage_type_value}"
+                )
+
+            # Handle case where download returns None
+            if encrypted_data is None:
+                raise ExternalDataStorageError(
+                    f"No data found at path: {metadata.file_key}"
+                )
+
+            # Decrypt and deserialize
+            data = decrypt_data(encrypted_data)
+
+            logger.info(
+                f"Retrieved {metadata.filesize:,} bytes from {storage_type_value} storage "
+                f"at path: {metadata.file_key}"
+            )
+
+            return data
+
+        except ExternalDataStorageError:
+            raise
+        except Exception as e:
+            logger.error(f"Failed to retrieve data from external storage: {str(e)}")
+            raise ExternalDataStorageError(f"Failed to retrieve data: {str(e)}") from e
+
+    @staticmethod
+    def delete_data(
+        db: Session,
+        metadata: ExternalStorageMetadata,
+    ) -> None:
+        """
+        Delete data from external storage.
+
+        Args:
+            db: Database session
+            metadata: Storage metadata containing location
+
+        Note:
+            This operation is best-effort and will log warnings on failure
+            rather than raising exceptions, to support cleanup scenarios.
+        """
+        try:
+            storage_config = ExternalDataStorageService._get_storage_config(
+                db, metadata.storage_key
+            )
+
+            # Delete from external storage based on type
+            storage_type_value = (
+                metadata.storage_type.value
+                if isinstance(metadata.storage_type, StorageType)
+                else metadata.storage_type
+            )
+
+            if storage_type_value == StorageType.s3.value:
+                ExternalDataStorageService._delete_from_s3(storage_config, metadata)
+            elif storage_type_value == StorageType.gcs.value:
+                ExternalDataStorageService._delete_from_gcs(storage_config, metadata)
+            elif storage_type_value == StorageType.local.value:
+                ExternalDataStorageService._delete_from_local(metadata)
+            else:
+                logger.warning(
+                    f"Unsupported storage type for cleanup: {storage_type_value}"
+                )
+                return
+
+            logger.info(
+                f"Deleted external storage file from {storage_type_value} storage "
+                f"at path: {metadata.file_key}"
+            )
+
+        except Exception as e:
+            # Log but don't raise - cleanup should be best effort
+            logger.warning(
+                f"Failed to delete external storage file at {metadata.file_key}: {str(e)}"
+            )
+
+    # Private helper methods for each storage type
+
+    @staticmethod
+    def _store_to_s3(config: StorageConfig, file_key: str, data: bytes) -> None:
+        """Store data to S3 using existing generic_upload_to_s3"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        document = BytesIO(data)
+        generic_upload_to_s3(
+            storage_secrets=config.secrets,
+            bucket_name=bucket_name,
+            file_key=file_key,
+            auth_method=auth_method,
+            document=document,
+        )
+
+    @staticmethod
+    def _store_to_gcs(config: StorageConfig, file_key: str, data: bytes) -> None:
+        """Store data to GCS using existing get_gcs_client"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        storage_client = get_gcs_client(auth_method, config.secrets)
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_key)
+
+        blob.upload_from_string(data, content_type="application/octet-stream")
+
+    @staticmethod
+    def _store_to_local(file_key: str, data: bytes) -> None:
+        """Store data to local filesystem using existing get_local_filename"""
+        file_path = get_local_filename(file_key)
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, "wb") as f:
+            f.write(data)
+
+    @staticmethod
+    def _retrieve_from_s3(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> bytes:
+        """Retrieve data from S3 directly, bypassing file size limits"""
+
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        # Get S3 client directly and download content regardless of file size
+        s3_client = get_s3_client(auth_method, config.secrets)
+
+        try:
+            # Download content directly to BytesIO buffer
+            file_obj = BytesIO()
+            s3_client.download_fileobj(
+                Bucket=bucket_name, Key=metadata.file_key, Fileobj=file_obj
+            )
+            file_obj.seek(0)  # Reset file pointer to beginning
+            return file_obj.read()
+        except Exception as e:
+            logger.error(f"Error retrieving file from S3: {e}")
+            raise e
+
+    @staticmethod
+    def _retrieve_from_gcs(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> bytes:
+        """Retrieve data from GCS using existing get_gcs_client"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        storage_client = get_gcs_client(auth_method, config.secrets)
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(metadata.file_key)
+        return blob.download_as_bytes()
+
+    @staticmethod
+    def _retrieve_from_local(metadata: ExternalStorageMetadata) -> bytes:
+        """Retrieve data from local filesystem"""
+        file_path = get_local_filename(metadata.file_key)
+        with open(file_path, "rb") as f:
+            return f.read()
+
+    @staticmethod
+    def _delete_from_s3(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> None:
+        """Delete data from S3 using existing generic_delete_from_s3"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        generic_delete_from_s3(
+            storage_secrets=config.secrets,
+            bucket_name=bucket_name,
+            file_key=metadata.file_key,
+            auth_method=auth_method,
+        )
+
+    @staticmethod
+    def _delete_from_gcs(
+        config: StorageConfig, metadata: ExternalStorageMetadata
+    ) -> None:
+        """Delete data from GCS using existing get_gcs_client"""
+        bucket_name = config.details[StorageDetails.BUCKET.value]
+        auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+
+        storage_client = get_gcs_client(auth_method, config.secrets)
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(metadata.file_key)
+        blob.delete()
+
+    @staticmethod
+    def _delete_from_local(metadata: ExternalStorageMetadata) -> None:
+        """Delete data from local filesystem"""
+        file_path = get_local_filename(metadata.file_key)
+        if os.path.exists(file_path):
+            os.remove(file_path)