ethyca-fides 2.63.0rc3__py2.py3-none-any.whl → 2.63.1__py2.py3-none-any.whl

fides/api/models/manual_tasks/manual_task.py

@@ -0,0 +1,110 @@
+from typing import Any
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
+from fides.api.schemas.manual_tasks.manual_task_schemas import (
+    ManualTaskLogStatus,
+    ManualTaskParentEntityType,
+    ManualTaskReferenceType,
+    ManualTaskType,
+)
+
+
+class ManualTask(Base):
+    """Model for storing manual tasks.
+
+    This model can be used for both privacy request tasks and general tasks.
+    For privacy requests, it replaces the functionality of manual webhooks.
+    For other use cases, it provides a flexible task management system.
+
+    There can only be one ManualTask per parent entity.
+    You can create multiple Configs for the same ManualTask.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task"
+
+    # Database columns
+    task_type = Column(
+        EnumColumn(ManualTaskType),
+        nullable=False,
+        default=ManualTaskType.privacy_request,
+    )
+    parent_entity_id = Column(String, nullable=False)
+    parent_entity_type = Column(
+        EnumColumn(ManualTaskParentEntityType),
+        nullable=False,
+        default=ManualTaskParentEntityType.connection_config,
+    )
+    due_date = Column(DateTime, nullable=True)
+
+    # Relationships
+    references = relationship(
+        "ManualTaskReference",
+        back_populates="task",
+        uselist=True,
+        cascade="all, delete-orphan",
+    )
+    logs = relationship(
+        "ManualTaskLog",
+        back_populates="task",
+        primaryjoin="and_(ManualTask.id == ManualTaskLog.task_id)",
+        viewonly=True,
+        order_by="ManualTaskLog.created_at",
+    )
+
+    # Properties
+    @property
+    def assigned_users(self) -> list[str]:
+        """Get all users assigned to this task."""
+        if not self.references:
+            return []
+        return [
+            ref.reference_id
+            for ref in self.references
+            if ref.reference_type == ManualTaskReferenceType.assigned_user
+        ]
+
+    # CRUD Operations
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = True
+    ) -> "ManualTask":
+        """Create a new manual task."""
+        task = super().create(db=db, data=data, check_name=check_name)
+        ManualTaskLog.create_log(
+            db=db,
+            task_id=task.id,
+            status=ManualTaskLogStatus.created,
+            message=f"Created manual task for {data['task_type']}",
+        )
+        return task
+
+
+class ManualTaskReference(Base):
+    """Join table to associate manual tasks with multiple references.
+
+    A single task may have many references including privacy requests, configurations, and assigned users.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_reference"
+
+    # Database columns
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(ManualTaskReferenceType), nullable=False)
+
+    # Relationships
+    task = relationship("ManualTask", back_populates="references")

fides/api/models/manual_tasks/manual_task_log.py

@@ -0,0 +1,100 @@
+from typing import TYPE_CHECKING, Any, Optional
+
+from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskLogStatus
+
+if TYPE_CHECKING:
+    from fides.api.models.manual_tasks.manual_task import ManualTask
+
+
+class ManualTaskLog(Base):
+    """Model for storing manual task execution logs."""
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_log"
+
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    # TODO: Add foreign key constraints when config and instance are implemented
+    config_id = Column(String, nullable=True)
+    instance_id = Column(String, nullable=True)
+    status = Column(String, nullable=False)
+    message = Column(String, nullable=True)
+    details = Column(JSONB, nullable=True)
+
+    # Relationships - using string references to avoid circular imports
+    task = relationship("ManualTask", back_populates="logs", foreign_keys=[task_id])
+    # TODO: Add config and instance relationships when they are implemented
+    # config = relationship("ManualTaskConfig", back_populates="logs")
+    # instance = relationship("ManualTaskInstance", back_populates="logs")
+
+    @classmethod
+    def create_log(
+        cls,
+        db: Session,
+        status: ManualTaskLogStatus,
+        task_id: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        message: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new task log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            status: Status of the log entry
+            message: Optional message describing the event
+            details: Optional additional details about the event
+        """
+        data = {
+            "task_id": task_id,
+            "config_id": config_id,
+            "instance_id": instance_id,
+            "status": status,
+            "message": message,
+            "details": details,
+        }
+        return cls.create(db=db, data=data)
+
+    @classmethod
+    def create_error_log(
+        cls,
+        db: Session,
+        task_id: str,
+        message: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new error log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            message: Error message describing what went wrong
+            config_id: Optional ID of the configuration
+            instance_id: Optional ID of the instance
+            details: Optional additional details about the error
+
+        Returns:
+            The created error log entry
+        """
+        return cls.create_log(
+            db=db,
+            status=ManualTaskLogStatus.error,
+            task_id=task_id,
+            config_id=config_id,
+            instance_id=instance_id,
+            message=message,
+            details=details,
+        )

fides/api/models/privacy_preference.py

@@ -22,8 +22,8 @@ from fides.api.models.privacy_notice import (
     UserConsentPreference,
 )
 from fides.api.models.privacy_request import PrivacyRequest, ProvidedIdentity
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.language import SupportedLanguage
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.schemas.redis_cache import MultiValue
 from fides.config import CONFIG
 

fides/api/models/privacy_request/execution_log.py

@@ -4,15 +4,14 @@ from __future__ import annotations
 
 from typing import Optional
 
-from sqlalchemy import Column, DateTime, String
+from sqlalchemy import Column, String
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.mutable import MutableList
-from sqlalchemy.sql import text
 
 from fides.api.db.base_class import Base  # type: ignore[attr-defined]
 from fides.api.db.util import EnumColumn
+from fides.api.models.worker_task import ExecutionLogStatus, TaskExecutionLog
 from fides.api.schemas.policy import ActionType, CurrentStep
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 
 # Locations from which privacy request execution can be resumed, in order.
 EXECUTION_CHECKPOINTS = [
@@ -53,7 +52,7 @@ def can_run_checkpoint(
     ) >= EXECUTION_CHECKPOINTS.index(from_checkpoint)
 
 
-class ExecutionLog(Base):
+class ExecutionLog(TaskExecutionLog, Base):
     """
     Stores the individual execution logs associated with a PrivacyRequest.
 
@@ -68,41 +67,14 @@ class ExecutionLog(Base):
     collection_name = Column(String, index=True)
     # A JSON Array describing affected fields along with their data categories and paths
     fields_affected = Column(MutableList.as_mutable(JSONB), nullable=True)
-    # Contains info, warning, or error messages
-    message = Column(String)
     action_type = Column(
         EnumColumn(ActionType),
         index=True,
         nullable=False,
     )
-    status = Column(
-        EnumColumn(
-            ExecutionLogStatus,
-            native_enum=True,
-            values_callable=lambda x: [
-                i.value for i in x
-            ],  # Using ExecutionLogStatus values in database, even though app is using the names.
-        ),
-        index=True,
-        nullable=False,
-    )
 
     privacy_request_id = Column(
         String,
         nullable=False,
         index=True,
     )
-
-    # Use clock_timestamp() instead of NOW() to get the actual current time at row creation,
-    # regardless of transaction state. This prevents timestamp caching within transactions
-    # and ensures more accurate creation times.
-    # https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-CURRENT
-
-    created_at = Column(
-        DateTime(timezone=True), server_default=text("clock_timestamp()")
-    )
-    updated_at = Column(
-        DateTime(timezone=True),
-        server_default=text("clock_timestamp()"),
-        onupdate=text("clock_timestamp()"),
-    )

fides/api/models/privacy_request/privacy_request.py

@@ -48,6 +48,7 @@ from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
 from fides.api.models.comment import Comment, CommentReference, CommentReferenceType
 from fides.api.models.fides_user import FidesUser
+from fides.api.models.field_types import EncryptedLargeDataDescriptor
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.policy import (
     Policy,
@@ -72,13 +73,13 @@ from fides.api.models.privacy_request.webhook import (
     generate_request_callback_pre_approval_jwe,
     generate_request_callback_resume_jwe,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.external_https import SecondPartyResponseFormat
 from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.privacy_request import (
     CheckpointActionRequired,
-    ExecutionLogStatus,
     ManualAction,
     PrivacyRequestSource,
     PrivacyRequestStatus,
@@ -251,7 +252,8 @@ class PrivacyRequest(
     awaiting_email_send_at = Column(DateTime(timezone=True), nullable=True)
 
     # Encrypted filtered access results saved for later retrieval
-    filtered_final_upload = Column(  # An encrypted JSON String - Dict[Dict[str, List[Row]]] - rule keys mapped to the filtered access results
+    _filtered_final_upload = Column(  # An encrypted JSON String - Dict[Dict[str, List[Row]]] - rule keys mapped to the filtered access results
+        "filtered_final_upload",
         StringEncryptedType(
             type_in=JSONTypeOverride,
             key=CONFIG.security.app_encryption_key,
@@ -260,6 +262,11 @@ class PrivacyRequest(
         ),
     )
 
+    # Use descriptor for automatic external storage handling
+    filtered_final_upload = EncryptedLargeDataDescriptor(
+        field_name="filtered_final_upload", empty_default={}
+    )
+
     # Encrypted filtered access results saved for later retrieval
     access_result_urls = Column(  # An encrypted JSON String - Dict[Dict[str, List[Row]]] - rule keys mapped to the filtered access results
         StringEncryptedType(
@@ -334,6 +341,7 @@ class PrivacyRequest(
         deleting this object from the database
         """
         self.clear_cached_values()
+        self.cleanup_external_storage()
         Attachment.delete_attachments_for_reference_and_type(
             db, self.id, AttachmentReferenceType.privacy_request
         )
@@ -1257,6 +1265,11 @@ class PrivacyRequest(
         # DSR 2.0 does not cache the results so nothing to do here
         return {}
 
+    def cleanup_external_storage(self) -> None:
+        """Clean up all external storage files for this privacy request"""
+        # Access the descriptor from the class to call cleanup
+        PrivacyRequest.filtered_final_upload.cleanup(self)
+
     def save_filtered_access_results(
         self, db: Session, results: Dict[str, Dict[str, List[Row]]]
     ) -> None:
@@ -1544,7 +1557,7 @@ def get_action_required_details(
 
 
 def _parse_cache_to_checkpoint_action_required(
-    cache: dict[str, Any]
+    cache: dict[str, Any],
 ) -> CheckpointActionRequired:
     collection = (
         CollectionAddress(

fides/api/models/privacy_request/request_task.py

@@ -14,20 +14,19 @@ from sqlalchemy_utils.types.encrypted.encrypted_type import (
     StringEncryptedType,
 )
 
-from fides.api.db.base_class import Base  # type: ignore[attr-defined]
-from fides.api.db.base_class import JSONTypeOverride
-from fides.api.db.util import EnumColumn
+from fides.api.db.base_class import Base, JSONTypeOverride  # type: ignore[attr-defined]
 from fides.api.graph.config import (
     ROOT_COLLECTION_ADDRESS,
     TERMINATOR_ADDRESS,
     CollectionAddress,
 )
+from fides.api.models.field_types import EncryptedLargeDataDescriptor
 from fides.api.models.privacy_request.execution_log import (
     COMPLETED_EXECUTION_LOG_STATUSES,
 )
+from fides.api.models.worker_task import ExecutionLogStatus, WorkerTask
 from fides.api.schemas.base_class import FidesSchema
 from fides.api.schemas.policy import ActionType
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.util.cache import (
     FidesopsRedis,
     celery_tasks_in_flight,
@@ -68,7 +67,8 @@ class TraversalDetails(FidesSchema):
         )
 
 
-class RequestTask(Base):
+# TODO: At some point we will refactor this model to store all task types in a common table that links to tables with specific task attributes.
+class RequestTask(WorkerTask, Base):
     """
     An individual Task for a Privacy Request.
 
@@ -91,21 +91,6 @@ class RequestTask(Base):
     )  # Of the format dataset_name:collection_name for convenience
     dataset_name = Column(String, nullable=False, index=True)
     collection_name = Column(String, nullable=False, index=True)
-    action_type = Column(EnumColumn(ActionType), nullable=False, index=True)
-
-    # Note that RequestTasks share statuses with ExecutionLogs.  When a RequestTask changes state, an ExecutionLog
-    # is also created with that state.  These are tied tightly together in GraphTask.
-    status = Column(
-        EnumColumn(
-            ExecutionLogStatus,
-            native_enum=False,
-            values_callable=lambda x: [
-                i.value for i in x
-            ],  # Using ExecutionLogStatus values in database, even though app is using the names.
-        ),  # character varying in database
-        index=True,
-        nullable=False,
-    )
 
     upstream_tasks = Column(
         MutableList.as_mutable(JSONB)
@@ -121,7 +106,8 @@ class RequestTask(Base):
     # Raw data retrieved from an access request is stored here.  This contains all of the
     # intermediate data we retrieved, needed for downstream tasks, but hasn't been filtered
     # by data category for the end user.
-    access_data = Column(  # An encrypted JSON String - saved as a list of Rows
+    _access_data = Column(  # An encrypted JSON String - saved as a list of Rows
+        "access_data",
         StringEncryptedType(
             type_in=JSONTypeOverride,
             key=CONFIG.security.app_encryption_key,
@@ -132,7 +118,8 @@ class RequestTask(Base):
 
     # This is the raw access data saved in erasure format (with placeholders preserved) to perform a masking request.
     # First saved on the access node, and then copied to the corresponding erasure node.
-    data_for_erasures = Column(  # An encrypted JSON String - saved as a list of rows
+    _data_for_erasures = Column(  # An encrypted JSON String - saved as a list of rows
+        "data_for_erasures",
         StringEncryptedType(
             type_in=JSONTypeOverride,
             key=CONFIG.security.app_encryption_key,
@@ -141,6 +128,15 @@ class RequestTask(Base):
         ),
     )
 
+    # Use descriptors for automatic external storage handling
+    access_data = EncryptedLargeDataDescriptor(
+        field_name="access_data", empty_default=[]
+    )
+
+    data_for_erasures = EncryptedLargeDataDescriptor(
+        field_name="data_for_erasures", empty_default=[]
+    )
+
     # Written after an erasure is completed
     rows_masked = Column(Integer)
     # Written after a consent request is completed - not all consent
@@ -177,12 +173,22 @@ class RequestTask(Base):
         """Convenience helper for asserting whether the task is a terminator task"""
         return self.request_task_address == TERMINATOR_ADDRESS
 
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        return [e.value for e in ActionType]
+
     def get_cached_task_id(self) -> Optional[str]:
         """Gets the cached celery task ID for this request task."""
         cache: FidesopsRedis = get_cache()
         task_id = cache.get(get_async_task_tracking_cache_key(self.id))
         return task_id
 
+    def cleanup_external_storage(self) -> None:
+        """Clean up all external storage files for this request task"""
+        # Access the descriptor from the class to call cleanup
+        RequestTask.access_data.cleanup(self)
+        RequestTask.data_for_erasures.cleanup(self)
+
     def get_access_data(self) -> List[Row]:
         """Helper to retrieve access data or default to empty list"""
         return self.access_data or []
@@ -191,6 +197,11 @@ class RequestTask(Base):
         """Helper to retrieve erasure data needed to build masking requests or default to empty list"""
         return self.data_for_erasures or []
 
+    def delete(self, db: Session) -> None:
+        """Override delete to cleanup external storage first"""
+        self.cleanup_external_storage()
+        super().delete(db)
+
     def update_status(self, db: Session, status: ExecutionLogStatus) -> None:
         """Helper method to update a task's status"""
         self.status = status
@@ -236,7 +247,7 @@ class RequestTask(Base):
         if not tasks_complete and should_log:
             logger.debug(
                 "Upstream tasks incomplete for {} task {}.",
-                self.action_type.value,
+                self.action_type,
                 self.collection_address,
             )
 
@@ -267,7 +278,7 @@ class RequestTask(Base):
             logger.debug(
                 "Celery Task ID {} found for {} task {}.",
                 celery_task_id,
-                self.action_type.value,
+                self.action_type,
                 self.collection_address,
             )
 
@@ -277,7 +288,7 @@ class RequestTask(Base):
             logger.debug(
                 "Celery Task {} already processing for {} task {}.",
                 celery_task_id,
-                self.action_type.value,
+                self.action_type,
                 self.collection_address,
             )
 

fides/api/models/worker_task.py

@@ -0,0 +1,96 @@
+import enum
+from typing import Any, List
+
+from sqlalchemy import Column, DateTime, String
+from sqlalchemy.sql import text
+
+from fides.api.db.util import EnumColumn
+
+
+class ExecutionLogStatus(enum.Enum):
+    """Enum for task execution log statuses, reflecting where they are in their workflow"""
+
+    in_processing = "in_processing"
+    pending = "pending"
+    complete = "complete"
+    error = "error"
+    awaiting_processing = "paused"  # "paused" in the database to avoid a migration, but use "awaiting_processing" in the app
+    retrying = "retrying"
+    skipped = "skipped"
+
+
+class WorkerTask:
+    """
+    A task for a worker to execute.
+    """
+
+    # Field called action_type to avoid migrations in RequestTask when creating this model
+    action_type = Column(String, nullable=False, index=True)
+    # Note that WorkerTask share statuses with ExecutionLogs.  When a WorkerTask changes state, an ExecutionLog
+    # is also created with that state.  These are tied tightly together in GraphTask.
+    status = Column(
+        EnumColumn(
+            ExecutionLogStatus,
+            native_enum=False,
+            values_callable=lambda x: [
+                i.value for i in x
+            ],  # Using ExecutionLogStatus values in database, even though app is using the names.
+        ),  # character varying in database
+        index=True,
+        nullable=False,
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        action_type = kwargs.get("action_type")
+        if action_type is not None:
+            self.validate_action_type(action_type)
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def validate_action_type(cls, action_type: str) -> None:
+        """
+        Validates that the action type is allowed for the worker task.
+        """
+        if action_type not in cls.allowed_action_types():
+            raise ValueError(f"Invalid action_type '{action_type}' for {cls.__name__}")
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        """
+        Subclasses must implement this method to return a list of allowed action types.
+        """
+        raise NotImplementedError("Subclasses must implement allowed_action_types")
+
+
+class TaskExecutionLog:
+    """
+    Stores the individual execution logs associated with a WorkerTask.
+    """
+
+    status = Column(
+        EnumColumn(
+            ExecutionLogStatus,
+            native_enum=True,
+            values_callable=lambda x: [
+                i.value for i in x
+            ],  # Using ExecutionLogStatus values in database, even though app is using the names.
+        ),
+        index=True,
+        nullable=False,
+    )
+    # Contains info, warning, or error messages
+    message = Column(String)
+
+    # Use clock_timestamp() instead of NOW() to get the actual current time at row creation,
+    # regardless of transaction state. This prevents timestamp caching within transactions
+    # and ensures more accurate creation times.
+    # https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-CURRENT
+
+    created_at = Column(
+        DateTime(timezone=True), server_default=text("clock_timestamp()")
+    )
+    updated_at = Column(
+        DateTime(timezone=True),
+        server_default=text("clock_timestamp()"),
+        onupdate=text("clock_timestamp()"),
+    )

fides/api/schemas/external_storage.py

@@ -0,0 +1,22 @@
+"""Schema for external storage metadata."""
+
+from typing import Optional
+
+from pydantic import Field
+
+from fides.api.schemas.base_class import FidesSchema
+from fides.api.schemas.storage.storage import StorageType
+
+
+class ExternalStorageMetadata(FidesSchema):
+    """Metadata for externally stored encrypted data."""
+
+    storage_type: StorageType
+    file_key: str = Field(description="Path/key of the file in external storage")
+    filesize: int = Field(description="Size of the stored file in bytes", ge=0)
+    storage_key: Optional[str] = Field(
+        default=None, description="Storage configuration key used"
+    )
+
+    class Config:
+        use_enum_values = True