ethyca-fides 2.63.1b3__py2.py3-none-any.whl → 2.63.1b4__py2.py3-none-any.whl

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -75,6 +75,7 @@ from fides.api.models.privacy_request import (
     ProvidedIdentity,
     RequestTask,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.oauth.utils import (
     verify_callback_oauth_policy_pre_webhook,
     verify_callback_oauth_pre_approval_webhook,
@@ -91,7 +92,6 @@ from fides.api.schemas.privacy_request import (
     CheckpointActionRequired,
     DenyPrivacyRequests,
     ExecutionLogDetailResponse,
-    ExecutionLogStatus,
     FilteredPrivacyRequestResults,
     LogEntry,
     ManualWebhookData,
@@ -1940,16 +1940,16 @@ def request_task_async_callback(
     ]:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
+            detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
         )
     if request_task.status != ExecutionLogStatus.awaiting_processing:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with request task status '{request_task.status.value}'",
+            detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with request task status '{request_task.status.value}'",
         )
     logger.info(
         "Callback received for {} task {} {}",
-        request_task.action_type.value,
+        request_task.action_type,
         request_task.collection_address,
         request_task.id,
     )

fides/api/db/base.py

@@ -16,7 +16,11 @@ from fides.api.models.custom_connector_template import CustomConnectorTemplate
 from fides.api.models.custom_report import CustomReport
 from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.models.db_cache import DBCache
-from fides.api.models.detection_discovery import MonitorConfig, StagedResource
+from fides.api.models.detection_discovery.core import MonitorConfig, StagedResource
+from fides.api.models.detection_discovery.monitor_task import (
+    MonitorTask,
+    MonitorTaskExecutionLog,
+)
 from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.fides_cloud import FidesCloud
 from fides.api.models.fides_user import FidesUser

fides/api/models/connectionconfig.py

@@ -23,7 +23,7 @@ from fides.api.schemas.saas.saas_config import SaaSConfig
 from fides.config import CONFIG
 
 if TYPE_CHECKING:
-    from fides.api.models.detection_discovery import MonitorConfig
+    from fides.api.models.detection_discovery.core import MonitorConfig
     from fides.api.schemas.connection_configuration.enums.system_type import SystemType
 
 

fides/api/models/detection_discovery/__init__.py

@@ -0,0 +1,35 @@
+from .core import (
+    DiffStatus,
+    MonitorConfig,
+    MonitorExecution,
+    MonitorFrequency,
+    SharedMonitorConfig,
+    StagedResource,
+    StagedResourceAncestor,
+    fetch_staged_resources_by_type_query,
+)
+from .monitor_task import (
+    MonitorTask,
+    MonitorTaskExecutionLog,
+    MonitorTaskType,
+    TaskRunType,
+    create_monitor_task_with_execution_log,
+    update_monitor_task_with_execution_log,
+)
+
+__all__ = [
+    "DiffStatus",
+    "MonitorConfig",
+    "MonitorExecution",
+    "MonitorFrequency",
+    "SharedMonitorConfig",
+    "StagedResource",
+    "StagedResourceAncestor",
+    "fetch_staged_resources_by_type_query",
+    "MonitorTask",
+    "MonitorTaskExecutionLog",
+    "MonitorTaskType",
+    "TaskRunType",
+    "create_monitor_task_with_execution_log",
+    "update_monitor_task_with_execution_log",
+]

fides/api/models/detection_discovery/monitor_task.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Optional
+
+from sqlalchemy import ARRAY, Column
+from sqlalchemy import Enum as SQLAlchemyEnum
+from sqlalchemy import ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base, FidesBase  # type: ignore[attr-defined]
+from fides.api.models.detection_discovery.core import MonitorConfig
+from fides.api.models.worker_task import (
+    ExecutionLogStatus,
+    TaskExecutionLog,
+    WorkerTask,
+)
+
+
+class MonitorTaskType(Enum):
+    """
+    Types of tasks that can be executed by a worker.
+    """
+
+    DETECTION = "detection"
+    CLASSIFICATION = "classification"
+    PROMOTION = "promotion"
+
+
+class MonitorTask(WorkerTask, Base):
+    """
+    A monitor task executed by a worker.
+    """
+
+    # celery_id is used to track task executions. While MonitorTask.id remains constant,
+    # celery_id changes with each execution or retry of the task, allowing us to track
+    # the current execution state while maintaining a stable reference to the original task.
+    celery_id = Column(
+        String(255), unique=True, nullable=False, default=FidesBase.generate_uuid
+    )
+    task_arguments = Column(JSONB, nullable=True)  # To be able to rerun the task
+    # Contains info, warning, or error messages
+    message = Column(String)
+    monitor_config_id = Column(
+        String,
+        ForeignKey(MonitorConfig.id_field_path, ondelete="CASCADE"),
+        index=True,
+        nullable=False,
+    )
+    staged_resource_urns = Column(ARRAY(String), nullable=True)
+    child_resource_urns = Column(ARRAY(String), nullable=True)
+
+    monitor_config = relationship(MonitorConfig, cascade="all, delete")
+    execution_logs = relationship(
+        "MonitorTaskExecutionLog", back_populates="monitor_task", cascade="all, delete"
+    )
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        return [e.value for e in MonitorTaskType]
+
+
+class TaskRunType(Enum):
+    """
+    Type of task run.
+    """
+
+    MANUAL = "manual"
+    SYSTEM = "system"
+
+
+class MonitorTaskExecutionLog(TaskExecutionLog, Base):
+    """
+    Stores the individual execution logs associated with a MonitorTask.
+    """
+
+    # This celery_id preserves the specific execution ID for historical tracking,
+    # unlike MonitorTask.celery_id which is updated with each execution.
+    # This allows us to maintain a complete history of all task execution attempts.
+    celery_id = Column(String(255), nullable=False)
+    monitor_task_id = Column(
+        String,
+        ForeignKey(MonitorTask.id_field_path, ondelete="CASCADE"),
+        index=True,
+        nullable=False,
+    )
+    run_type = Column(
+        SQLAlchemyEnum(TaskRunType), nullable=False, default=TaskRunType.SYSTEM
+    )
+
+    monitor_task = relationship("MonitorTask", back_populates="execution_logs")
+
+
+def create_monitor_task_with_execution_log(
+    db: Session, monitor_task_data: dict
+) -> MonitorTask:
+    """
+    Creates a monitor task with an execution log.
+    The default status is pending for the task and pending for the execution log.
+    """
+    status = ExecutionLogStatus.pending
+    task_record = MonitorTask(  # type: ignore
+        status=status.value,
+        **monitor_task_data,
+    )
+    db.add(task_record)
+    db.flush()
+
+    execution_log = MonitorTaskExecutionLog(  # type: ignore
+        monitor_task=task_record, celery_id=task_record.celery_id, status=status
+    )
+    db.add(execution_log)
+
+    db.commit()
+    db.refresh(task_record)
+    return task_record
+
+
+def update_monitor_task_with_execution_log(
+    db: Session,
+    status: ExecutionLogStatus,
+    task_record: Optional[MonitorTask] = None,
+    celery_id: Optional[str] = None,
+    message: Optional[str] = None,
+    run_type: TaskRunType = TaskRunType.SYSTEM,
+) -> MonitorTask:
+    """
+    Updates a monitor task with an execution log.
+
+    It must be either celery_id or task_record. If it doesn't receive a celery_id, it's assumed a new one needs to be created because a new run is about to be performed.
+    If it receives a celery_id, it means it only needs to update the status of an existing run. It can receive task_record to avoid querying the database again to get it.
+    """
+    if not celery_id and not task_record:
+        raise ValueError("Either celery_id or task_record must be provided")
+
+    if celery_id and not task_record:
+        task_record = MonitorTask.get_by(db=db, field="celery_id", value=celery_id)
+        if not task_record:
+            raise ValueError(f"Could not find MonitorTask with celery_id {celery_id}")
+
+    assert task_record is not None  # help type checker understand the control flow
+
+    if not celery_id:
+        celery_id = task_record.generate_uuid()
+        task_record.celery_id = celery_id
+
+    task_record.status = status.value  # type: ignore
+    task_record.message = message
+
+    MonitorTaskExecutionLog(  # type: ignore
+        monitor_task=task_record,
+        status=status,
+        message=message,
+        celery_id=celery_id,
+        run_type=run_type,
+    )
+
+    db.commit()
+    db.refresh(task_record)
+    return task_record

fides/api/models/field_types/__init__.py

@@ -0,0 +1,5 @@
+from .encrypted_large_data import EncryptedLargeDataDescriptor
+
+__all__ = [
+    "EncryptedLargeDataDescriptor",
+]

fides/api/models/field_types/encrypted_large_data.py

@@ -0,0 +1,151 @@
+from datetime import datetime
+from typing import Any, Optional, Type
+
+from loguru import logger
+
+from fides.api.api.deps import get_autoclose_db_session
+from fides.api.schemas.external_storage import ExternalStorageMetadata
+from fides.api.service.external_data_storage import (
+    ExternalDataStorageError,
+    ExternalDataStorageService,
+)
+from fides.api.util.data_size import LARGE_DATA_THRESHOLD_BYTES, calculate_data_size
+
+
+class EncryptedLargeDataDescriptor:
+    """
+    A Python descriptor for database fields with encrypted external storage fallback.
+
+    See the original implementation for detailed docstrings.
+    """
+
+    def __init__(
+        self,
+        field_name: str,
+        empty_default: Optional[Any] = None,
+        threshold_bytes: Optional[int] = None,
+    ):
+        self.field_name = field_name
+        self.private_field = f"_{field_name}"
+        self.empty_default = empty_default if empty_default is not None else []
+        self.threshold_bytes = threshold_bytes or LARGE_DATA_THRESHOLD_BYTES
+        self.model_class: Optional[str] = None
+        self.name: Optional[str] = None
+
+    # Descriptor protocol helpers
+
+    def __set_name__(
+        self, owner: Type, name: str
+    ) -> None:  # noqa: D401 (docstring in orig file)
+        self.name = name
+        self.model_class = owner.__name__
+
+    def _generate_storage_path(self, instance: Any) -> str:
+        instance_id = getattr(instance, "id", None)
+        if not instance_id:
+            raise ValueError(f"Instance {instance} must have an 'id' attribute")
+        timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S-%f")
+        return f"{self.model_class}/{instance_id}/{self.field_name}/{timestamp}.txt"
+
+    def __get__(self, instance: Any, owner: Type) -> Any:  # noqa: D401
+        if instance is None:
+            return self
+        raw_data = getattr(instance, self.private_field)
+        if raw_data is None:
+            return None
+        if isinstance(raw_data, dict) and "storage_type" in raw_data:
+            logger.info(
+                f"Reading {self.model_class}.{self.field_name} from external storage "
+                f"({raw_data.get('storage_type')})"
+            )
+            try:
+                metadata = ExternalStorageMetadata.model_validate(raw_data)
+                data = self._retrieve_external_data(metadata)
+                record_count = len(data) if isinstance(data, list) else "N/A"
+                logger.info(
+                    f"Successfully retrieved {self.model_class}.{self.field_name} "
+                    f"from external storage (records: {record_count})"
+                )
+                return data if data is not None else self.empty_default
+            except Exception as e:  # pylint: disable=broad-except
+                logger.error(
+                    f"Failed to retrieve {self.model_class}.{self.field_name} "
+                    f"from external storage: {str(e)}"
+                )
+                raise ExternalDataStorageError(
+                    f"Failed to retrieve {self.field_name}: {str(e)}"
+                ) from e
+        else:
+            return raw_data
+
+    def __set__(self, instance: Any, value: Any) -> None:  # noqa: D401
+        if not value:
+            self._cleanup_external_data(instance)
+            setattr(instance, self.private_field, self.empty_default)
+            return
+        try:
+            current_data = self.__get__(instance, type(instance))
+            if current_data == value:
+                return
+        except Exception:  # pylint: disable=broad-except
+            pass
+
+        data_size = calculate_data_size(value)
+        if data_size > self.threshold_bytes:
+            logger.info(
+                f"{self.model_class}.{self.field_name}: Data size ({data_size:,} bytes) "
+                f"exceeds threshold ({self.threshold_bytes:,} bytes), storing externally"
+            )
+            self._cleanup_external_data(instance)
+            metadata = self._store_external_data(instance, value)
+            setattr(instance, self.private_field, metadata.model_dump())
+        else:
+            self._cleanup_external_data(instance)
+            setattr(instance, self.private_field, value)
+
+    # External storage helpers
+
+    def _store_external_data(self, instance: Any, data: Any) -> ExternalStorageMetadata:
+        storage_path = self._generate_storage_path(instance)
+        with get_autoclose_db_session() as session:
+            metadata = ExternalDataStorageService.store_data(
+                db=session,
+                storage_path=storage_path,
+                data=data,
+            )
+            logger.info(
+                f"Stored {self.model_class}.{self.field_name} to external storage: {storage_path}"
+            )
+            return metadata
+
+    @staticmethod
+    def _retrieve_external_data(metadata: ExternalStorageMetadata) -> Any:  # noqa: D401
+        with get_autoclose_db_session() as session:
+            return ExternalDataStorageService.retrieve_data(
+                db=session,
+                metadata=metadata,
+            )
+
+    def _cleanup_external_data(self, instance: Any) -> None:  # noqa: D401
+        raw_data = getattr(instance, self.private_field, None)
+        if isinstance(raw_data, dict) and "storage_type" in raw_data:
+            try:
+                metadata = ExternalStorageMetadata.model_validate(raw_data)
+                with get_autoclose_db_session() as session:
+                    ExternalDataStorageService.delete_data(
+                        db=session,
+                        metadata=metadata,
+                    )
+                logger.info(
+                    f"Cleaned up external storage for {self.model_class}.{self.field_name}: "
+                    f"{metadata.file_key}"
+                )
+            except Exception as e:  # pylint: disable=broad-except
+                logger.warning(
+                    f"Failed to cleanup external {self.field_name}: {str(e)}"
+                )
+
+    # Public helper
+
+    def cleanup(self, instance: Any) -> None:  # noqa: D401
+        self._cleanup_external_data(instance)

fides/api/models/privacy_preference.py

@@ -22,8 +22,8 @@ from fides.api.models.privacy_notice import (
     UserConsentPreference,
 )
 from fides.api.models.privacy_request import PrivacyRequest, ProvidedIdentity
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.language import SupportedLanguage
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 from fides.api.schemas.redis_cache import MultiValue
 from fides.config import CONFIG
 

fides/api/models/privacy_request/execution_log.py

@@ -4,15 +4,14 @@ from __future__ import annotations
 
 from typing import Optional
 
-from sqlalchemy import Column, DateTime, String
+from sqlalchemy import Column, String
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.mutable import MutableList
-from sqlalchemy.sql import text
 
 from fides.api.db.base_class import Base  # type: ignore[attr-defined]
 from fides.api.db.util import EnumColumn
+from fides.api.models.worker_task import ExecutionLogStatus, TaskExecutionLog
 from fides.api.schemas.policy import ActionType, CurrentStep
-from fides.api.schemas.privacy_request import ExecutionLogStatus
 
 # Locations from which privacy request execution can be resumed, in order.
 EXECUTION_CHECKPOINTS = [
@@ -53,7 +52,7 @@ def can_run_checkpoint(
     ) >= EXECUTION_CHECKPOINTS.index(from_checkpoint)
 
 
-class ExecutionLog(Base):
+class ExecutionLog(TaskExecutionLog, Base):
     """
     Stores the individual execution logs associated with a PrivacyRequest.
 
@@ -68,41 +67,14 @@ class ExecutionLog(Base):
     collection_name = Column(String, index=True)
     # A JSON Array describing affected fields along with their data categories and paths
     fields_affected = Column(MutableList.as_mutable(JSONB), nullable=True)
-    # Contains info, warning, or error messages
-    message = Column(String)
     action_type = Column(
         EnumColumn(ActionType),
         index=True,
         nullable=False,
     )
-    status = Column(
-        EnumColumn(
-            ExecutionLogStatus,
-            native_enum=True,
-            values_callable=lambda x: [
-                i.value for i in x
-            ],  # Using ExecutionLogStatus values in database, even though app is using the names.
-        ),
-        index=True,
-        nullable=False,
-    )
 
     privacy_request_id = Column(
         String,
         nullable=False,
         index=True,
     )
-
-    # Use clock_timestamp() instead of NOW() to get the actual current time at row creation,
-    # regardless of transaction state. This prevents timestamp caching within transactions
-    # and ensures more accurate creation times.
-    # https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-CURRENT
-
-    created_at = Column(
-        DateTime(timezone=True), server_default=text("clock_timestamp()")
-    )
-    updated_at = Column(
-        DateTime(timezone=True),
-        server_default=text("clock_timestamp()"),
-        onupdate=text("clock_timestamp()"),
-    )

fides/api/models/privacy_request/privacy_request.py

@@ -48,6 +48,7 @@ from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
 from fides.api.models.comment import Comment, CommentReference, CommentReferenceType
 from fides.api.models.fides_user import FidesUser
+from fides.api.models.field_types import EncryptedLargeDataDescriptor
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.policy import (
     Policy,
@@ -72,13 +73,13 @@ from fides.api.models.privacy_request.webhook import (
     generate_request_callback_pre_approval_jwe,
     generate_request_callback_resume_jwe,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.external_https import SecondPartyResponseFormat
 from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.privacy_request import (
     CheckpointActionRequired,
-    ExecutionLogStatus,
     ManualAction,
     PrivacyRequestSource,
     PrivacyRequestStatus,
@@ -251,7 +252,8 @@ class PrivacyRequest(
     awaiting_email_send_at = Column(DateTime(timezone=True), nullable=True)
 
     # Encrypted filtered access results saved for later retrieval
-    filtered_final_upload = Column(  # An encrypted JSON String - Dict[Dict[str, List[Row]]] - rule keys mapped to the filtered access results
+    _filtered_final_upload = Column(  # An encrypted JSON String - Dict[Dict[str, List[Row]]] - rule keys mapped to the filtered access results
+        "filtered_final_upload",
         StringEncryptedType(
             type_in=JSONTypeOverride,
             key=CONFIG.security.app_encryption_key,
@@ -260,6 +262,11 @@ class PrivacyRequest(
         ),
     )
 
+    # Use descriptor for automatic external storage handling
+    filtered_final_upload = EncryptedLargeDataDescriptor(
+        field_name="filtered_final_upload", empty_default={}
+    )
+
     # Encrypted filtered access results saved for later retrieval
     access_result_urls = Column(  # An encrypted JSON String - Dict[Dict[str, List[Row]]] - rule keys mapped to the filtered access results
         StringEncryptedType(
@@ -334,6 +341,7 @@ class PrivacyRequest(
         deleting this object from the database
         """
         self.clear_cached_values()
+        self.cleanup_external_storage()
         Attachment.delete_attachments_for_reference_and_type(
             db, self.id, AttachmentReferenceType.privacy_request
         )
@@ -1257,6 +1265,11 @@ class PrivacyRequest(
         # DSR 2.0 does not cache the results so nothing to do here
         return {}
 
+    def cleanup_external_storage(self) -> None:
+        """Clean up all external storage files for this privacy request"""
+        # Access the descriptor from the class to call cleanup
+        PrivacyRequest.filtered_final_upload.cleanup(self)
+
     def save_filtered_access_results(
         self, db: Session, results: Dict[str, Dict[str, List[Row]]]
     ) -> None:
@@ -1544,7 +1557,7 @@ def get_action_required_details(
 
 
 def _parse_cache_to_checkpoint_action_required(
-    cache: dict[str, Any]
+    cache: dict[str, Any],
 ) -> CheckpointActionRequired:
     collection = (
         CollectionAddress(