ethyca-fides 2.63.1b3__py2.py3-none-any.whl → 2.63.1rc0__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/29e56fa1fdb3_add_monitor_tasks.py

@@ -0,0 +1,147 @@
+"""add_monitor_tasks
+
+Revision ID: 29e56fa1fdb3
+Revises: 5efcdf18438e
+Create Date: 2025-06-11 14:40:08.384571
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "29e56fa1fdb3"
+down_revision = "5efcdf18438e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "monitortask",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("action_type", sa.String(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "in_processing",
+                "pending",
+                "complete",
+                "error",
+                "paused",
+                "retrying",
+                "skipped",
+                name="executionlogstatus",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+        sa.Column("celery_id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "task_arguments", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column("message", sa.String(), nullable=True),
+        sa.Column("monitor_config_id", sa.String(), nullable=False),
+        sa.Column("staged_resource_urns", sa.ARRAY(sa.String()), nullable=True),
+        sa.Column("child_resource_urns", sa.ARRAY(sa.String()), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["monitor_config_id"], ["monitorconfig.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("celery_id"),
+    )
+    op.create_index(
+        op.f("ix_monitortask_action_type"), "monitortask", ["action_type"], unique=False
+    )
+    op.create_index(op.f("ix_monitortask_id"), "monitortask", ["id"], unique=False)
+    op.create_index(
+        op.f("ix_monitortask_monitor_config_id"),
+        "monitortask",
+        ["monitor_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_monitortask_status"), "monitortask", ["status"], unique=False
+    )
+    op.create_table(
+        "monitortaskexecutionlog",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "status",
+            postgresql.ENUM(name="executionlogstatus", create_type=False),
+            nullable=False,
+        ),
+        sa.Column("message", sa.String(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("clock_timestamp()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("clock_timestamp()"),
+            nullable=True,
+        ),
+        sa.Column("celery_id", sa.String(length=255), nullable=False),
+        sa.Column("monitor_task_id", sa.String(), nullable=False),
+        sa.Column(
+            "run_type", sa.Enum("MANUAL", "SYSTEM", name="taskruntype"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["monitor_task_id"], ["monitortask.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_monitortaskexecutionlog_id"),
+        "monitortaskexecutionlog",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_monitortaskexecutionlog_monitor_task_id"),
+        "monitortaskexecutionlog",
+        ["monitor_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_monitortaskexecutionlog_status"),
+        "monitortaskexecutionlog",
+        ["status"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_monitortaskexecutionlog_status"), table_name="monitortaskexecutionlog"
+    )
+    op.drop_index(
+        op.f("ix_monitortaskexecutionlog_monitor_task_id"),
+        table_name="monitortaskexecutionlog",
+    )
+    op.drop_index(
+        op.f("ix_monitortaskexecutionlog_id"), table_name="monitortaskexecutionlog"
+    )
+    op.drop_table("monitortaskexecutionlog")
+    op.drop_index(op.f("ix_monitortask_status"), table_name="monitortask")
+    op.drop_index(op.f("ix_monitortask_monitor_config_id"), table_name="monitortask")
+    op.drop_index(op.f("ix_monitortask_id"), table_name="monitortask")
+    op.drop_index(op.f("ix_monitortask_action_type"), table_name="monitortask")
+    op.drop_table("monitortask")
+    op.execute("DROP TYPE IF EXISTS taskruntype")

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -75,6 +75,7 @@ from fides.api.models.privacy_request import (
     ProvidedIdentity,
     RequestTask,
 )
+from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.oauth.utils import (
     verify_callback_oauth_policy_pre_webhook,
     verify_callback_oauth_pre_approval_webhook,
@@ -91,7 +92,6 @@ from fides.api.schemas.privacy_request import (
     CheckpointActionRequired,
     DenyPrivacyRequests,
     ExecutionLogDetailResponse,
-    ExecutionLogStatus,
     FilteredPrivacyRequestResults,
     LogEntry,
     ManualWebhookData,
@@ -1940,16 +1940,16 @@ def request_task_async_callback(
     ]:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
+            detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
         )
     if request_task.status != ExecutionLogStatus.awaiting_processing:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with request task status '{request_task.status.value}'",
+            detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with request task status '{request_task.status.value}'",
         )
     logger.info(
         "Callback received for {} task {} {}",
-        request_task.action_type.value,
+        request_task.action_type,
         request_task.collection_address,
         request_task.id,
     )

fides/api/db/base.py

@@ -16,7 +16,11 @@ from fides.api.models.custom_connector_template import CustomConnectorTemplate
 from fides.api.models.custom_report import CustomReport
 from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.models.db_cache import DBCache
-from fides.api.models.detection_discovery import MonitorConfig, StagedResource
+from fides.api.models.detection_discovery.core import MonitorConfig, StagedResource
+from fides.api.models.detection_discovery.monitor_task import (
+    MonitorTask,
+    MonitorTaskExecutionLog,
+)
 from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.fides_cloud import FidesCloud
 from fides.api.models.fides_user import FidesUser
@@ -27,8 +31,6 @@ from fides.api.models.fides_user_respondent_email_verification import (
 )
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
-from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
-from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.messaging import MessagingConfig
 from fides.api.models.messaging_template import MessagingTemplate

fides/api/main.py

@@ -244,7 +244,6 @@ async def log_request(request: Request, call_next: Callable) -> Response:
         status_code=response.status_code,
         handler_time=f"{total_time}ms",
         path=request.url.path,
-        fides_client=request.headers.get("Fides-Client", "unknown"),
     ).info("Request received")
     return response
 

fides/api/models/attachment.py

@@ -1,6 +1,5 @@
 import os
 from enum import Enum as EnumType
-from io import BytesIO
 from typing import IO, TYPE_CHECKING, Any, Tuple
 
 from fideslang.validation import AnyHttpUrlString
@@ -20,6 +19,7 @@ from fides.api.service.storage.s3 import (
     generic_upload_to_s3,
 )
 from fides.api.service.storage.util import AllowedFileType, get_local_filename
+from fides.config import CONFIG
 
 if TYPE_CHECKING:
     from fides.api.models.comment import Comment
@@ -28,11 +28,6 @@ if TYPE_CHECKING:
     from fides.api.models.storage import StorageConfig
 
 
-# This is 7 days in seconds and is currently the max allowed
-# configurable expiration time for presigned URLs for both s3 and gcs.
-MAX_TTL_SECONDS = 604800
-
-
 class AttachmentType(str, EnumType):
     """
     Enum for attachment types. Indicates attachment usage.
@@ -127,11 +122,6 @@ class Attachment(Base):
         """Returns the content type of the attachment."""
         return AllowedFileType[self.file_name.split(".")[-1]].value
 
-    @property
-    def file_key(self) -> str:
-        """Returns the file key of the attachment."""
-        return f"{self.id}/{self.file_name}"
-
     def upload(self, attachment: IO[bytes]) -> None:
         """Uploads an attachment to S3, GCS, or local storage."""
         if self.config.type == StorageType.s3:
@@ -140,7 +130,7 @@ class Attachment(Base):
             generic_upload_to_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=self.file_key,
+                file_key=f"{self.id}/{self.file_name}",
                 document=attachment,
                 auth_method=auth_method,
             )
@@ -152,7 +142,7 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(self.file_key)
+            blob = bucket.blob(f"{self.id}/{self.file_name}")
 
             # Reset the file pointer to the beginning
             try:
@@ -165,7 +155,7 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(self.file_key)
+            filename = get_local_filename(f"{self.id}/{self.file_name}")
 
             # Validate that attachment is a file-like object
             if not hasattr(attachment, "read"):
@@ -215,10 +205,9 @@ class Attachment(Base):
             size, url = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=self.file_key,
+                file_key=f"{self.id}/{self.file_name}",
                 auth_method=auth_method,
                 get_content=False,
-                ttl_seconds=MAX_TTL_SECONDS,
             )
             return size, url
 
@@ -227,20 +216,19 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(self.file_key)
+            blob = bucket.blob(f"{self.id}/{self.file_name}")
 
             # Ensure we have the blob metadata
             blob.reload()
-            # Expiration is set to 7 days
             url = blob.generate_signed_url(
                 version="v4",
-                expiration=MAX_TTL_SECONDS,
+                expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
                 method="GET",
             )
             return blob.size, url
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(self.file_key)
+            filename = get_local_filename(f"{self.id}/{self.file_name}")
             size = os.path.getsize(filename)
             return size, filename
 
@@ -248,7 +236,7 @@ class Attachment(Base):
 
     def retrieve_attachment_content(
         self,
-    ) -> Tuple[int, IO[bytes]]:
+    ) -> Tuple[int, bytes]:
         """
         Retrieves the size of the attachment and its actual content.
         - For s3:
@@ -264,33 +252,30 @@ class Attachment(Base):
         if self.config.type == StorageType.s3:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
-            size, fileobj = generic_retrieve_from_s3(
+            size, content = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=self.file_key,
+                file_key=f"{self.id}/{self.file_name}",
                 auth_method=auth_method,
                 get_content=True,
             )
-            return size, fileobj
+            return size, content
 
         if self.config.type == StorageType.gcs:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(self.file_key)
+            blob = bucket.blob(f"{self.id}/{self.file_name}")
 
-            fileobj = BytesIO()
-            blob.download_to_file(fileobj)
-            fileobj.seek(0)  # Reset pointer to beginning after download
-            return blob.size, fileobj
+            content = blob.download_as_bytes()
+            return len(content), content
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(self.file_key)
-            size = os.path.getsize(filename)
-            with open(filename, "rb") as fileobj:
-                content = BytesIO(fileobj.read())
-                content.seek(0)  # Reset pointer to beginning
+            filename = get_local_filename(f"{self.id}/{self.file_name}")
+            with open(filename, "rb") as file:
+                content = file.read()
+                size = len(content)
                 return size, content
 
         raise ValueError(f"Unsupported storage type: {self.config.type}")
@@ -303,7 +288,7 @@ class Attachment(Base):
             generic_delete_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=self.file_key,
+                file_key=f"{self.id}/{self.file_name}",
                 auth_method=auth_method,
             )
             return
@@ -322,7 +307,9 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            folder_path = os.path.dirname(get_local_filename(self.file_key))
+            folder_path = os.path.dirname(
+                get_local_filename(f"{self.id}/{self.file_name}")
+            )
             if os.path.exists(folder_path):
                 import shutil
 

fides/api/models/connectionconfig.py

@@ -23,7 +23,7 @@ from fides.api.schemas.saas.saas_config import SaaSConfig
 from fides.config import CONFIG
 
 if TYPE_CHECKING:
-    from fides.api.models.detection_discovery import MonitorConfig
+    from fides.api.models.detection_discovery.core import MonitorConfig
     from fides.api.schemas.connection_configuration.enums.system_type import SystemType
 
 

fides/api/models/detection_discovery/__init__.py

@@ -0,0 +1,35 @@
+from .core import (
+    DiffStatus,
+    MonitorConfig,
+    MonitorExecution,
+    MonitorFrequency,
+    SharedMonitorConfig,
+    StagedResource,
+    StagedResourceAncestor,
+    fetch_staged_resources_by_type_query,
+)
+from .monitor_task import (
+    MonitorTask,
+    MonitorTaskExecutionLog,
+    MonitorTaskType,
+    TaskRunType,
+    create_monitor_task_with_execution_log,
+    update_monitor_task_with_execution_log,
+)
+
+__all__ = [
+    "DiffStatus",
+    "MonitorConfig",
+    "MonitorExecution",
+    "MonitorFrequency",
+    "SharedMonitorConfig",
+    "StagedResource",
+    "StagedResourceAncestor",
+    "fetch_staged_resources_by_type_query",
+    "MonitorTask",
+    "MonitorTaskExecutionLog",
+    "MonitorTaskType",
+    "TaskRunType",
+    "create_monitor_task_with_execution_log",
+    "update_monitor_task_with_execution_log",
+]

fides/api/models/detection_discovery/monitor_task.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Optional
+
+from sqlalchemy import ARRAY, Column
+from sqlalchemy import Enum as SQLAlchemyEnum
+from sqlalchemy import ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base, FidesBase  # type: ignore[attr-defined]
+from fides.api.models.detection_discovery.core import MonitorConfig
+from fides.api.models.worker_task import (
+    ExecutionLogStatus,
+    TaskExecutionLog,
+    WorkerTask,
+)
+
+
+class MonitorTaskType(Enum):
+    """
+    Types of tasks that can be executed by a worker.
+    """
+
+    DETECTION = "detection"
+    CLASSIFICATION = "classification"
+    PROMOTION = "promotion"
+
+
+class MonitorTask(WorkerTask, Base):
+    """
+    A monitor task executed by a worker.
+    """
+
+    # celery_id is used to track task executions. While MonitorTask.id remains constant,
+    # celery_id changes with each execution or retry of the task, allowing us to track
+    # the current execution state while maintaining a stable reference to the original task.
+    celery_id = Column(
+        String(255), unique=True, nullable=False, default=FidesBase.generate_uuid
+    )
+    task_arguments = Column(JSONB, nullable=True)  # To be able to rerun the task
+    # Contains info, warning, or error messages
+    message = Column(String)
+    monitor_config_id = Column(
+        String,
+        ForeignKey(MonitorConfig.id_field_path, ondelete="CASCADE"),
+        index=True,
+        nullable=False,
+    )
+    staged_resource_urns = Column(ARRAY(String), nullable=True)
+    child_resource_urns = Column(ARRAY(String), nullable=True)
+
+    monitor_config = relationship(MonitorConfig, cascade="all, delete")
+    execution_logs = relationship(
+        "MonitorTaskExecutionLog", back_populates="monitor_task", cascade="all, delete"
+    )
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        return [e.value for e in MonitorTaskType]
+
+
+class TaskRunType(Enum):
+    """
+    Type of task run.
+    """
+
+    MANUAL = "manual"
+    SYSTEM = "system"
+
+
+class MonitorTaskExecutionLog(TaskExecutionLog, Base):
+    """
+    Stores the individual execution logs associated with a MonitorTask.
+    """
+
+    # This celery_id preserves the specific execution ID for historical tracking,
+    # unlike MonitorTask.celery_id which is updated with each execution.
+    # This allows us to maintain a complete history of all task execution attempts.
+    celery_id = Column(String(255), nullable=False)
+    monitor_task_id = Column(
+        String,
+        ForeignKey(MonitorTask.id_field_path, ondelete="CASCADE"),
+        index=True,
+        nullable=False,
+    )
+    run_type = Column(
+        SQLAlchemyEnum(TaskRunType), nullable=False, default=TaskRunType.SYSTEM
+    )
+
+    monitor_task = relationship("MonitorTask", back_populates="execution_logs")
+
+
+def create_monitor_task_with_execution_log(
+    db: Session, monitor_task_data: dict
+) -> MonitorTask:
+    """
+    Creates a monitor task with an execution log.
+    The default status is pending for the task and pending for the execution log.
+    """
+    status = ExecutionLogStatus.pending
+    task_record = MonitorTask(  # type: ignore
+        status=status.value,
+        **monitor_task_data,
+    )
+    db.add(task_record)
+    db.flush()
+
+    execution_log = MonitorTaskExecutionLog(  # type: ignore
+        monitor_task=task_record, celery_id=task_record.celery_id, status=status
+    )
+    db.add(execution_log)
+
+    db.commit()
+    db.refresh(task_record)
+    return task_record
+
+
+def update_monitor_task_with_execution_log(
+    db: Session,
+    status: ExecutionLogStatus,
+    task_record: Optional[MonitorTask] = None,
+    celery_id: Optional[str] = None,
+    message: Optional[str] = None,
+    run_type: TaskRunType = TaskRunType.SYSTEM,
+) -> MonitorTask:
+    """
+    Updates a monitor task with an execution log.
+
+    It must be either celery_id or task_record. If it doesn't receive a celery_id, it's assumed a new one needs to be created because a new run is about to be performed.
+    If it receives a celery_id, it means it only needs to update the status of an existing run. It can receive task_record to avoid querying the database again to get it.
+    """
+    if not celery_id and not task_record:
+        raise ValueError("Either celery_id or task_record must be provided")
+
+    if celery_id and not task_record:
+        task_record = MonitorTask.get_by(db=db, field="celery_id", value=celery_id)
+        if not task_record:
+            raise ValueError(f"Could not find MonitorTask with celery_id {celery_id}")
+
+    assert task_record is not None  # help type checker understand the control flow
+
+    if not celery_id:
+        celery_id = task_record.generate_uuid()
+        task_record.celery_id = celery_id
+
+    task_record.status = status.value  # type: ignore
+    task_record.message = message
+
+    MonitorTaskExecutionLog(  # type: ignore
+        monitor_task=task_record,
+        status=status,
+        message=message,
+        celery_id=celery_id,
+        run_type=run_type,
+    )
+
+    db.commit()
+    db.refresh(task_record)
+    return task_record

fides/api/models/field_types/__init__.py

@@ -0,0 +1,5 @@
+from .encrypted_large_data import EncryptedLargeDataDescriptor
+
+__all__ = [
+    "EncryptedLargeDataDescriptor",
+]