ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.71.2b0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-01T18:54:14+0200",
+ "date": "2025-10-03T09:34:20-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "ed0d3226e0f8af1ecab0ac2d2dabff1ee209680d",
- "version": "2.71.1b1"
+ "full-revisionid": "521a48a9c7dbc813d08cf2053f318328912e86b6",
+ "version": "2.71.2b0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/4bfbeff34611_add_polling_status.py

@@ -0,0 +1,35 @@
+"""add polling status
+
+Revision ID: 4bfbeff34611
+Revises: 7db29f9cd77b
+Create Date: 2025-09-20 23:02:45.550170
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4bfbeff34611"
+down_revision = "7db29f9cd77b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Check if value already exists
+    connection = op.get_bind()
+    result = connection.execute(
+        sa.text(
+            "SELECT 1 FROM pg_enum WHERE enumlabel = 'polling' "
+            "AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'executionlogstatus')"
+        )
+    )
+
+    if not result.fetchone():
+        op.execute("ALTER TYPE executionlogstatus ADD VALUE 'polling'")
+
+
+def downgrade():
+    # The 'polling' value will remain in the enum but won't be used by older app versions
+    pass

fides/api/alembic/migrations/versions/7db29f9cd77b_create_new_sub_request_table.py

@@ -0,0 +1,95 @@
+"""Create new Sub Request Table
+
+Revision ID: 7db29f9cd77b
+Revises: b97e92b038d2
+Create Date: 2025-09-16 14:00:16.282996
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy_utils.types.encrypted.encrypted_type import (
+    AesGcmEngine,
+    StringEncryptedType,
+)
+
+from fides.api.db.base_class import JSONTypeOverride
+from fides.config import CONFIG
+
+# revision identifiers, used by Alembic.
+revision = "7db29f9cd77b"
+down_revision = "b97e92b038d2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "request_task_sub_request",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("request_task_id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "param_values",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=False,
+        ),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column(
+            "access_data",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=True,
+        ),
+        sa.Column("rows_masked", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["request_task_id"],
+            ["requesttask.id"],
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_id"),
+        "request_task_sub_request",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        "request_task_sub_request",
+        ["request_task_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        table_name="request_task_sub_request",
+    )
+    op.drop_index(
+        op.f("ix_request_task_sub_request_id"), table_name="request_task_sub_request"
+    )
+    op.drop_table("request_task_sub_request")

fides/api/alembic/migrations/versions/b97e92b038d2_add_digest_execution_model.py

@@ -0,0 +1,117 @@
+"""add digest execution model
+
+Revision ID: b97e92b038d2
+Revises: 3efe14d4469a
+Create Date: 2025-10-01 16:42:41.900651
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b97e92b038d2"
+down_revision = "3efe14d4469a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "digest_task_execution",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("action_type", sa.String(), nullable=False),
+        sa.Column("digest_config_id", sa.String(), nullable=False),
+        sa.Column("celery_task_id", sa.String(), nullable=True),
+        sa.Column(
+            "status",
+            postgresql.ENUM(name="executionlogstatus", create_type=False),
+            nullable=False,
+        ),
+        sa.Column("total_recipients", sa.Integer(), nullable=True),
+        sa.Column("processed_recipients", sa.Integer(), nullable=False),
+        sa.Column("successful_communications", sa.Integer(), nullable=False),
+        sa.Column("failed_communications", sa.Integer(), nullable=False),
+        sa.Column(
+            "execution_state", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column(
+            "processed_user_ids", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_checkpoint_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["digest_config_id"], ["digest_config.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_action_type"),
+        "digest_task_execution",
+        ["action_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        "digest_task_execution",
+        ["celery_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        "digest_task_execution",
+        ["digest_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_id"),
+        "digest_task_execution",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_status"),
+        "digest_task_execution",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_digest_task_execution_status"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_id"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_action_type"), table_name="digest_task_execution"
+    )
+    op.drop_table("digest_task_execution")
+    # ### end Alembic commands ###

fides/api/common_exceptions.py

@@ -179,6 +179,10 @@ class AwaitingAsyncTask(BaseException):
     """Request Task is Awaiting Processing - Awaiting Async Task"""
 
 
+class AwaitingAsyncProcessing(BaseException):
+    """Request Task is actively being processed by external system - Fides is polling"""
+
+
 class UpstreamTasksNotReady(BaseException):
     """Privacy Request Task awaiting upstream tasks"""
 

fides/api/main.py

@@ -45,9 +45,9 @@ from fides.api.service.privacy_request.email_batch_service import (
     initiate_scheduled_batch_email_send,
 )
 from fides.api.service.privacy_request.request_service import (
-    initiate_async_tasks_status_polling,
     initiate_interrupted_task_requeue_poll,
     initiate_poll_for_exited_privacy_request_tasks,
+    initiate_polling_task_requeue,
     initiate_scheduled_dsr_data_removal,
 )
 
@@ -103,7 +103,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_poll_for_exited_privacy_request_tasks()
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
-    initiate_async_tasks_status_polling()
+    initiate_polling_task_requeue()
     initiate_bcrypt_migration_task()
     initiate_post_upgrade_index_creation()
 

fides/api/models/attachment.py

@@ -52,6 +52,7 @@ class AttachmentReferenceType(str, EnumType):
     privacy_request = "privacy_request"
     comment = "comment"
     manual_task_submission = "manual_task_submission"
+    request_task = "request_task"
 
 
 class AttachmentReference(Base):

fides/api/models/digest/__init__.py

@@ -5,10 +5,12 @@ from fides.api.models.digest.conditional_dependencies import (
     DigestConditionType,
 )
 from fides.api.models.digest.digest_config import DigestConfig, DigestType
+from fides.api.models.digest.digest_execution import DigestTaskExecution
 
 __all__ = [
     "DigestConfig",
     "DigestType",
     "DigestCondition",
     "DigestConditionType",
+    "DigestTaskExecution",
 ]

fides/api/models/digest/digest_config.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 
 from sqlalchemy import Boolean, Column, DateTime, String, Text
 from sqlalchemy.dialects.postgresql import JSONB
@@ -19,6 +19,9 @@ from fides.api.task.conditional_dependencies.schemas import (
     ConditionLeaf,
 )
 
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_execution import DigestTaskExecution
+
 
 class DigestType(str, Enum):
     """Types of digests that can be configured."""
@@ -60,6 +63,12 @@ class DigestConfig(Base):
         back_populates="digest_config",
         cascade="all, delete-orphan",
     )
+    executions = relationship(
+        "DigestTaskExecution",
+        back_populates="digest_config",
+        cascade="all, delete-orphan",
+        order_by="DigestTaskExecution.created_at.desc()",
+    )
 
     def get_receiver_condition(
         self, db: Session

fides/api/models/digest/digest_execution.py

@@ -0,0 +1,132 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+from sqlalchemy.sql import func
+
+from fides.api.db.base_class import Base
+from fides.api.models.worker_task import ExecutionLogStatus, WorkerTask
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_config import DigestConfig
+
+
+class DigestTaskExecution(
+    WorkerTask, Base
+):  # pylint: disable=too-many-instance-attributes
+    """
+    Model for tracking digest task execution state and progress.
+
+    This model enables graceful resumption of digest tasks after worker
+    interruptions by persisting execution state and progress information.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_task_execution"
+
+    # Foreign key to digest config
+    digest_config_id = Column(
+        String,
+        ForeignKey("digest_config.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Celery task tracking
+    celery_task_id = Column(String, nullable=True, index=True)
+
+    # Progress tracking
+    total_recipients = Column(Integer, nullable=True)
+    processed_recipients = Column(Integer, nullable=False, default=0)
+    successful_communications = Column(Integer, nullable=False, default=0)
+    failed_communications = Column(Integer, nullable=False, default=0)
+
+    # State persistence for resumption
+    execution_state = Column(JSONB, nullable=True, default={})
+    processed_user_ids = Column(JSONB, nullable=True, default=[])
+
+    # Timing information
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    completed_at = Column(DateTime(timezone=True), nullable=True)
+    last_checkpoint_at = Column(DateTime(timezone=True), nullable=True)
+
+    # Error information
+    error_message = Column(Text, nullable=True)
+
+    # Relationships
+    digest_config = relationship("DigestConfig", back_populates="executions")
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        """Return allowed action types for digest task execution."""
+        return ["digest_processing"]
+
+    def mark_started(self, db: Session, celery_task_id: str) -> None:
+        """Mark the execution as started."""
+        self.status = ExecutionLogStatus.in_processing
+        self.celery_task_id = celery_task_id
+        self.started_at = func.now()
+        self.save(db)
+
+    def mark_awaiting_processing(self, db: Session) -> None:
+        """Mark the execution as awaiting processing."""
+        self.status = ExecutionLogStatus.awaiting_processing
+        self.save(db)
+
+    def update_progress(
+        self,
+        db: Session,
+        processed_count: int,
+        successful_count: int,
+        failed_count: int,
+        processed_user_ids: List[str],
+        execution_state: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Update execution progress and create checkpoint."""
+        self.processed_recipients = processed_count
+        self.successful_communications = successful_count
+        self.failed_communications = failed_count
+        self.processed_user_ids = processed_user_ids
+        self.last_checkpoint_at = func.now()
+
+        if execution_state:
+            self.execution_state = execution_state
+
+        self.save(db)
+
+    def mark_completed(self, db: Session) -> None:
+        """Mark the execution as completed."""
+        self.status = ExecutionLogStatus.complete
+        self.completed_at = func.now()
+        self.save(db)
+
+    def mark_failed(self, db: Session, error_message: str) -> None:
+        """Mark the execution as failed."""
+        self.status = ExecutionLogStatus.error
+        self.error_message = error_message
+        self.completed_at = func.now()
+        self.save(db)
+
+    def can_resume(self) -> bool:
+        """Check if this execution can be resumed."""
+        return (
+            self.status
+            in [
+                ExecutionLogStatus.in_processing,
+                ExecutionLogStatus.awaiting_processing,
+            ]
+            and self.processed_user_ids is not None
+        )
+
+    def get_remaining_work(self) -> Dict[str, Any]:
+        """Get information about remaining work for resumption."""
+        return {
+            "processed_user_ids": self.processed_user_ids or [],
+            "execution_state": self.execution_state or {},
+            "processed_count": self.processed_recipients or 0,
+            "successful_count": self.successful_communications,
+            "failed_count": self.failed_communications,
+        }

fides/api/models/event_audit.py

@@ -31,6 +31,14 @@ class EventAuditType(str, EnumType):
     taxonomy_element_updated = "taxonomy.element.updated"
     taxonomy_element_deleted = "taxonomy.element.deleted"
 
+    # Digest
+    digest_execution_started = "digest.execution.started"
+    digest_execution_completed = "digest.execution.completed"
+    digest_execution_interrupted = "digest.execution.interrupted"
+    digest_execution_resumed = "digest.execution.resumed"
+    digest_communications_sent = "digest.communications.sent"
+    digest_checkpoint_created = "digest.checkpoint.created"
+
 
 class EventAuditStatus(str, EnumType):
     """Status enum for event audit logging."""

fides/api/models/privacy_request/privacy_request.py

@@ -369,7 +369,7 @@ class PrivacyRequest(
         return {
             "id": self.id,
             "external_id": self.external_id,
-            "status": self.status.value,
+            "status": self.status.value if self.status is not None else None,
             "requested_at": (
                 self.requested_at.isoformat() if self.requested_at else None
             ),

fides/api/models/privacy_request/request_task.py

@@ -8,8 +8,9 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 from loguru import logger
 from sqlalchemy import Boolean, Column, ForeignKey, Integer, String
 from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict, MutableList
-from sqlalchemy.orm import Query, Session, relationship
+from sqlalchemy.orm import Query, RelationshipProperty, Session, relationship
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
     AesGcmEngine,
     StringEncryptedType,
@@ -183,6 +184,14 @@ class RequestTask(WorkerTask, Base):
         uselist=False,
     )
 
+    # Stores the sub-requests data for async polling tasks
+    sub_requests: "RelationshipProperty[List[RequestTaskSubRequest]]" = relationship(
+        "RequestTaskSubRequest",
+        back_populates="request_task",
+        cascade="all, delete-orphan",
+        order_by="RequestTaskSubRequest.created_at",
+    )
+
     @property
     def request_task_address(self) -> CollectionAddress:
         """Convert the collection_address into Collection Address format"""
@@ -318,3 +327,91 @@ class RequestTask(WorkerTask, Base):
             )
 
         return task_in_flight
+
+
+class RequestTaskSubRequest(Base):
+    """
+    Model for storing individual sub-request data during the execution of a request task.
+    Supports 1:N relationship - each RequestTask can have multiple sub-requests.
+    Currently used for storing request data for polling tasks.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "request_task_sub_request"
+
+    request_task_id = Column(
+        String(255),
+        ForeignKey(
+            "requesttask.id",
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        nullable=False,
+        index=True,
+    )
+
+    request_task = relationship(
+        "RequestTask",
+        back_populates="sub_requests",
+    )
+
+    # Individual sub-request data (e.g., request_id, status, result data)
+    # Additional fields for enhanced sub-request tracking
+    param_values = Column(  # An encrypted JSON String - saved as a dict
+        StringEncryptedType(
+            type_in=JSONTypeOverride,
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+        nullable=False,
+    )
+    status = Column(String, nullable=False)
+
+    # Raw data retrieved from an access request is stored here.  This contains all of the
+    # intermediate data we retrieved, needed for downstream tasks, but hasn't been filtered
+    # by data category for the end user.
+    _access_data = Column(  # An encrypted JSON String - saved as a list of Rows
+        "access_data",
+        StringEncryptedType(
+            type_in=JSONTypeOverride,
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+    )
+
+    # Use descriptors for automatic external storage handling
+    access_data = EncryptedLargeDataDescriptor(
+        field_name="access_data", empty_default=[]
+    )
+
+    # Written after an erasure is completed
+    rows_masked = Column(Integer)
+
+    def get_correlation_id(self) -> Optional[str]:
+        """Helper method to extract correlation_id from param_values."""
+        if self.param_values and "request_id" in self.param_values:
+            return self.param_values["request_id"]
+        return None
+
+    def update_status(self, db: Session, status: str) -> None:
+        """Helper method to update the status of this sub-request."""
+        self.status = status
+        self.save(db)
+
+    def cleanup_external_storage(self) -> None:
+        """Clean up all external storage files for this sub-request"""
+        # Access the descriptor from the class to call cleanup
+        RequestTaskSubRequest.access_data.cleanup(self)
+
+    def get_access_data(self) -> List[Row]:
+        """Helper to retrieve access data or default to empty list"""
+        return self.access_data or []
+
+    def delete(self, db: Session) -> None:
+        """Override delete to cleanup external storage first"""
+        self.cleanup_external_storage()
+        super().delete(db)

fides/api/models/worker_task.py

@@ -17,6 +17,14 @@ class ExecutionLogStatus(enum.Enum):
     awaiting_processing = "paused"  # "paused" in the database to avoid a migration, but use "awaiting_processing" in the app
     retrying = "retrying"
     skipped = "skipped"
+    polling = "polling"
+
+
+# Statuses that can be resumed
+RESUMABLE_EXECUTION_LOG_STATUSES = [
+    ExecutionLogStatus.pending,
+    ExecutionLogStatus.polling,
+]
 
 
 class WorkerTask: