ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.71.1rc1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-01T18:54:14+0200",
+ "date": "2025-10-03T13:56:10-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "ed0d3226e0f8af1ecab0ac2d2dabff1ee209680d",
- "version": "2.71.1b1"
+ "full-revisionid": "af803c59f2d40a0f5be64f230bdee02b61acc1f5",
+ "version": "2.71.1rc1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/4bfbeff34611_add_polling_status.py

@@ -0,0 +1,68 @@
+"""add polling status
+
+Revision ID: 4bfbeff34611
+Revises: 7db29f9cd77b
+Create Date: 2025-09-20 23:02:45.550170
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4bfbeff34611"
+down_revision = "7db29f9cd77b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TYPE executionlogstatus ADD VALUE 'polling'")
+
+
+def downgrade():
+    # Remove any records that have the 'polling' enum value before dropping it
+    # Fallback to 'paused' (awaiting_processing) for backward compatibility
+
+    op.execute("UPDATE requesttask SET status = 'paused' WHERE status = 'polling'")
+    op.execute("UPDATE executionlog SET status = 'paused' WHERE status = 'polling'")
+    op.execute(
+        "UPDATE digest_task_execution SET status = 'paused' WHERE status = 'polling'"
+    )
+    op.execute("UPDATE monitortask SET status = 'paused' WHERE status = 'polling'")
+    op.execute(
+        "UPDATE monitortaskexecutionlog SET status = 'paused' WHERE status = 'polling'"
+    )
+
+    # Recreate the enum without the 'polling' value
+    op.execute("ALTER TYPE executionlogstatus RENAME TO executionlogstatus_old")
+    op.execute(
+        "CREATE TYPE executionlogstatus AS ENUM ("
+        "'in_processing', 'pending', 'complete', 'error', 'paused', 'retrying', 'skipped'"
+        ")"
+    )
+
+    # Update all tables that use the enum type
+    op.execute(
+        "ALTER TABLE requesttask ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE executionlog ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE digest_task_execution ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE monitortask ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE monitortaskexecutionlog ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+
+    # Drop the old enum type
+    op.execute("DROP TYPE executionlogstatus_old")

fides/api/alembic/migrations/versions/7db29f9cd77b_create_new_sub_request_table.py

@@ -0,0 +1,95 @@
+"""Create new Sub Request Table
+
+Revision ID: 7db29f9cd77b
+Revises: b97e92b038d2
+Create Date: 2025-09-16 14:00:16.282996
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy_utils.types.encrypted.encrypted_type import (
+    AesGcmEngine,
+    StringEncryptedType,
+)
+
+from fides.api.db.base_class import JSONTypeOverride
+from fides.config import CONFIG
+
+# revision identifiers, used by Alembic.
+revision = "7db29f9cd77b"
+down_revision = "b97e92b038d2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "request_task_sub_request",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("request_task_id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "param_values",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=False,
+        ),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column(
+            "access_data",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=True,
+        ),
+        sa.Column("rows_masked", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["request_task_id"],
+            ["requesttask.id"],
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_id"),
+        "request_task_sub_request",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        "request_task_sub_request",
+        ["request_task_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        table_name="request_task_sub_request",
+    )
+    op.drop_index(
+        op.f("ix_request_task_sub_request_id"), table_name="request_task_sub_request"
+    )
+    op.drop_table("request_task_sub_request")

fides/api/alembic/migrations/versions/b97e92b038d2_add_digest_execution_model.py

@@ -0,0 +1,117 @@
+"""add digest execution model
+
+Revision ID: b97e92b038d2
+Revises: 3efe14d4469a
+Create Date: 2025-10-01 16:42:41.900651
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b97e92b038d2"
+down_revision = "3efe14d4469a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "digest_task_execution",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("action_type", sa.String(), nullable=False),
+        sa.Column("digest_config_id", sa.String(), nullable=False),
+        sa.Column("celery_task_id", sa.String(), nullable=True),
+        sa.Column(
+            "status",
+            postgresql.ENUM(name="executionlogstatus", create_type=False),
+            nullable=False,
+        ),
+        sa.Column("total_recipients", sa.Integer(), nullable=True),
+        sa.Column("processed_recipients", sa.Integer(), nullable=False),
+        sa.Column("successful_communications", sa.Integer(), nullable=False),
+        sa.Column("failed_communications", sa.Integer(), nullable=False),
+        sa.Column(
+            "execution_state", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column(
+            "processed_user_ids", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_checkpoint_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["digest_config_id"], ["digest_config.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_action_type"),
+        "digest_task_execution",
+        ["action_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        "digest_task_execution",
+        ["celery_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        "digest_task_execution",
+        ["digest_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_id"),
+        "digest_task_execution",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_status"),
+        "digest_task_execution",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_digest_task_execution_status"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_id"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_action_type"), table_name="digest_task_execution"
+    )
+    op.drop_table("digest_task_execution")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/generic_overrides.py

@@ -54,7 +54,6 @@ from fides.common.api.v1.urn_registry import DATASETS_CLEAN, V1_URL_PREFIX
 from fides.service.dataset.dataset_service import (
     DatasetNotFoundException,
     DatasetService,
-    LinkedDatasetException,
 )
 from fides.service.taxonomy.taxonomy_service import TaxonomyService
 
@@ -127,7 +126,7 @@ async def update_dataset(
     except DatasetNotFoundException as e:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
-            detail=str(e),
+            detail={"message": str(e)},
         )
 
 
@@ -249,7 +248,7 @@ async def get_dataset(
     except DatasetNotFoundException as e:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
-            detail=str(e),
+            detail={"message": str(e)},
         )
 
 
@@ -276,12 +275,7 @@ async def delete_dataset(
     except DatasetNotFoundException as e:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
-            detail=str(e),
-        )
-    except LinkedDatasetException as e:
-        raise HTTPException(
-            status_code=HTTP_400_BAD_REQUEST,
-            detail=str(e),
+            detail={"message": str(e)},
         )
 
 

fides/api/common_exceptions.py

@@ -179,6 +179,10 @@ class AwaitingAsyncTask(BaseException):
     """Request Task is Awaiting Processing - Awaiting Async Task"""
 
 
+class AwaitingAsyncProcessing(BaseException):
+    """Request Task is actively being processed by external system - Fides is polling"""
+
+
 class UpstreamTasksNotReady(BaseException):
     """Privacy Request Task awaiting upstream tasks"""
 

fides/api/main.py

@@ -45,9 +45,9 @@ from fides.api.service.privacy_request.email_batch_service import (
     initiate_scheduled_batch_email_send,
 )
 from fides.api.service.privacy_request.request_service import (
-    initiate_async_tasks_status_polling,
     initiate_interrupted_task_requeue_poll,
     initiate_poll_for_exited_privacy_request_tasks,
+    initiate_polling_task_requeue,
     initiate_scheduled_dsr_data_removal,
 )
 
@@ -103,7 +103,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_poll_for_exited_privacy_request_tasks()
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
-    initiate_async_tasks_status_polling()
+    initiate_polling_task_requeue()
     initiate_bcrypt_migration_task()
     initiate_post_upgrade_index_creation()
 

fides/api/models/attachment.py

@@ -52,6 +52,7 @@ class AttachmentReferenceType(str, EnumType):
     privacy_request = "privacy_request"
     comment = "comment"
     manual_task_submission = "manual_task_submission"
+    request_task = "request_task"
 
 
 class AttachmentReference(Base):

fides/api/models/digest/__init__.py

@@ -5,10 +5,12 @@ from fides.api.models.digest.conditional_dependencies import (
     DigestConditionType,
 )
 from fides.api.models.digest.digest_config import DigestConfig, DigestType
+from fides.api.models.digest.digest_execution import DigestTaskExecution
 
 __all__ = [
     "DigestConfig",
     "DigestType",
     "DigestCondition",
     "DigestConditionType",
+    "DigestTaskExecution",
 ]

fides/api/models/digest/digest_config.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 
 from sqlalchemy import Boolean, Column, DateTime, String, Text
 from sqlalchemy.dialects.postgresql import JSONB
@@ -19,6 +19,9 @@ from fides.api.task.conditional_dependencies.schemas import (
     ConditionLeaf,
 )
 
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_execution import DigestTaskExecution
+
 
 class DigestType(str, Enum):
     """Types of digests that can be configured."""
@@ -60,6 +63,12 @@ class DigestConfig(Base):
         back_populates="digest_config",
         cascade="all, delete-orphan",
     )
+    executions = relationship(
+        "DigestTaskExecution",
+        back_populates="digest_config",
+        cascade="all, delete-orphan",
+        order_by="DigestTaskExecution.created_at.desc()",
+    )
 
     def get_receiver_condition(
         self, db: Session

fides/api/models/digest/digest_execution.py

@@ -0,0 +1,132 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+from sqlalchemy.sql import func
+
+from fides.api.db.base_class import Base
+from fides.api.models.worker_task import ExecutionLogStatus, WorkerTask
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_config import DigestConfig
+
+
+class DigestTaskExecution(
+    WorkerTask, Base
+):  # pylint: disable=too-many-instance-attributes
+    """
+    Model for tracking digest task execution state and progress.
+
+    This model enables graceful resumption of digest tasks after worker
+    interruptions by persisting execution state and progress information.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_task_execution"
+
+    # Foreign key to digest config
+    digest_config_id = Column(
+        String,
+        ForeignKey("digest_config.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Celery task tracking
+    celery_task_id = Column(String, nullable=True, index=True)
+
+    # Progress tracking
+    total_recipients = Column(Integer, nullable=True)
+    processed_recipients = Column(Integer, nullable=False, default=0)
+    successful_communications = Column(Integer, nullable=False, default=0)
+    failed_communications = Column(Integer, nullable=False, default=0)
+
+    # State persistence for resumption
+    execution_state = Column(JSONB, nullable=True, default={})
+    processed_user_ids = Column(JSONB, nullable=True, default=[])
+
+    # Timing information
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    completed_at = Column(DateTime(timezone=True), nullable=True)
+    last_checkpoint_at = Column(DateTime(timezone=True), nullable=True)
+
+    # Error information
+    error_message = Column(Text, nullable=True)
+
+    # Relationships
+    digest_config = relationship("DigestConfig", back_populates="executions")
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        """Return allowed action types for digest task execution."""
+        return ["digest_processing"]
+
+    def mark_started(self, db: Session, celery_task_id: str) -> None:
+        """Mark the execution as started."""
+        self.status = ExecutionLogStatus.in_processing
+        self.celery_task_id = celery_task_id
+        self.started_at = func.now()
+        self.save(db)
+
+    def mark_awaiting_processing(self, db: Session) -> None:
+        """Mark the execution as awaiting processing."""
+        self.status = ExecutionLogStatus.awaiting_processing
+        self.save(db)
+
+    def update_progress(
+        self,
+        db: Session,
+        processed_count: int,
+        successful_count: int,
+        failed_count: int,
+        processed_user_ids: List[str],
+        execution_state: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Update execution progress and create checkpoint."""
+        self.processed_recipients = processed_count
+        self.successful_communications = successful_count
+        self.failed_communications = failed_count
+        self.processed_user_ids = processed_user_ids
+        self.last_checkpoint_at = func.now()
+
+        if execution_state:
+            self.execution_state = execution_state
+
+        self.save(db)
+
+    def mark_completed(self, db: Session) -> None:
+        """Mark the execution as completed."""
+        self.status = ExecutionLogStatus.complete
+        self.completed_at = func.now()
+        self.save(db)
+
+    def mark_failed(self, db: Session, error_message: str) -> None:
+        """Mark the execution as failed."""
+        self.status = ExecutionLogStatus.error
+        self.error_message = error_message
+        self.completed_at = func.now()
+        self.save(db)
+
+    def can_resume(self) -> bool:
+        """Check if this execution can be resumed."""
+        return (
+            self.status
+            in [
+                ExecutionLogStatus.in_processing,
+                ExecutionLogStatus.awaiting_processing,
+            ]
+            and self.processed_user_ids is not None
+        )
+
+    def get_remaining_work(self) -> Dict[str, Any]:
+        """Get information about remaining work for resumption."""
+        return {
+            "processed_user_ids": self.processed_user_ids or [],
+            "execution_state": self.execution_state or {},
+            "processed_count": self.processed_recipients or 0,
+            "successful_count": self.successful_communications,
+            "failed_count": self.failed_communications,
+        }

fides/api/models/event_audit.py

@@ -31,6 +31,14 @@ class EventAuditType(str, EnumType):
     taxonomy_element_updated = "taxonomy.element.updated"
     taxonomy_element_deleted = "taxonomy.element.deleted"
 
+    # Digest
+    digest_execution_started = "digest.execution.started"
+    digest_execution_completed = "digest.execution.completed"
+    digest_execution_interrupted = "digest.execution.interrupted"
+    digest_execution_resumed = "digest.execution.resumed"
+    digest_communications_sent = "digest.communications.sent"
+    digest_checkpoint_created = "digest.checkpoint.created"
+
 
 class EventAuditStatus(str, EnumType):
     """Status enum for event audit logging."""

fides/api/models/privacy_notice.py

@@ -530,10 +530,6 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         # to prevent the dry update from being added to Session.new
         updated_attributes.pop("translations", [])
         updated_attributes.pop("children", [])
-        # The source PrivacyNotice may have cached/computed attributes (e.g., @cached_property)
-        # stored on the instance dict. These should not be included in historical payloads.
-        # For example, 'cookies' is cached on the PrivacyNotice instance when accessed.
-        updated_attributes.pop("cookies", None)
 
         # create a new object with the updated attribute data to keep this
         # ORM object (i.e., `self`) pristine
@@ -680,10 +676,6 @@ def create_historical_record_for_notice_and_translation(
     history_data: dict = create_historical_data_from_record(privacy_notice)
     history_data.pop("translations", None)
     history_data.pop("parent_id", None)
-    # The source PrivacyNotice may have cached/computed attributes (e.g., @cached_property)
-    # stored on the instance dict. These should not be included in historical payloads.
-    # For example, 'cookies' is cached on the PrivacyNotice instance when accessed.
-    history_data.pop("cookies", None)
 
     updated_translation_data: dict = create_historical_data_from_record(
         notice_translation