ethyca-fides 2.71.0rc3__py2.py3-none-any.whl → 2.71.1__py2.py3-none-any.whl

fides/api/models/digest/conditional_dependencies.py

@@ -1,9 +1,275 @@
 from enum import Enum
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from sqlalchemy import Column, ForeignKey, Index, String, text
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import FidesBase
+from fides.api.db.util import EnumColumn
+from fides.api.models.conditional_dependency.conditional_dependency_base import (
+    ConditionalDependencyBase,
+    ConditionalDependencyError,
+)
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+)
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_config import DigestConfig
 
 
 class DigestConditionType(str, Enum):
-    """Types of digest conditions - each can have their own tree."""
+    """Types of digest conditions - each can have their own tree.
+
+    Types:
+        - RECEIVER: Conditions that determine who gets the digest
+        - CONTENT: Conditions that determine what gets included in the digest
+        - PRIORITY: Conditions that determine what is considered high priority for the digest
+          - This could be used to determine sending the digest at a different time or how
+            often it should be sent. It could also be used to format content.
+          - Example:
+            - DSRs that are due within the next week
+            - Privacy requests that are due within the next week
+            - Privacy requests for certain geographic regions
+    """
 
     RECEIVER = "receiver"
     CONTENT = "content"
     PRIORITY = "priority"
+
+
+class DigestCondition(ConditionalDependencyBase):
+    """Digest conditional dependencies - multi-type hierarchies.
+
+    - Multi-type hierarchy means one digest_config can have multiple independent
+      condition trees, each with a different digest_condition_type (RECEIVER, CONTENT, PRIORITY)
+    - Within each tree, all nodes must have the same digest_condition_type
+    - This enables separate condition logic for different aspects of digest processing
+
+    Ensures that all conditions within the same tree have the same digest_condition_type.
+    This prevents logical errors where different condition types are mixed in a single
+    condition tree structure.
+
+    Example Tree Structure:
+        DigestConfig (e.g., "Weekly Privacy Digest")
+        ├── RECEIVER Dependency Condition Tree (who gets the digest)
+        │   └── Group (AND)
+        │       ├── Leaf: user.role == "admin"
+        │       └── Leaf: user.department == "privacy"
+        ├── CONTENT Dependency Condition Tree (what gets included)
+        │   └── Group (OR)
+        │       ├── Leaf: task.priority == "high"
+        │       └── Leaf: task.overdue == true
+        └── PRIORITY Dependency Condition Tree (when to send)
+            └── Leaf: task.count >= 5
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_condition"
+
+    # We need to redefine it here so that self-referential relationships
+    # can properly reference the `id` column instead of the built-in Python function.
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+
+    # Foreign key relationships
+    digest_config_id = Column(
+        String(255),
+        ForeignKey("digest_config.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    parent_id = Column(
+        String(255),
+        ForeignKey("digest_condition.id", ondelete="CASCADE"),
+        nullable=True,
+        index=True,
+    )
+
+    # Digest-specific: condition category
+    digest_condition_type = Column(
+        EnumColumn(DigestConditionType), nullable=False, index=True
+    )
+
+    # Relationships
+    digest_config = relationship("DigestConfig", back_populates="conditions")
+    parent = relationship(
+        "DigestCondition",
+        remote_side=[id],
+        back_populates="children",
+        foreign_keys=[parent_id],
+    )
+    children = relationship(
+        "DigestCondition",
+        back_populates="parent",
+        cascade="all, delete-orphan",
+        foreign_keys=[parent_id],
+    )
+
+    # Ensure only one root condition per digest_condition_type per digest_config
+    __table_args__ = (
+        Index(
+            "ix_digest_condition_unique_root_per_type",
+            "digest_config_id",
+            "digest_condition_type",
+            unique=True,
+            postgresql_where=text("parent_id IS NULL"),
+        ),
+    )
+
+    @staticmethod
+    def _validate_condition_type_consistency(db: Session, data: dict[str, Any]) -> None:
+        """Validate that a condition's digest_condition_type matches its parent's type.
+
+        Since each parent was validated when created, checking against the immediate parent
+        is sufficient to ensure tree-wide consistency.
+
+        Args:
+            db: Database session for querying
+            data: Dictionary containing condition data to validate (must include both parent_id and digest_condition_type)
+
+        Raises:
+            ValueError: If parent doesn't exist or digest_condition_type doesn't match parent's type
+        """
+        parent_id = data.get("parent_id")
+        digest_condition_type = data.get("digest_condition_type")
+
+        if not parent_id:
+            # Root condition - no validation needed
+            return
+
+        # Get the parent condition
+        parent = (
+            db.query(DigestCondition).filter(DigestCondition.id == parent_id).first()
+        )
+        if not parent:
+            raise ValueError(f"Parent condition with id '{parent_id}' does not exist")
+
+        # Validate that the new condition matches the parent's digest_condition_type
+        if parent.digest_condition_type != digest_condition_type:
+            raise ValueError(
+                f"Cannot create condition with type '{digest_condition_type}' under parent "
+                f"with type '{parent.digest_condition_type}'. All conditions in the same tree "
+                f"must have the same digest_condition_type."
+            )
+
+    @classmethod
+    def create(
+        cls,
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DigestCondition":
+        """Create a new DigestCondition with validation."""
+        # Validate condition type consistency
+        cls._validate_condition_type_consistency(db, data)
+        try:
+            return super().create(db=db, data=data, check_name=check_name)
+        except Exception as e:
+            raise ConditionalDependencyError(str(e))
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DigestCondition":
+        """Update DigestCondition with validation."""
+        # Ensure validation data includes current values for fields not being updated
+        validation_data = {
+            "parent_id": data.get("parent_id", self.parent_id),
+            "digest_condition_type": data.get(
+                "digest_condition_type", self.digest_condition_type
+            ),
+        }
+
+        # Validate before updating
+        self._validate_condition_type_consistency(db, validation_data)
+        return super().update(db=db, data=data)  # type: ignore[return-value]
+
+    def save(self, db: Session) -> "DigestCondition":
+        """Save DigestCondition with validation."""
+        # Extract current object data for validation
+        data = {
+            "parent_id": self.parent_id,
+            "digest_condition_type": self.digest_condition_type,
+        }
+
+        # Validate before saving (only if this has a parent)
+        if self.parent_id:
+            self._validate_condition_type_consistency(db, data)
+        return super().save(db=db)  # type: ignore[return-value]
+
+    @classmethod
+    def get_root_condition(
+        cls,
+        db: Session,
+        **kwargs: Any,
+    ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
+        """Get the root condition tree for a specific digest condition type.
+
+        Implementation of the abstract base method for DigestCondition's multi-type hierarchy.
+        Each digest_config can have separate condition trees for RECEIVER, CONTENT, and PRIORITY
+        types. This method retrieves the root of one specific tree.
+
+        Args:
+            db: SQLAlchemy database session for querying
+            **kwargs: Keyword arguments containing:
+                digest_config_id: ID of the digest config
+                digest_condition_type: DigestConditionType enum value
+                                     Must be one of: RECEIVER, CONTENT, PRIORITY
+
+        Returns:
+            Optional[Union[ConditionLeaf, ConditionGroup]]: Root condition tree for the specified
+                                                          type, or None if no conditions exist
+
+        Raises:
+            ValueError: If required parameters are missing
+
+        Example:
+            >>> # Get receiver conditions for a digest
+            >>> receiver_conditions = DigestCondition.get_root_condition(
+            ...     db, digest_config_id=digest_config.id,
+            ...     digest_condition_type=DigestConditionType.RECEIVER
+            ... )
+            >>> # Get content conditions for the same digest
+            >>> content_conditions = DigestCondition.get_root_condition(
+            ...     db, digest_config_id=digest_config.id,
+            ...     digest_condition_type=DigestConditionType.CONTENT
+            ... )
+        """
+        digest_config_id = kwargs.get("digest_config_id")
+        digest_condition_type = kwargs.get("digest_condition_type")
+
+        if not digest_config_id or not digest_condition_type:
+            raise ValueError(
+                "digest_config_id and digest_condition_type are required keyword arguments"
+            )
+
+        root = (
+            db.query(cls)
+            .filter(
+                cls.digest_config_id == digest_config_id,
+                cls.digest_condition_type == digest_condition_type,
+                cls.parent_id.is_(None),
+            )
+            .one_or_none()
+        )
+
+        if not root:
+            return None
+
+        return root.to_correct_condition_type()
+
+    @classmethod
+    def get_all_root_conditions(
+        cls, db: Session, digest_config_id: str
+    ) -> dict[DigestConditionType, Optional[Condition]]:
+        """Get root conditions for all digest condition types"""
+        return {
+            condition_type: cls.get_root_condition(
+                db,
+                digest_config_id=digest_config_id,
+                digest_condition_type=condition_type,
+            )
+            for condition_type in DigestConditionType
+        }

fides/api/models/digest/digest_config.py

@@ -1,14 +1,17 @@
 from enum import Enum
-from typing import Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 
 from sqlalchemy import Boolean, Column, DateTime, String, Text
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.declarative import declared_attr
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
 from fides.api.db.util import EnumColumn
-from fides.api.models.digest.conditional_dependencies import DigestConditionType
+from fides.api.models.digest.conditional_dependencies import (
+    DigestCondition,
+    DigestConditionType,
+)
 from fides.api.schemas.messaging.messaging import MessagingMethod
 from fides.api.task.conditional_dependencies.schemas import (
     Condition,
@@ -16,6 +19,9 @@ from fides.api.task.conditional_dependencies.schemas import (
     ConditionLeaf,
 )
 
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_execution import DigestTaskExecution
+
 
 class DigestType(str, Enum):
     """Types of digests that can be configured."""
@@ -51,26 +57,54 @@ class DigestConfig(Base):
     next_scheduled_at = Column(DateTime(timezone=True), nullable=True, index=True)
     config_metadata = Column(JSONB, nullable=True, default={})
 
-    def get_receiver_conditions(
+    # Relationships
+    conditions = relationship(
+        "DigestCondition",
+        back_populates="digest_config",
+        cascade="all, delete-orphan",
+    )
+    executions = relationship(
+        "DigestTaskExecution",
+        back_populates="digest_config",
+        cascade="all, delete-orphan",
+        order_by="DigestTaskExecution.created_at.desc()",
+    )
+
+    def get_receiver_condition(
         self, db: Session
     ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
         """Get receiver conditions for this digest config."""
-        raise NotImplementedError("Conditions are not implemented for digests")
+        return DigestCondition.get_root_condition(
+            db,
+            digest_config_id=self.id,
+            digest_condition_type=DigestConditionType.RECEIVER,
+        )
 
-    def get_content_conditions(
+    def get_content_condition(
         self, db: Session
     ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
         """Get content conditions for this digest config."""
-        raise NotImplementedError("Conditions are not implemented for digests")
 
-    def get_priority_conditions(
+        return DigestCondition.get_root_condition(
+            db,
+            digest_config_id=self.id,
+            digest_condition_type=DigestConditionType.CONTENT,
+        )
+
+    def get_priority_condition(
         self, db: Session
     ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
         """Get priority conditions for this digest config."""
-        raise NotImplementedError("Conditions are not implemented for digests")
+
+        return DigestCondition.get_root_condition(
+            db,
+            digest_config_id=self.id,
+            digest_condition_type=DigestConditionType.PRIORITY,
+        )
 
     def get_all_conditions(
         self, db: Session
     ) -> dict["DigestConditionType", Optional[Condition]]:
         """Get all condition types for this digest config."""
-        raise NotImplementedError("Conditions are not implemented for digests")
+
+        return DigestCondition.get_all_root_conditions(db, self.id)

fides/api/models/digest/digest_execution.py

@@ -0,0 +1,132 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+from sqlalchemy.sql import func
+
+from fides.api.db.base_class import Base
+from fides.api.models.worker_task import ExecutionLogStatus, WorkerTask
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_config import DigestConfig
+
+
+class DigestTaskExecution(
+    WorkerTask, Base
+):  # pylint: disable=too-many-instance-attributes
+    """
+    Model for tracking digest task execution state and progress.
+
+    This model enables graceful resumption of digest tasks after worker
+    interruptions by persisting execution state and progress information.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_task_execution"
+
+    # Foreign key to digest config
+    digest_config_id = Column(
+        String,
+        ForeignKey("digest_config.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Celery task tracking
+    celery_task_id = Column(String, nullable=True, index=True)
+
+    # Progress tracking
+    total_recipients = Column(Integer, nullable=True)
+    processed_recipients = Column(Integer, nullable=False, default=0)
+    successful_communications = Column(Integer, nullable=False, default=0)
+    failed_communications = Column(Integer, nullable=False, default=0)
+
+    # State persistence for resumption
+    execution_state = Column(JSONB, nullable=True, default={})
+    processed_user_ids = Column(JSONB, nullable=True, default=[])
+
+    # Timing information
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    completed_at = Column(DateTime(timezone=True), nullable=True)
+    last_checkpoint_at = Column(DateTime(timezone=True), nullable=True)
+
+    # Error information
+    error_message = Column(Text, nullable=True)
+
+    # Relationships
+    digest_config = relationship("DigestConfig", back_populates="executions")
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        """Return allowed action types for digest task execution."""
+        return ["digest_processing"]
+
+    def mark_started(self, db: Session, celery_task_id: str) -> None:
+        """Mark the execution as started."""
+        self.status = ExecutionLogStatus.in_processing
+        self.celery_task_id = celery_task_id
+        self.started_at = func.now()
+        self.save(db)
+
+    def mark_awaiting_processing(self, db: Session) -> None:
+        """Mark the execution as awaiting processing."""
+        self.status = ExecutionLogStatus.awaiting_processing
+        self.save(db)
+
+    def update_progress(
+        self,
+        db: Session,
+        processed_count: int,
+        successful_count: int,
+        failed_count: int,
+        processed_user_ids: List[str],
+        execution_state: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Update execution progress and create checkpoint."""
+        self.processed_recipients = processed_count
+        self.successful_communications = successful_count
+        self.failed_communications = failed_count
+        self.processed_user_ids = processed_user_ids
+        self.last_checkpoint_at = func.now()
+
+        if execution_state:
+            self.execution_state = execution_state
+
+        self.save(db)
+
+    def mark_completed(self, db: Session) -> None:
+        """Mark the execution as completed."""
+        self.status = ExecutionLogStatus.complete
+        self.completed_at = func.now()
+        self.save(db)
+
+    def mark_failed(self, db: Session, error_message: str) -> None:
+        """Mark the execution as failed."""
+        self.status = ExecutionLogStatus.error
+        self.error_message = error_message
+        self.completed_at = func.now()
+        self.save(db)
+
+    def can_resume(self) -> bool:
+        """Check if this execution can be resumed."""
+        return (
+            self.status
+            in [
+                ExecutionLogStatus.in_processing,
+                ExecutionLogStatus.awaiting_processing,
+            ]
+            and self.processed_user_ids is not None
+        )
+
+    def get_remaining_work(self) -> Dict[str, Any]:
+        """Get information about remaining work for resumption."""
+        return {
+            "processed_user_ids": self.processed_user_ids or [],
+            "execution_state": self.execution_state or {},
+            "processed_count": self.processed_recipients or 0,
+            "successful_count": self.successful_communications,
+            "failed_count": self.failed_communications,
+        }

fides/api/models/event_audit.py

@@ -31,6 +31,14 @@ class EventAuditType(str, EnumType):
     taxonomy_element_updated = "taxonomy.element.updated"
     taxonomy_element_deleted = "taxonomy.element.deleted"
 
+    # Digest
+    digest_execution_started = "digest.execution.started"
+    digest_execution_completed = "digest.execution.completed"
+    digest_execution_interrupted = "digest.execution.interrupted"
+    digest_execution_resumed = "digest.execution.resumed"
+    digest_communications_sent = "digest.communications.sent"
+    digest_checkpoint_created = "digest.checkpoint.created"
+
 
 class EventAuditStatus(str, EnumType):
     """Status enum for event audit logging."""

fides/api/models/fides_user.py

@@ -31,6 +31,7 @@ if TYPE_CHECKING:
     from fides.api.models.fides_user_respondent_email_verification import (
         FidesUserRespondentEmailVerification,
     )
+    from fides.api.models.manual_task.manual_task import ManualTaskReference
     from fides.api.models.sql_models import System  # type: ignore[attr-defined]
     from fides.api.models.system_manager import SystemManager
 
@@ -90,6 +91,14 @@ class FidesUser(Base):
         uselist=False,
     )
 
+    # Manual task assignments relationship
+    manual_task_references = relationship(
+        "ManualTaskReference",
+        primaryjoin="and_(FidesUser.id == foreign(ManualTaskReference.reference_id), "
+        "ManualTaskReference.reference_type == 'assigned_user')",
+        viewonly=True,
+    )
+
     @property
     def system_ids(self) -> List[str]:
         return [system.id for system in self.systems]

fides/api/models/manual_task/conditional_dependency.py

@@ -1,13 +1,12 @@
 from typing import TYPE_CHECKING, Any, Optional, Union
 
-from sqlalchemy import Column, ForeignKey, Index, String
+from sqlalchemy import Column, ForeignKey, String
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import FidesBase
 from fides.api.models.conditional_dependency.conditional_dependency_base import (
     ConditionalDependencyBase,
-    ConditionalDependencyType,
 )
 from fides.api.task.conditional_dependencies.schemas import (
     ConditionGroup,
@@ -30,19 +29,16 @@ class ManualTaskConditionalDependency(ConditionalDependencyBase):
     id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
     # Foreign key relationships
     manual_task_id = Column(
-        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+        String,
+        ForeignKey("manual_task.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
     )
     parent_id = Column(
         String,
         ForeignKey("manual_task_conditional_dependency.id", ondelete="CASCADE"),
         nullable=True,
-    )
-
-    __table_args__ = (
-        Index("ix_manual_task_conditional_dependency_manual_task_id", "manual_task_id"),
-        Index("ix_manual_task_conditional_dependency_parent_id", "parent_id"),
-        Index("ix_manual_task_conditional_dependency_condition_type", "condition_type"),
-        Index("ix_manual_task_conditional_dependency_sort_order", "sort_order"),
+        index=True,
     )
 
     # Relationships
@@ -62,18 +58,22 @@ class ManualTaskConditionalDependency(ConditionalDependencyBase):
 
     @classmethod
     def get_root_condition(
-        cls, db: Session, *args: Any, **kwargs: Any
+        cls, db: Session, **kwargs: Any
     ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
         """Get the root condition for a manual task
 
         Args:
             db: Database session
-            manual_task_id: ID of the manual task (first positional arg)
+            **kwargs: Keyword arguments containing:
+                manual_task_id: ID of the manual task
+
+        Raises:
+            ValueError: If manual_task_id is not provided
         """
-        if not args:
-            raise ValueError("manual_task_id is required as first positional argument")
+        manual_task_id = kwargs.get("manual_task_id")
 
-        manual_task_id = args[0]
+        if not manual_task_id:
+            raise ValueError("manual_task_id is required as a keyword argument")
         root = (
             db.query(cls)
             .filter(cls.manual_task_id == manual_task_id, cls.parent_id.is_(None))
@@ -83,6 +83,4 @@ class ManualTaskConditionalDependency(ConditionalDependencyBase):
         if not root:
             return None
 
-        if root.condition_type == ConditionalDependencyType.leaf:
-            return root.to_condition_leaf()
-        return root.to_condition_group()
+        return root.to_correct_condition_type()

fides/api/models/privacy_experience.py

@@ -108,6 +108,10 @@ class PrivacyExperienceConfigBase:
 
     show_layer1_notices = Column(Boolean, nullable=True, default=False)
 
+    # Vendor/Asset disclosure configuration
+    allow_vendor_asset_disclosure = Column(Boolean, nullable=True, default=False)
+    asset_disclosure_include_types = Column(ARRAY(String), nullable=True)
+
     @declared_attr
     def layer1_button_options(cls) -> Column:
         return Column(
@@ -142,6 +146,9 @@ class ExperienceConfigTemplate(PrivacyExperienceConfigBase, Base):
     dismissable = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    allow_vendor_asset_disclosure = Column(
+        Boolean, nullable=False, default=False, server_default="f"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     name = Column(
         String, nullable=False
     )  # Overriding PrivacyExperienceConfigBase to make non-nullable
@@ -216,6 +223,9 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
     dismissable = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    allow_vendor_asset_disclosure = Column(
+        Boolean, nullable=False, default=False, server_default="f"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     name = Column(
         String, nullable=False
     )  # Overriding PrivacyExperienceConfigBase to make non-nullable