ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/models/detection_discovery/monitor_task.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from enum import Enum
 from typing import List, Optional
 
-from sqlalchemy import ARRAY, Column
+from sqlalchemy import ARRAY, Boolean, Column
 from sqlalchemy import Enum as SQLAlchemyEnum
 from sqlalchemy import ForeignKey, String
 from sqlalchemy.dialects.postgresql import JSONB
@@ -25,6 +25,7 @@ class MonitorTaskType(Enum):
 
     DETECTION = "detection"
     CLASSIFICATION = "classification"
+    LLM_CLASSIFICATION = "llm_classification"
     PROMOTION = "promotion"
     REMOVAL_PROMOTION = "removal_promotion"
 
@@ -51,6 +52,7 @@ class MonitorTask(WorkerTask, Base):
     )
     staged_resource_urns = Column(ARRAY(String), nullable=True)
     child_resource_urns = Column(ARRAY(String), nullable=True)
+    dismissed = Column(Boolean, nullable=False, default=False)
 
     monitor_config = relationship(MonitorConfig, cascade="all, delete")
     execution_logs = relationship(

fides/api/models/detection_discovery/staged_resource_error.py

@@ -0,0 +1,25 @@
+"""Module for staged resource error tracking."""
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+
+from fides.api.db.base_class import Base, FidesBase
+
+
+class StagedResourceError(Base):
+    """
+    DB model for tracking errors associated with staged resources.
+    Linked to StagedResource via FK with cascade delete.
+    """
+
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+    staged_resource_urn = Column(
+        String,
+        ForeignKey("stagedresource.urn", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    message = Column(String, nullable=False)
+    timestamp = Column(DateTime(timezone=True), nullable=False)
+    error_type = Column(String, nullable=False)
+    diff_status = Column(String, nullable=False)  # Phase where error occurred
+    task_id = Column(String, nullable=True, index=True)

fides/api/models/detection_discovery/web_monitor.py

@@ -0,0 +1,61 @@
+from typing import Any, Optional
+
+from sqlalchemy import Boolean, Column, String, insert, select
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.declarative import declared_attr
+
+from fides.api.db.base_class import Base
+
+
+class WebMonitorGroupJob(Base):
+    """SQL model for storing data returned from a web monitor group job"""
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "web_monitor_group_job"
+
+    # The id for the web monitor group on the web monitor service
+    group_id = Column(String, nullable=False, unique=True)
+
+    # The raw experiences data obtained for that group from the web monitor service
+    raw_experiences_data = Column(JSONB, nullable=True)
+    # The processed experiences data, used to calculate an StagedResource's consent status
+    processed_experiences_data = Column(JSONB, nullable=True)
+
+    # Whether this data is from a test_website monitor rather than a real one
+    is_test_run = Column(Boolean, nullable=False, default=False)
+
+    @classmethod
+    async def create_async(
+        cls,
+        async_session: AsyncSession,
+        *,
+        group_id: str,
+        is_test_run: Optional[bool] = False,
+        raw_experiences_data: Optional[dict[str, Any]] = None,
+        processed_experiences_data: Optional[dict[str, Any]] = None,
+    ) -> "WebMonitorGroupJob":
+        """Create a new WebMonitorGroupJob"""
+        values = {
+            "group_id": group_id,
+            "is_test_run": is_test_run,
+            "raw_experiences_data": raw_experiences_data,
+            "processed_experiences_data": processed_experiences_data,
+        }
+        stmt = insert(cls.__table__).values(**values).returning(*cls.__table__.columns)
+        result = await async_session.execute(stmt)
+        row = result.first()
+        return cls(**dict(row))
+
+    @classmethod
+    async def get_by_group_id_async(
+        cls, async_session: AsyncSession, group_id: str
+    ) -> Optional["WebMonitorGroupJob"]:
+        """Get a WebMonitorGroupJob by group_id"""
+        stmt = select(*cls.__table__.columns).where(
+            cls.__table__.c.group_id == group_id
+        )
+        result = await async_session.execute(stmt)
+        row = result.first()
+        return cls(**dict(row)) if row else None

fides/api/models/digest/__init__.py

@@ -0,0 +1,16 @@
+"""Digest models package."""
+
+from fides.api.models.digest.conditional_dependencies import (
+    DigestCondition,
+    DigestConditionType,
+)
+from fides.api.models.digest.digest_config import DigestConfig, DigestType
+from fides.api.models.digest.digest_execution import DigestTaskExecution
+
+__all__ = [
+    "DigestConfig",
+    "DigestType",
+    "DigestCondition",
+    "DigestConditionType",
+    "DigestTaskExecution",
+]

fides/api/models/digest/conditional_dependencies.py

@@ -0,0 +1,275 @@
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from sqlalchemy import Column, ForeignKey, Index, String, text
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import FidesBase
+from fides.api.db.util import EnumColumn
+from fides.api.models.conditional_dependency.conditional_dependency_base import (
+    ConditionalDependencyBase,
+    ConditionalDependencyError,
+)
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+)
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_config import DigestConfig
+
+
+class DigestConditionType(str, Enum):
+    """Types of digest conditions - each can have their own tree.
+
+    Types:
+        - RECEIVER: Conditions that determine who gets the digest
+        - CONTENT: Conditions that determine what gets included in the digest
+        - PRIORITY: Conditions that determine what is considered high priority for the digest
+          - This could be used to determine sending the digest at a different time or how
+            often it should be sent. It could also be used to format content.
+          - Example:
+            - DSRs that are due within the next week
+            - Privacy requests that are due within the next week
+            - Privacy requests for certain geographic regions
+    """
+
+    RECEIVER = "receiver"
+    CONTENT = "content"
+    PRIORITY = "priority"
+
+
+class DigestCondition(ConditionalDependencyBase):
+    """Digest conditional dependencies - multi-type hierarchies.
+
+    - Multi-type hierarchy means one digest_config can have multiple independent
+      condition trees, each with a different digest_condition_type (RECEIVER, CONTENT, PRIORITY)
+    - Within each tree, all nodes must have the same digest_condition_type
+    - This enables separate condition logic for different aspects of digest processing
+
+    Ensures that all conditions within the same tree have the same digest_condition_type.
+    This prevents logical errors where different condition types are mixed in a single
+    condition tree structure.
+
+    Example Tree Structure:
+        DigestConfig (e.g., "Weekly Privacy Digest")
+        ├── RECEIVER Dependency Condition Tree (who gets the digest)
+        │   └── Group (AND)
+        │       ├── Leaf: user.role == "admin"
+        │       └── Leaf: user.department == "privacy"
+        ├── CONTENT Dependency Condition Tree (what gets included)
+        │   └── Group (OR)
+        │       ├── Leaf: task.priority == "high"
+        │       └── Leaf: task.overdue == true
+        └── PRIORITY Dependency Condition Tree (when to send)
+            └── Leaf: task.count >= 5
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_condition"
+
+    # We need to redefine it here so that self-referential relationships
+    # can properly reference the `id` column instead of the built-in Python function.
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+
+    # Foreign key relationships
+    digest_config_id = Column(
+        String(255),
+        ForeignKey("digest_config.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    parent_id = Column(
+        String(255),
+        ForeignKey("digest_condition.id", ondelete="CASCADE"),
+        nullable=True,
+        index=True,
+    )
+
+    # Digest-specific: condition category
+    digest_condition_type = Column(
+        EnumColumn(DigestConditionType), nullable=False, index=True
+    )
+
+    # Relationships
+    digest_config = relationship("DigestConfig", back_populates="conditions")
+    parent = relationship(
+        "DigestCondition",
+        remote_side=[id],
+        back_populates="children",
+        foreign_keys=[parent_id],
+    )
+    children = relationship(
+        "DigestCondition",
+        back_populates="parent",
+        cascade="all, delete-orphan",
+        foreign_keys=[parent_id],
+    )
+
+    # Ensure only one root condition per digest_condition_type per digest_config
+    __table_args__ = (
+        Index(
+            "ix_digest_condition_unique_root_per_type",
+            "digest_config_id",
+            "digest_condition_type",
+            unique=True,
+            postgresql_where=text("parent_id IS NULL"),
+        ),
+    )
+
+    @staticmethod
+    def _validate_condition_type_consistency(db: Session, data: dict[str, Any]) -> None:
+        """Validate that a condition's digest_condition_type matches its parent's type.
+
+        Since each parent was validated when created, checking against the immediate parent
+        is sufficient to ensure tree-wide consistency.
+
+        Args:
+            db: Database session for querying
+            data: Dictionary containing condition data to validate (must include both parent_id and digest_condition_type)
+
+        Raises:
+            ValueError: If parent doesn't exist or digest_condition_type doesn't match parent's type
+        """
+        parent_id = data.get("parent_id")
+        digest_condition_type = data.get("digest_condition_type")
+
+        if not parent_id:
+            # Root condition - no validation needed
+            return
+
+        # Get the parent condition
+        parent = (
+            db.query(DigestCondition).filter(DigestCondition.id == parent_id).first()
+        )
+        if not parent:
+            raise ValueError(f"Parent condition with id '{parent_id}' does not exist")
+
+        # Validate that the new condition matches the parent's digest_condition_type
+        if parent.digest_condition_type != digest_condition_type:
+            raise ValueError(
+                f"Cannot create condition with type '{digest_condition_type}' under parent "
+                f"with type '{parent.digest_condition_type}'. All conditions in the same tree "
+                f"must have the same digest_condition_type."
+            )
+
+    @classmethod
+    def create(
+        cls,
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DigestCondition":
+        """Create a new DigestCondition with validation."""
+        # Validate condition type consistency
+        cls._validate_condition_type_consistency(db, data)
+        try:
+            return super().create(db=db, data=data, check_name=check_name)
+        except Exception as e:
+            raise ConditionalDependencyError(str(e))
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DigestCondition":
+        """Update DigestCondition with validation."""
+        # Ensure validation data includes current values for fields not being updated
+        validation_data = {
+            "parent_id": data.get("parent_id", self.parent_id),
+            "digest_condition_type": data.get(
+                "digest_condition_type", self.digest_condition_type
+            ),
+        }
+
+        # Validate before updating
+        self._validate_condition_type_consistency(db, validation_data)
+        return super().update(db=db, data=data)  # type: ignore[return-value]
+
+    def save(self, db: Session) -> "DigestCondition":
+        """Save DigestCondition with validation."""
+        # Extract current object data for validation
+        data = {
+            "parent_id": self.parent_id,
+            "digest_condition_type": self.digest_condition_type,
+        }
+
+        # Validate before saving (only if this has a parent)
+        if self.parent_id:
+            self._validate_condition_type_consistency(db, data)
+        return super().save(db=db)  # type: ignore[return-value]
+
+    @classmethod
+    def get_root_condition(
+        cls,
+        db: Session,
+        **kwargs: Any,
+    ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
+        """Get the root condition tree for a specific digest condition type.
+
+        Implementation of the abstract base method for DigestCondition's multi-type hierarchy.
+        Each digest_config can have separate condition trees for RECEIVER, CONTENT, and PRIORITY
+        types. This method retrieves the root of one specific tree.
+
+        Args:
+            db: SQLAlchemy database session for querying
+            **kwargs: Keyword arguments containing:
+                digest_config_id: ID of the digest config
+                digest_condition_type: DigestConditionType enum value
+                                     Must be one of: RECEIVER, CONTENT, PRIORITY
+
+        Returns:
+            Optional[Union[ConditionLeaf, ConditionGroup]]: Root condition tree for the specified
+                                                          type, or None if no conditions exist
+
+        Raises:
+            ValueError: If required parameters are missing
+
+        Example:
+            >>> # Get receiver conditions for a digest
+            >>> receiver_conditions = DigestCondition.get_root_condition(
+            ...     db, digest_config_id=digest_config.id,
+            ...     digest_condition_type=DigestConditionType.RECEIVER
+            ... )
+            >>> # Get content conditions for the same digest
+            >>> content_conditions = DigestCondition.get_root_condition(
+            ...     db, digest_config_id=digest_config.id,
+            ...     digest_condition_type=DigestConditionType.CONTENT
+            ... )
+        """
+        digest_config_id = kwargs.get("digest_config_id")
+        digest_condition_type = kwargs.get("digest_condition_type")
+
+        if not digest_config_id or not digest_condition_type:
+            raise ValueError(
+                "digest_config_id and digest_condition_type are required keyword arguments"
+            )
+
+        root = (
+            db.query(cls)
+            .filter(
+                cls.digest_config_id == digest_config_id,
+                cls.digest_condition_type == digest_condition_type,
+                cls.parent_id.is_(None),
+            )
+            .one_or_none()
+        )
+
+        if not root:
+            return None
+
+        return root.to_correct_condition_type()
+
+    @classmethod
+    def get_all_root_conditions(
+        cls, db: Session, digest_config_id: str
+    ) -> dict[DigestConditionType, Optional[Condition]]:
+        """Get root conditions for all digest condition types"""
+        return {
+            condition_type: cls.get_root_condition(
+                db,
+                digest_config_id=digest_config_id,
+                digest_condition_type=condition_type,
+            )
+            for condition_type in DigestConditionType
+        }

fides/api/models/digest/digest_config.py

@@ -0,0 +1,110 @@
+from enum import Enum
+from typing import TYPE_CHECKING, Optional, Union
+
+from sqlalchemy import Boolean, Column, DateTime, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+from fides.api.models.digest.conditional_dependencies import (
+    DigestCondition,
+    DigestConditionType,
+)
+from fides.api.schemas.messaging.messaging import MessagingMethod
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+)
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_execution import DigestTaskExecution
+
+
+class DigestType(str, Enum):
+    """Types of digests that can be configured."""
+
+    MANUAL_TASKS = "manual_tasks"
+    PRIVACY_REQUESTS = "privacy_requests"
+    # More types can be added here
+
+
+class DigestConfig(Base):
+    """Generic configuration for any type of digest."""
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_config"
+
+    # Digest identification
+    digest_type = Column(EnumColumn(DigestType), nullable=False, index=True)
+    name = Column(String(255), nullable=False)
+    description = Column(Text, nullable=True)
+
+    # Configuration
+    enabled = Column(Boolean, nullable=False, default=True, index=True)
+    messaging_service_type = Column(
+        EnumColumn(MessagingMethod),
+        nullable=False,
+        default=MessagingMethod.EMAIL,
+        index=True,
+    )
+    cron_expression = Column(String(100), nullable=False, default="0 9 * * 1")
+    timezone = Column(String(50), nullable=False, default="US/Eastern")
+    last_sent_at = Column(DateTime(timezone=True), nullable=True)
+    next_scheduled_at = Column(DateTime(timezone=True), nullable=True, index=True)
+    config_metadata = Column(JSONB, nullable=True, default={})
+
+    # Relationships
+    conditions = relationship(
+        "DigestCondition",
+        back_populates="digest_config",
+        cascade="all, delete-orphan",
+    )
+    executions = relationship(
+        "DigestTaskExecution",
+        back_populates="digest_config",
+        cascade="all, delete-orphan",
+        order_by="DigestTaskExecution.created_at.desc()",
+    )
+
+    def get_receiver_condition(
+        self, db: Session
+    ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
+        """Get receiver conditions for this digest config."""
+        return DigestCondition.get_root_condition(
+            db,
+            digest_config_id=self.id,
+            digest_condition_type=DigestConditionType.RECEIVER,
+        )
+
+    def get_content_condition(
+        self, db: Session
+    ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
+        """Get content conditions for this digest config."""
+
+        return DigestCondition.get_root_condition(
+            db,
+            digest_config_id=self.id,
+            digest_condition_type=DigestConditionType.CONTENT,
+        )
+
+    def get_priority_condition(
+        self, db: Session
+    ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
+        """Get priority conditions for this digest config."""
+
+        return DigestCondition.get_root_condition(
+            db,
+            digest_config_id=self.id,
+            digest_condition_type=DigestConditionType.PRIORITY,
+        )
+
+    def get_all_conditions(
+        self, db: Session
+    ) -> dict["DigestConditionType", Optional[Condition]]:
+        """Get all condition types for this digest config."""
+
+        return DigestCondition.get_all_root_conditions(db, self.id)

fides/api/models/digest/digest_execution.py

@@ -0,0 +1,142 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+from sqlalchemy.orm.attributes import flag_modified
+from sqlalchemy.sql import func
+
+from fides.api.db.base_class import Base
+from fides.api.models.worker_task import ExecutionLogStatus, WorkerTask
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_config import DigestConfig
+
+
+class DigestTaskExecution(
+    WorkerTask, Base
+):  # pylint: disable=too-many-instance-attributes
+    """
+    Model for tracking digest task execution state and progress.
+
+    This model enables graceful resumption of digest tasks after worker
+    interruptions by persisting execution state and progress information.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_task_execution"
+
+    # Foreign key to digest config
+    digest_config_id = Column(
+        String,
+        ForeignKey("digest_config.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Celery task tracking
+    celery_task_id = Column(String, nullable=True, index=True)
+
+    # Progress tracking
+    total_recipients = Column(Integer, nullable=True)
+    processed_recipients = Column(Integer, nullable=False, default=0)
+    successful_communications = Column(Integer, nullable=False, default=0)
+    failed_communications = Column(Integer, nullable=False, default=0)
+
+    # State persistence for resumption
+    execution_state = Column(JSONB, nullable=True, default={})
+    processed_user_ids = Column(JSONB, nullable=True, default=[])
+
+    # Timing information
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    completed_at = Column(DateTime(timezone=True), nullable=True)
+    last_checkpoint_at = Column(DateTime(timezone=True), nullable=True)
+
+    # Error information
+    error_message = Column(Text, nullable=True)
+
+    # Relationships
+    digest_config = relationship("DigestConfig", back_populates="executions")
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        """Return allowed action types for digest task execution."""
+        return ["digest_processing"]
+
+    def mark_started(self, db: Session, celery_task_id: str) -> None:
+        """Mark the execution as started."""
+        self.status = ExecutionLogStatus.in_processing
+        self.celery_task_id = celery_task_id
+        self.started_at = func.now()
+        self.save(db)
+
+    def mark_awaiting_processing(self, db: Session) -> None:
+        """Mark the execution as awaiting processing."""
+        self.status = ExecutionLogStatus.awaiting_processing
+        self.save(db)
+
+    def update_progress(
+        self,
+        db: Session,
+        processed_count: int,
+        successful_count: int,
+        failed_count: int,
+        processed_user_ids: List[str],
+        execution_state: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Update execution progress and create checkpoint."""
+        self.processed_recipients = processed_count
+        self.successful_communications = successful_count
+        self.failed_communications = failed_count
+        self.processed_user_ids = processed_user_ids
+        self.last_checkpoint_at = func.now()
+
+        # Mark JSONB fields as modified so SQLAlchemy knows they changed
+        flag_modified(self, "processed_user_ids")
+
+        if execution_state:
+            self.execution_state = execution_state
+            flag_modified(self, "execution_state")
+
+        self.save(db)
+
+    def mark_completed(self, db: Session) -> None:
+        """Mark the execution as completed."""
+        self.status = ExecutionLogStatus.complete
+        self.completed_at = func.now()
+        self.save(db)
+
+    def mark_failed(self, db: Session, error_message: str) -> None:
+        """Mark the execution as failed."""
+        self.status = ExecutionLogStatus.error
+        self.error_message = error_message
+        self.completed_at = func.now()
+        self.save(db)
+
+    def can_resume(self) -> bool:
+        """Check if this execution can be resumed."""
+        return (
+            self.status
+            in [
+                ExecutionLogStatus.in_processing,
+                ExecutionLogStatus.awaiting_processing,
+            ]
+            and self.processed_user_ids is not None
+        )
+
+    def get_remaining_work(self) -> Dict[str, Any]:
+        """Get information about remaining work for resumption."""
+        # Ensure processed_user_ids is always a list
+        raw_processed_user_ids = self.processed_user_ids or []
+        processed_user_ids = (
+            raw_processed_user_ids if isinstance(raw_processed_user_ids, list) else []
+        )
+        return {
+            "processed_user_ids": processed_user_ids,
+            "execution_state": self.execution_state or {},
+            "processed_count": self.processed_recipients or 0,
+            "successful_count": self.successful_communications,
+            "failed_count": self.failed_communications,
+        }