ethyca-fides 2.71.1b0__py2.py3-none-any.whl → 2.71.1rc0__py2.py3-none-any.whl

fides/api/models/digest/digest_execution.py

@@ -0,0 +1,132 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+from sqlalchemy.sql import func
+
+from fides.api.db.base_class import Base
+from fides.api.models.worker_task import ExecutionLogStatus, WorkerTask
+
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_config import DigestConfig
+
+
+class DigestTaskExecution(
+    WorkerTask, Base
+):  # pylint: disable=too-many-instance-attributes
+    """
+    Model for tracking digest task execution state and progress.
+
+    This model enables graceful resumption of digest tasks after worker
+    interruptions by persisting execution state and progress information.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "digest_task_execution"
+
+    # Foreign key to digest config
+    digest_config_id = Column(
+        String,
+        ForeignKey("digest_config.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Celery task tracking
+    celery_task_id = Column(String, nullable=True, index=True)
+
+    # Progress tracking
+    total_recipients = Column(Integer, nullable=True)
+    processed_recipients = Column(Integer, nullable=False, default=0)
+    successful_communications = Column(Integer, nullable=False, default=0)
+    failed_communications = Column(Integer, nullable=False, default=0)
+
+    # State persistence for resumption
+    execution_state = Column(JSONB, nullable=True, default={})
+    processed_user_ids = Column(JSONB, nullable=True, default=[])
+
+    # Timing information
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    completed_at = Column(DateTime(timezone=True), nullable=True)
+    last_checkpoint_at = Column(DateTime(timezone=True), nullable=True)
+
+    # Error information
+    error_message = Column(Text, nullable=True)
+
+    # Relationships
+    digest_config = relationship("DigestConfig", back_populates="executions")
+
+    @classmethod
+    def allowed_action_types(cls) -> List[str]:
+        """Return allowed action types for digest task execution."""
+        return ["digest_processing"]
+
+    def mark_started(self, db: Session, celery_task_id: str) -> None:
+        """Mark the execution as started."""
+        self.status = ExecutionLogStatus.in_processing
+        self.celery_task_id = celery_task_id
+        self.started_at = func.now()
+        self.save(db)
+
+    def mark_awaiting_processing(self, db: Session) -> None:
+        """Mark the execution as awaiting processing."""
+        self.status = ExecutionLogStatus.awaiting_processing
+        self.save(db)
+
+    def update_progress(
+        self,
+        db: Session,
+        processed_count: int,
+        successful_count: int,
+        failed_count: int,
+        processed_user_ids: List[str],
+        execution_state: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Update execution progress and create checkpoint."""
+        self.processed_recipients = processed_count
+        self.successful_communications = successful_count
+        self.failed_communications = failed_count
+        self.processed_user_ids = processed_user_ids
+        self.last_checkpoint_at = func.now()
+
+        if execution_state:
+            self.execution_state = execution_state
+
+        self.save(db)
+
+    def mark_completed(self, db: Session) -> None:
+        """Mark the execution as completed."""
+        self.status = ExecutionLogStatus.complete
+        self.completed_at = func.now()
+        self.save(db)
+
+    def mark_failed(self, db: Session, error_message: str) -> None:
+        """Mark the execution as failed."""
+        self.status = ExecutionLogStatus.error
+        self.error_message = error_message
+        self.completed_at = func.now()
+        self.save(db)
+
+    def can_resume(self) -> bool:
+        """Check if this execution can be resumed."""
+        return (
+            self.status
+            in [
+                ExecutionLogStatus.in_processing,
+                ExecutionLogStatus.awaiting_processing,
+            ]
+            and self.processed_user_ids is not None
+        )
+
+    def get_remaining_work(self) -> Dict[str, Any]:
+        """Get information about remaining work for resumption."""
+        return {
+            "processed_user_ids": self.processed_user_ids or [],
+            "execution_state": self.execution_state or {},
+            "processed_count": self.processed_recipients or 0,
+            "successful_count": self.successful_communications,
+            "failed_count": self.failed_communications,
+        }

fides/api/models/event_audit.py

@@ -31,6 +31,14 @@ class EventAuditType(str, EnumType):
     taxonomy_element_updated = "taxonomy.element.updated"
     taxonomy_element_deleted = "taxonomy.element.deleted"
 
+    # Digest
+    digest_execution_started = "digest.execution.started"
+    digest_execution_completed = "digest.execution.completed"
+    digest_execution_interrupted = "digest.execution.interrupted"
+    digest_execution_resumed = "digest.execution.resumed"
+    digest_communications_sent = "digest.communications.sent"
+    digest_checkpoint_created = "digest.checkpoint.created"
+
 
 class EventAuditStatus(str, EnumType):
     """Status enum for event audit logging."""

fides/api/models/privacy_experience.py

@@ -108,6 +108,10 @@ class PrivacyExperienceConfigBase:
 
     show_layer1_notices = Column(Boolean, nullable=True, default=False)
 
+    # Vendor/Asset disclosure configuration
+    allow_vendor_asset_disclosure = Column(Boolean, nullable=True, default=False)
+    asset_disclosure_include_types = Column(ARRAY(String), nullable=True)
+
     @declared_attr
     def layer1_button_options(cls) -> Column:
         return Column(
@@ -142,6 +146,9 @@ class ExperienceConfigTemplate(PrivacyExperienceConfigBase, Base):
     dismissable = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    allow_vendor_asset_disclosure = Column(
+        Boolean, nullable=False, default=False, server_default="f"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     name = Column(
         String, nullable=False
     )  # Overriding PrivacyExperienceConfigBase to make non-nullable
@@ -216,6 +223,9 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
     dismissable = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    allow_vendor_asset_disclosure = Column(
+        Boolean, nullable=False, default=False, server_default="f"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     name = Column(
         String, nullable=False
     )  # Overriding PrivacyExperienceConfigBase to make non-nullable

fides/api/models/privacy_notice.py

@@ -1,17 +1,20 @@
 from __future__ import annotations
 
+import itertools
 import re
 from enum import Enum
+from functools import cached_property
 from typing import Any, Dict, List, Optional, Set, Type
 
 from fideslang.validation import FidesKey, validate_fides_key
 from sqlalchemy import Boolean, Column
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import Float, ForeignKey, String, UniqueConstraint, or_, text
+from sqlalchemy import Float, ForeignKey, String, UniqueConstraint, false, or_, text
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlalchemy.ext.mutable import MutableList
-from sqlalchemy.orm import RelationshipProperty, Session, relationship
+from sqlalchemy.orm import RelationshipProperty, Session, relationship, selectinload
 from sqlalchemy.orm.dynamic import AppenderQuery
+from sqlalchemy.sql.elements import ColumnElement
 from sqlalchemy.util import hybridproperty
 
 from fides.api.db.base_class import Base, FidesBase
@@ -189,21 +192,17 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
 
         raise Exception("Invalid notice consent mechanism.")
 
-    @property
-    def cookies(self) -> List[Asset]:
+    @staticmethod
+    def _get_cookie_filter_for_data_uses(data_uses: List[str]) -> ColumnElement:
         """
-        Return the privacy notice's assets of type 'cookie'.
-        Cookies are matched to the privacy notice if they have at least one data use
-        that is either an exact match or a hierarchical descendant of a one of the
-        data uses in the privacy notice.
+        Returns the SQLAlchemy filter clause to find cookies for the given data uses.
+        This is a helper method to keep the query logic consistent and safe.
         """
-        db = Session.object_session(self)
-
-        if not self.data_uses:
-            return []
+        if not data_uses:
+            return false()
 
         # Use array overlap operator (&&) for exact matches - GIN index friendly
-        exact_matches_condition = Asset.data_uses.op("&&")(self.data_uses)
+        exact_matches_condition = Asset.data_uses.op("&&")(data_uses)
 
         # For hierarchical children, we still need to check individual elements with LIKE
         # They have to match the data_use and the period separator, so we know it's a hierarchical descendant
@@ -211,20 +210,132 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
             text(
                 f"EXISTS(SELECT 1 FROM unnest(data_uses) AS data_use WHERE data_use LIKE :pattern_{i})"
             ).bindparams(**{f"pattern_{i}": f"{data_use}.%"})
-            for i, data_use in enumerate(self.data_uses)
+            for i, data_use in enumerate(data_uses)
         ]
 
-        asset_matching_condition = or_(
-            exact_matches_condition, *hierarchical_conditions
-        )
+        return or_(exact_matches_condition, *hierarchical_conditions)
+
+    @classmethod
+    def _query_cookie_assets_for_data_uses(
+        cls,
+        db: Session,
+        data_uses: Set[str],
+        exclude_cookies_from_systems: Optional[Set[str]] = None,
+    ) -> List[Asset]:
+        """
+        Query cookie Assets for the given set of data uses using the shared filter logic.
+        Applies optional exclusion by `System.fides_key` and eagerly loads the `system` relationship.
+        """
+        if not data_uses:
+            return []
 
-        query = db.query(Asset).filter(
-            Asset.asset_type == "Cookie",
-            asset_matching_condition,
+        cookie_filter = cls._get_cookie_filter_for_data_uses(list(data_uses))
+        query = (
+            db.query(Asset)
+            .options(selectinload("system"))
+            .filter(
+                Asset.asset_type == "Cookie",
+                cookie_filter,
+            )
         )
+        if exclude_cookies_from_systems:
+            query = query.outerjoin(System).filter(
+                or_(
+                    Asset.system_id.is_(None),
+                    System.fides_key.not_in(exclude_cookies_from_systems),
+                )
+            )
 
         return query.all()
 
+    @staticmethod
+    def _group_cookies_by_data_use(cookies: List[Asset]) -> Dict[str, List[Asset]]:
+        """Build a mapping of data_use -> list of cookie Assets."""
+        cookies_by_data_use: Dict[str, List[Asset]] = {}
+        for cookie in cookies:
+            for data_use in cookie.data_uses or []:
+                cookies_by_data_use.setdefault(data_use, []).append(cookie)
+        return cookies_by_data_use
+
+    @staticmethod
+    def _select_cookies_for_notice_data_uses(
+        notice_data_uses: List[str],
+        cookies_by_data_use: Dict[str, List[Asset]],
+    ) -> List[Asset]:
+        """
+        Return cookies that match the notice data uses either exactly or as hierarchical descendants.
+        Deduplicate by object identity; ordering is not guaranteed.
+        """
+        unique_cookies_by_id: Dict[int, Asset] = {}
+
+        for notice_data_use in notice_data_uses or []:
+            # Exact matches
+            for cookie in cookies_by_data_use.get(notice_data_use, []):
+                unique_cookies_by_id[id(cookie)] = cookie
+
+            # Hierarchical descendants: e.g., "analytics" matches "analytics.reporting"
+            prefix = f"{notice_data_use}."
+            for du_key, cookie_list in cookies_by_data_use.items():
+                if du_key.startswith(prefix):
+                    for cookie in cookie_list:
+                        unique_cookies_by_id[id(cookie)] = cookie
+
+        return list(unique_cookies_by_id.values())
+
+    @cached_property
+    def cookies(self) -> List[Asset]:
+        """
+        Return relevant assets of type 'cookie' (via the data use)
+
+        Cookies are matched to the privacy notice if they have at least one data use
+        that is either an exact match or a hierarchical descendant of a one of the
+        data uses in the privacy notice.
+
+        This is a cached_property, so the database query is only executed
+        once per instance, and the result is cached for subsequent accesses.
+        """
+        db = Session.object_session(self)
+        cookie_filter = self._get_cookie_filter_for_data_uses(self.data_uses)
+
+        return (
+            db.query(Asset)
+            .filter(
+                Asset.asset_type == "Cookie",
+                cookie_filter,
+            )
+            .all()
+        )
+
+    @classmethod
+    def load_cookie_data_for_notices(
+        cls,
+        db: Session,
+        notices: List["PrivacyNotice"],
+        exclude_cookies_from_systems: Optional[Set[str]] = None,
+    ) -> None:
+        """
+        An efficient method to bulk-load cookie data for a list of PrivacyNotice objects.
+        This prevents the "N+1" query problem by pre-populating the `cookies`
+        cached_property for each notice.
+        """
+        if not notices:
+            return
+
+        all_data_uses = set(itertools.chain.from_iterable(n.data_uses for n in notices))
+        all_relevant_cookies = cls._query_cookie_assets_for_data_uses(
+            db, all_data_uses, exclude_cookies_from_systems
+        )
+        cookies_by_data_use = cls._group_cookies_by_data_use(all_relevant_cookies)
+
+        for notice in notices:
+            matching_cookies = cls._select_cookies_for_notice_data_uses(
+                notice.data_uses, cookies_by_data_use
+            )
+
+            # Pre-populate the cache of the 'cookies' cached_property.
+            # This directly sets the attribute that the decorator would otherwise compute.
+            setattr(notice, "cookies", matching_cookies)
+
     @property
     def calculated_systems_applicable(self) -> bool:
         """Convenience property to return if any systems overlap with this notice's data uses

fides/api/models/privacy_request/request_task.py

@@ -8,8 +8,9 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 from loguru import logger
 from sqlalchemy import Boolean, Column, ForeignKey, Integer, String
 from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict, MutableList
-from sqlalchemy.orm import Query, Session, relationship
+from sqlalchemy.orm import Query, RelationshipProperty, Session, relationship
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
     AesGcmEngine,
     StringEncryptedType,
@@ -183,6 +184,14 @@ class RequestTask(WorkerTask, Base):
         uselist=False,
     )
 
+    # Stores the sub-requests data for async polling tasks
+    sub_requests: "RelationshipProperty[List[RequestTaskSubRequest]]" = relationship(
+        "RequestTaskSubRequest",
+        back_populates="request_task",
+        cascade="all, delete-orphan",
+        order_by="RequestTaskSubRequest.created_at",
+    )
+
     @property
     def request_task_address(self) -> CollectionAddress:
         """Convert the collection_address into Collection Address format"""
@@ -318,3 +327,91 @@ class RequestTask(WorkerTask, Base):
             )
 
         return task_in_flight
+
+
+class RequestTaskSubRequest(Base):
+    """
+    Model for storing individual sub-request data during the execution of a request task.
+    Supports 1:N relationship - each RequestTask can have multiple sub-requests.
+    Currently used for storing request data for polling tasks.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "request_task_sub_request"
+
+    request_task_id = Column(
+        String(255),
+        ForeignKey(
+            "requesttask.id",
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        nullable=False,
+        index=True,
+    )
+
+    request_task = relationship(
+        "RequestTask",
+        back_populates="sub_requests",
+    )
+
+    # Individual sub-request data (e.g., request_id, status, result data)
+    # Additional fields for enhanced sub-request tracking
+    param_values = Column(  # An encrypted JSON String - saved as a dict
+        StringEncryptedType(
+            type_in=JSONTypeOverride,
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+        nullable=False,
+    )
+    status = Column(String, nullable=False)
+
+    # Raw data retrieved from an access request is stored here.  This contains all of the
+    # intermediate data we retrieved, needed for downstream tasks, but hasn't been filtered
+    # by data category for the end user.
+    _access_data = Column(  # An encrypted JSON String - saved as a list of Rows
+        "access_data",
+        StringEncryptedType(
+            type_in=JSONTypeOverride,
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+    )
+
+    # Use descriptors for automatic external storage handling
+    access_data = EncryptedLargeDataDescriptor(
+        field_name="access_data", empty_default=[]
+    )
+
+    # Written after an erasure is completed
+    rows_masked = Column(Integer)
+
+    def get_correlation_id(self) -> Optional[str]:
+        """Helper method to extract correlation_id from param_values."""
+        if self.param_values and "request_id" in self.param_values:
+            return self.param_values["request_id"]
+        return None
+
+    def update_status(self, db: Session, status: str) -> None:
+        """Helper method to update the status of this sub-request."""
+        self.status = status
+        self.save(db)
+
+    def cleanup_external_storage(self) -> None:
+        """Clean up all external storage files for this sub-request"""
+        # Access the descriptor from the class to call cleanup
+        RequestTaskSubRequest.access_data.cleanup(self)
+
+    def get_access_data(self) -> List[Row]:
+        """Helper to retrieve access data or default to empty list"""
+        return self.access_data or []
+
+    def delete(self, db: Session) -> None:
+        """Override delete to cleanup external storage first"""
+        self.cleanup_external_storage()
+        super().delete(db)

fides/api/models/worker_task.py

@@ -17,6 +17,14 @@ class ExecutionLogStatus(enum.Enum):
     awaiting_processing = "paused"  # "paused" in the database to avoid a migration, but use "awaiting_processing" in the app
     retrying = "retrying"
     skipped = "skipped"
+    polling = "polling"
+
+
+# Statuses that can be resumed
+RESUMABLE_EXECUTION_LOG_STATUSES = [
+    ExecutionLogStatus.pending,
+    ExecutionLogStatus.polling,
+]
 
 
 class WorkerTask:

fides/api/schemas/saas/async_polling_configuration.py

@@ -0,0 +1,81 @@
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, Field, model_validator
+
+from fides.api.schemas.saas.saas_config import SaaSRequest
+from fides.api.schemas.saas.strategy_configuration import StrategyConfiguration
+from fides.api.util.collection_util import Row
+
+
+class SupportedDataType(Enum):
+    """Supported data types for polling async DSR result requests."""
+
+    # Structured data types that can be parsed into rows
+    json = "json"  # Parsed into List[Row] from JSON response
+    csv = "csv"  # Parsed into List[Row] from CSV response
+    # Binary/non-parseable data stored as raw bytes
+    attachment = "attachment"  # Binary files (.zip, .pdf, .xml, etc.) stored as bytes
+
+
+class PollingResultType(Enum):
+    """Types of results from async polling operations."""
+
+    rows = "rows"  # Structured data parsed into List[Row]
+    attachment = "attachment"  # Binary file data stored as bytes
+
+
+class PollingResult(BaseModel):
+    """
+    Flexible result container for async polling operations.
+    Handles both structured data and file attachments.
+    """
+
+    data: Union[List[Row], bytes]
+    result_type: PollingResultType
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class PollingStatusRequest(SaaSRequest):
+    """
+    Extended SaaSRequest for checking async job status.
+    Uses request_override for custom status checking logic or standard fields for simple cases.
+    """
+
+    status_path: Optional[str] = None
+    status_completed_value: Optional[Union[str, bool, int]] = None
+
+    @model_validator(mode="after")
+    def validate_status_fields(self) -> "PollingStatusRequest":
+        """Ensure required fields are present unless using an override."""
+        if self.request_override:
+            return self
+
+        if not self.status_path:
+            raise ValueError("status_path is required when request_override is not set")
+        if self.status_completed_value is None:
+            raise ValueError(
+                "status_completed_value is required when request_override is not set"
+            )
+        return self
+
+
+class PollingResultRequest(SaaSRequest):
+    """
+    Extended SaaSRequest for retrieving async job results.
+    Uses request_override for custom result retrieval or standard HTTP request for simple cases.
+    Data type is automatically inferred from response.
+    """
+
+    result_path: Optional[str] = None
+
+
+class AsyncPollingConfiguration(StrategyConfiguration):
+    """
+    Simplified configuration for polling async DSR requests.
+    The main read request serves as the initial request.
+    """
+
+    status_request: PollingStatusRequest
+    # result_request is optional for delete/update operations
+    result_request: Optional[PollingResultRequest] = None

fides/api/schemas/saas/saas_config.py

@@ -2,7 +2,9 @@ from typing import Any, Dict, List, Optional, Set, Union
 
 from fideslang.models import FidesCollectionKey, FidesDatasetReference
 from fideslang.validation import FidesKey
-from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+from pydantic import BaseModel, ConfigDict
+from pydantic import Field as PydanticField
+from pydantic import field_validator, model_validator
 
 from fides.api.common_exceptions import ValidationError
 from fides.api.graph.config import (
@@ -115,6 +117,10 @@ class SaaSRequest(BaseModel):
     skip_missing_param_values: Optional[bool] = (
         False  # Skip instead of raising an exception if placeholders can't be populated in body
     )
+    correlation_id_path: Optional[str] = PydanticField(
+        default=None,
+        description="The path to the correlation ID in the response. For use with async polling.",
+    )
     model_config = ConfigDict(
         from_attributes=True, use_enum_values=True, extra="forbid"
     )
@@ -213,7 +219,7 @@ class SaaSRequest(BaseModel):
 class ReadSaaSRequest(SaaSRequest):
     """
     An extension of the base SaaSRequest that allows the inclusion of an output template
-    that is used to format each collection result.
+    that is used to format each collection result, and correlation_id_path for async polling.
     """
 
     output: Optional[str] = None
@@ -230,7 +236,8 @@ class ReadSaaSRequest(SaaSRequest):
                 raise ValueError(
                     "A read request must specify a method if a path is provided and no request_override is specified"
                 )
-        else:
+
+        if self.request_override:
             allowed_fields = {
                 "request_override",
                 "param_values",

fides/api/schemas/saas/strategy_configuration.py

@@ -178,15 +178,3 @@ class OAuth2ClientCredentialsConfiguration(OAuth2BaseConfiguration):
     """
 
     refresh_request: Optional[SaaSRequest] = Field(exclude=True)
-
-
-class PollingAsyncDSRConfiguration(StrategyConfiguration):
-    """
-    Configuration for polling async DSR requests.
-    """
-
-    status_request: SaaSRequest
-    status_path: str
-    status_completed_value: Optional[str] = None
-    result_request: SaaSRequest
-    result_path: str