ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/models/privacy_request/privacy_request.py

@@ -4,15 +4,22 @@ from __future__ import annotations
 
 import json
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
 
 from celery.result import AsyncResult
 from loguru import logger
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String
-from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict, MutableList
-from sqlalchemy.orm import Query, RelationshipProperty, Session, backref, relationship
+from sqlalchemy.orm import (
+    Query,
+    RelationshipProperty,
+    Session,
+    backref,
+    defer,
+    relationship,
+)
 from sqlalchemy.orm.dynamic import AppenderQuery
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
     AesGcmEngine,
@@ -69,6 +76,7 @@ from fides.api.models.pre_approval_webhook import (
     PreApprovalWebhook,
     PreApprovalWebhookReply,
 )
+from fides.api.models.privacy_request.duplicate_group import DuplicateGroup
 from fides.api.models.privacy_request.execution_log import (
     COMPLETED_EXECUTION_LOG_STATUSES,
     EXITED_EXECUTION_LOG_STATUSES,
@@ -224,6 +232,18 @@ class PrivacyRequest(
     source = Column(EnumColumn(PrivacyRequestSource), nullable=True)
     location = Column(String, nullable=True)
 
+    # Duplicate detection - group ID to link duplicates together
+    duplicate_request_group_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("duplicate_group.id"),
+        nullable=True,
+    )
+    duplicate_group = relationship(
+        DuplicateGroup,
+        back_populates="privacy_requests",
+        uselist=False,
+    )
+
     # A PrivacyRequest can be soft deleted, so we store when it was deleted
     deleted_at = Column(DateTime(timezone=True), nullable=True)
     # and who deleted it
@@ -261,6 +281,12 @@ class PrivacyRequest(
         foreign_keys=[reviewed_by],
     )
 
+    submitter = relationship(
+        FidesUser,
+        backref=backref("submitted_privacy_requests", passive_deletes=True),
+        foreign_keys=[submitted_by],
+    )
+
     pre_approval_webhook_replies = relationship(
         PreApprovalWebhookReply,
         back_populates="privacy_request",
@@ -355,6 +381,87 @@ class PrivacyRequest(
 
         return super().create(db=db, data=data, check_name=check_name)
 
+    @classmethod
+    def query_without_large_columns(cls, db: Session) -> Query:
+        """
+        Returns a query for PrivacyRequest with large columns deferred to prevent OOM errors.
+
+        This excludes the large columns (_filtered_final_upload and access_result_urls)
+        which can be very large (MBs per request). Use this method for list views, bulk
+        operations, and background tasks that don't need the filtered results.
+
+        The large columns can still be accessed later if needed by accessing them directly
+        on individual PrivacyRequest instances.
+
+        Args:
+            db: Database session
+
+        Returns:
+            Query with large columns deferred
+        """
+        return db.query(cls).options(
+            defer(cls._filtered_final_upload),
+            defer(cls.access_result_urls),
+        )
+
+    def to_safe_dict(self) -> Dict[str, Any]:
+        """
+        Return a dict representation of the PrivacyRequest, excluding any fields
+        that may contain sensitive information.
+        """
+        return {
+            "id": self.id,
+            "external_id": self.external_id,
+            "status": self.status.value if self.status is not None else None,
+            "requested_at": (
+                self.requested_at.isoformat() if self.requested_at else None
+            ),
+            "started_processing_at": (
+                self.started_processing_at.isoformat()
+                if self.started_processing_at
+                else None
+            ),
+            "finished_processing_at": (
+                self.finished_processing_at.isoformat()
+                if self.finished_processing_at
+                else None
+            ),
+            "reviewed_at": self.reviewed_at.isoformat() if self.reviewed_at else None,
+            "reviewed_by": self.reviewed_by,
+            "finalized_at": (
+                self.finalized_at.isoformat() if self.finalized_at else None
+            ),
+            "finalized_by": self.finalized_by,
+            "submitted_by": self.submitted_by,
+            "custom_privacy_request_fields_approved_by": (
+                self.custom_privacy_request_fields_approved_by
+            ),
+            "identity_verified_at": (
+                self.identity_verified_at.isoformat()
+                if self.identity_verified_at
+                else None
+            ),
+            "custom_privacy_request_fields_approved_at": (
+                self.custom_privacy_request_fields_approved_at.isoformat()
+                if self.custom_privacy_request_fields_approved_at
+                else None
+            ),
+            "client_id": self.client_id,
+            "origin": self.origin,
+            "policy_id": self.policy_id,
+            "policy": self.policy.to_safe_dict() if self.policy else None,
+            "property_id": self.property_id,
+            "cancel_reason": self.cancel_reason,
+            "canceled_at": self.canceled_at.isoformat() if self.canceled_at else None,
+            "consent_preferences": self.consent_preferences,
+            "source": self.source.value if self.source else None,
+            "paused_at": self.paused_at.isoformat() if self.paused_at else None,
+            "due_date": self.due_date.isoformat() if self.due_date else None,
+            "days_left": self.days_left,
+            "custom_fields": self.get_persisted_custom_privacy_request_fields() if self.custom_fields else None,  # type: ignore[attr-defined]
+            "location": self.location,
+        }
+
     def clear_cached_values(self) -> None:
         """
         Clears all cached values associated with this privacy request from Redis.
@@ -537,7 +644,10 @@ class PrivacyRequest(
         """Verify the identification code supplied by the user
         If verified, change the status of the request to "pending", and set the datetime the identity was verified.
         """
-        if not self.status == PrivacyRequestStatus.identity_unverified:
+        if not self.status in [
+            PrivacyRequestStatus.identity_unverified,
+            PrivacyRequestStatus.duplicate,
+        ]:
             raise IdentityVerificationException(
                 f"Invalid identity verification request. Privacy request '{self.id}' status = {self.status.value}."  # type: ignore # pylint: disable=no-member
             )
@@ -615,12 +725,29 @@ class PrivacyRequest(
                 },
             )
 
-    def get_cached_identity_data(self) -> Dict[str, Any]:
-        """Retrieves any identity data pertaining to this request from the cache"""
+    def identity_prefix_cache_and_keys(self) -> Tuple[str, FidesopsRedis, List[str]]:
+        """Returns the prefix and cache keys for the identity data for this request"""
         prefix = f"id-{self.id}-identity-*"
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+        return prefix, cache, keys
+
+    def verify_cache_for_identity_data(self) -> bool:
+        """Verifies if the identity data is cached for this request"""
+        _, _, keys = self.identity_prefix_cache_and_keys()
+        return len(keys) > 0
+
+    def get_cached_identity_data(self) -> Dict[str, Any]:
+        """Retrieves any identity data pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
+        prefix, cache, keys = self.identity_prefix_cache_and_keys()
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            identity = self.get_persisted_identity()
+            self.cache_identity(identity)
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:
@@ -638,10 +765,25 @@ class PrivacyRequest(
 
     def get_cached_custom_privacy_request_fields(self) -> Dict[str, Any]:
         """Retrieves any custom fields pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
         prefix = f"id-{self.id}-custom-privacy-request-field-*"
+
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            custom_privacy_request_fields = (
+                self.get_persisted_custom_privacy_request_fields()
+            )
+            self.cache_custom_privacy_request_fields(
+                {
+                    key: CustomPrivacyRequestFieldSchema(**value)
+                    for key, value in custom_privacy_request_fields.items()
+                }
+            )
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:
@@ -903,6 +1045,8 @@ class PrivacyRequest(
             callback_type=CallbackType.pre_approval,
             identity=self.get_cached_identity_data(),
             policy_action=policy_action,
+            privacy_request=self.to_safe_dict(),
+            timestamp=datetime.utcnow(),
         )
         headers = {
             "reply-to-approve": f"/privacy-request/{self.id}/pre-approve/eligible",
@@ -943,6 +1087,8 @@ class PrivacyRequest(
             callback_type=webhook.prefix,
             identity=self.get_cached_identity_data(),
             policy_action=policy_action,
+            privacy_request=self.to_safe_dict(),
+            timestamp=datetime.utcnow(),
         )
 
         headers = {}

fides/api/models/privacy_request/provided_identity.py

@@ -5,7 +5,8 @@ from __future__ import annotations
 from enum import Enum as EnumType
 from typing import TYPE_CHECKING
 
-from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy import Column, ForeignKey, Index, String
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.orm import relationship
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
@@ -46,6 +47,26 @@ class ProvidedIdentity(HashMigrationMixin, Base):  # pylint: disable=R0904
     creation time.
     """
 
+    @declared_attr
+    def __table_args__(cls) -> tuple:  # type: ignore[override]
+        """Define table-level constructs including indexes."""
+        return (
+            # Index on foreign key for better join performance and N+1 prevention
+            # Created via migration 3ff6449c099e
+            Index(
+                "ix_providedidentity_privacy_request_id",
+                "privacy_request_id",
+            ),
+            # Hash migration tracking index from HashMigrationMixin
+            Index(
+                "idx_providedidentity_unmigrated",
+                "is_hash_migrated",
+                postgresql_where=cls.is_hash_migrated.is_(  # type: ignore[attr-defined]  # pylint: disable=no-member
+                    False
+                ),
+            ),
+        )
+
     privacy_request_id = Column(
         String,
         ForeignKey("privacyrequest.id", ondelete="CASCADE", onupdate="CASCADE"),

fides/api/models/privacy_request/request_task.py

@@ -8,8 +8,9 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 from loguru import logger
 from sqlalchemy import Boolean, Column, ForeignKey, Integer, String
 from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict, MutableList
-from sqlalchemy.orm import Query, Session, relationship
+from sqlalchemy.orm import Query, RelationshipProperty, Session, relationship
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
     AesGcmEngine,
     StringEncryptedType,
@@ -183,6 +184,14 @@ class RequestTask(WorkerTask, Base):
         uselist=False,
     )
 
+    # Stores the sub-requests data for async polling tasks
+    sub_requests: "RelationshipProperty[List[RequestTaskSubRequest]]" = relationship(
+        "RequestTaskSubRequest",
+        back_populates="request_task",
+        cascade="all, delete-orphan",
+        order_by="RequestTaskSubRequest.created_at",
+    )
+
     @property
     def request_task_address(self) -> CollectionAddress:
         """Convert the collection_address into Collection Address format"""
@@ -318,3 +327,91 @@ class RequestTask(WorkerTask, Base):
             )
 
         return task_in_flight
+
+
+class RequestTaskSubRequest(Base):
+    """
+    Model for storing individual sub-request data during the execution of a request task.
+    Supports 1:N relationship - each RequestTask can have multiple sub-requests.
+    Currently used for storing request data for polling tasks.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "request_task_sub_request"
+
+    request_task_id = Column(
+        String(255),
+        ForeignKey(
+            "requesttask.id",
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        nullable=False,
+        index=True,
+    )
+
+    request_task = relationship(
+        "RequestTask",
+        back_populates="sub_requests",
+    )
+
+    # Individual sub-request data (e.g., request_id, status, result data)
+    # Additional fields for enhanced sub-request tracking
+    param_values = Column(  # An encrypted JSON String - saved as a dict
+        StringEncryptedType(
+            type_in=JSONTypeOverride,
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+        nullable=False,
+    )
+    status = Column(String, nullable=False)
+
+    # Raw data retrieved from an access request is stored here.  This contains all of the
+    # intermediate data we retrieved, needed for downstream tasks, but hasn't been filtered
+    # by data category for the end user.
+    _access_data = Column(  # An encrypted JSON String - saved as a list of Rows
+        "access_data",
+        StringEncryptedType(
+            type_in=JSONTypeOverride,
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+    )
+
+    # Use descriptors for automatic external storage handling
+    access_data = EncryptedLargeDataDescriptor(
+        field_name="access_data", empty_default=[]
+    )
+
+    # Written after an erasure is completed
+    rows_masked = Column(Integer)
+
+    def get_correlation_id(self) -> Optional[str]:
+        """Helper method to extract correlation_id from param_values."""
+        if self.param_values and "request_id" in self.param_values:
+            return self.param_values["request_id"]
+        return None
+
+    def update_status(self, db: Session, status: str) -> None:
+        """Helper method to update the status of this sub-request."""
+        self.status = status
+        self.save(db)
+
+    def cleanup_external_storage(self) -> None:
+        """Clean up all external storage files for this sub-request"""
+        # Access the descriptor from the class to call cleanup
+        RequestTaskSubRequest.access_data.cleanup(self)
+
+    def get_access_data(self) -> List[Row]:
+        """Helper to retrieve access data or default to empty list"""
+        return self.access_data or []
+
+    def delete(self, db: Session) -> None:
+        """Override delete to cleanup external storage first"""
+        self.cleanup_external_storage()
+        super().delete(db)

fides/api/models/privacy_request/webhook.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from datetime import datetime
 from enum import Enum as EnumType
-from typing import Optional
+from typing import Any, Dict, Optional
 
 from pydantic import BaseModel, ConfigDict
 
@@ -51,6 +51,8 @@ class SecondPartyRequestFormat(BaseModel):
     identity: Identity
     policy_action: Optional[ActionType] = None
     model_config = ConfigDict(use_enum_values=True)
+    privacy_request: Dict[str, Any]
+    timestamp: datetime
 
 
 def generate_request_callback_resume_jwe(webhook: PolicyPreWebhook) -> str:

fides/api/models/saas_template_dataset.py

@@ -0,0 +1,63 @@
+from typing import Dict
+
+from sqlalchemy import Column, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session
+
+from fides.api.db.base_class import Base
+
+
+class SaasTemplateDataset(Base):
+    """
+    Stores the latest template dataset for each SaaS connection type.
+
+    This table maintains a record of the template datasets that come with
+    Fides for each SaaS connector type. It's used as a baseline for
+    detecting customer modifications to datasets.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "saas_template_dataset"
+
+    connection_type = Column(String, nullable=False, unique=True, index=True)
+    dataset_json = Column(JSONB, nullable=False)
+
+    def __repr__(self) -> str:
+        return f"<SaasTemplateDataset(connection_type='{self.connection_type}')>"
+
+    @classmethod
+    def get_or_create(  # type: ignore[override]
+        cls,
+        db: Session,
+        connector_type: str,
+        dataset_json: Dict,
+    ) -> tuple[bool, "SaasTemplateDataset"]:
+        """
+        Get existing SaasTemplateDataset by connector_type or create if it doesn't exist.
+
+        Args:
+            db: Database session
+            connector_type: The connection type identifier
+            dataset_json: The dataset JSON to use if creating
+
+        Returns:
+            Existing or newly created SaasTemplateDataset instance
+        """
+        created = False
+        template_dataset = cls.get_by(
+            db=db, field="connection_type", value=connector_type
+        )
+
+        if not template_dataset:
+            template_dataset = cls.create(
+                db=db,
+                data={
+                    "connection_type": connector_type,
+                    "dataset_json": dataset_json,
+                },
+            )
+            created = True
+
+        return created, template_dataset