ethyca-fides 2.73.2b0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/models/privacy_notice.py

@@ -9,7 +9,16 @@ from typing import Any, Dict, List, Optional, Set, Type
 from fideslang.validation import FidesKey, validate_fides_key
 from sqlalchemy import Boolean, Column
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import Float, ForeignKey, String, UniqueConstraint, false, or_, text
+from sqlalchemy import (
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    UniqueConstraint,
+    false,
+    or_,
+    text,
+)
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlalchemy.ext.mutable import MutableList
 from sqlalchemy.orm import RelationshipProperty, Session, relationship, selectinload
@@ -162,11 +171,18 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         ForeignKey("privacynotice.id", ondelete="SET NULL"),
         nullable=True,
     )
+    display_order = Column(
+        Integer,
+        nullable=True,
+        doc="Optional ordering for child notices within a parent. Lower values appear first.",
+    )
+
     children: "RelationshipProperty[List[PrivacyNotice]]" = relationship(
         "PrivacyNotice",
         back_populates="parent",
         cascade="all",
         passive_deletes=True,
+        order_by=lambda: PrivacyNotice.display_order.asc().nullslast(),
     )
     parent: "RelationshipProperty[Optional[PrivacyNotice]]" = relationship(
         "PrivacyNotice",
@@ -375,6 +391,26 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         res = [result for result in experience_regions_cursor_result]
         return res[0]["regions"] if res else []
 
+    @staticmethod
+    def _set_child_notice_display_order(
+        child_notices: List[Dict[str, Any]],
+        existing_notices: List[PrivacyNotice],
+    ) -> None:
+        """
+        Sets display_order on child notice objects based on their position in the child_notices list.
+
+        Args:
+            child_notices: List of dictionaries containing notice IDs in the desired order
+            existing_notices: List of PrivacyNotice objects to update
+        """
+        notice_id_to_index = {
+            child_notice["id"]: index
+            for index, child_notice in enumerate(child_notices)
+        }
+        for child_notice_obj in existing_notices:
+            if child_notice_obj.id in notice_id_to_index:
+                child_notice_obj.display_order = notice_id_to_index[child_notice_obj.id]
+
     @classmethod
     def fetch_and_validate_notices(
         cls,
@@ -442,6 +478,10 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
 
         # automatically links and unlinks at the DB level with this assignment
         created.children = existing_notices
+
+        # Set display_order on children based on their position in the ORIGINAL child_notices list
+        cls._set_child_notice_display_order(child_notices, existing_notices)
+
         db.commit()
 
         data.pop(
@@ -481,6 +521,9 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         # automatically links and unlinks at the DB level with this assignment
         self.children = existing_notices
 
+        # Set display_order on children based on their position in the ORIGINAL child_notices list
+        self._set_child_notice_display_order(child_notices, existing_notices)
+
         # Performs a patch update of the base privacy notice
         base_notice_updated: bool = update_if_modified(self, db=db, data=data)
 
@@ -680,6 +723,7 @@ def create_historical_record_for_notice_and_translation(
     history_data: dict = create_historical_data_from_record(privacy_notice)
     history_data.pop("translations", None)
     history_data.pop("parent_id", None)
+    history_data.pop("display_order", None)
     # The source PrivacyNotice may have cached/computed attributes (e.g., @cached_property)
     # stored on the instance dict. These should not be included in historical payloads.
     # For example, 'cookies' is cached on the PrivacyNotice instance when accessed.

fides/api/models/privacy_preference.py

@@ -44,6 +44,7 @@ class RequestOrigin(Enum):
     banner_and_modal = "banner_and_modal"
     api = "api"
     tcf_overlay = "tcf_overlay"
+    headless = "headless"
 
 
 class ConsentMethod(Enum):

fides/api/models/privacy_request/duplicate_group.py

@@ -0,0 +1,84 @@
+import hashlib
+import json
+import uuid
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import Boolean, Column, Text
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base  # type: ignore[attr-defined]
+from fides.config.duplicate_detection_settings import DuplicateDetectionSettings
+
+if TYPE_CHECKING:
+    from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+
+
+def generate_deterministic_uuid(rule_version: str, dedup_key: str) -> uuid.UUID:
+    """Overrides the base class method to generate a deterministic group id.
+
+    The actual rule version is a hash of the duplicate detection settings config,
+    using simple examples here for illustration:
+    - under rule "email", duplicate@example.com → group A,
+    - under rule "email|phone", duplicate@example.com|1234567890 → group B.
+    - ... and so on.
+
+    No collisions, no overlap.
+    """
+    # Combine the rule version and the key into a stable hash
+    hash_input = f"{rule_version}:{dedup_key}".encode("utf-8")
+    return uuid.UUID(hashlib.md5(hash_input).hexdigest())
+
+
+def generate_rule_version(config: DuplicateDetectionSettings) -> str:
+    """Generate a stable short hash for the dedup rule config."""
+    normalized = json.dumps(config.model_dump(), sort_keys=True)
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8]
+
+
+class DuplicateGroup(Base):
+    """
+    A table for storing duplicate request group information.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "duplicate_group"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    rule_version = Column(Text, nullable=False)
+    # TODO: The is_active column is not used yet since we are defaulting to just email for
+    # duplicate detection. When we allow users to configure the duplicate detection settings,
+    # we will need to add a way to activate/deactivate duplicate groups.
+    # This should be done by getting the previous rule version and deactivating all groups with
+    # that rule version. New groups will be created with the new rule version.
+    is_active = Column(Boolean, default=True, nullable=False)
+    dedup_key = Column(Text, nullable=False)
+    privacy_requests = relationship(
+        "PrivacyRequest",
+        back_populates="duplicate_group",
+        lazy="dynamic",
+        primaryjoin="and_(DuplicateGroup.id == PrivacyRequest.duplicate_request_group_id)",
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "DuplicateGroup":
+        """Create a new duplicate group.
+        Override the base class method to generate a deterministic group id.
+        If the group already exists, update the is_active column to True and return the group.
+        """
+        group_id = generate_deterministic_uuid(data["rule_version"], data["dedup_key"])
+        # If the group already exists, update the is_active column to True and return the group
+        duplicate_group = (
+            db.query(DuplicateGroup).filter(DuplicateGroup.id == group_id).one_or_none()
+        )
+        if duplicate_group:
+            duplicate_group.update(db, data={"is_active": True})
+            return duplicate_group
+
+        # If the group does not exist, create a new one
+        data["id"] = group_id
+        return super().create(db, data=data, check_name=check_name)

fides/api/models/privacy_request/privacy_request.py

@@ -4,15 +4,22 @@ from __future__ import annotations
 
 import json
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
 
 from celery.result import AsyncResult
 from loguru import logger
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String
-from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict, MutableList
-from sqlalchemy.orm import Query, RelationshipProperty, Session, backref, relationship
+from sqlalchemy.orm import (
+    Query,
+    RelationshipProperty,
+    Session,
+    backref,
+    defer,
+    relationship,
+)
 from sqlalchemy.orm.dynamic import AppenderQuery
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
     AesGcmEngine,
@@ -69,6 +76,7 @@ from fides.api.models.pre_approval_webhook import (
     PreApprovalWebhook,
     PreApprovalWebhookReply,
 )
+from fides.api.models.privacy_request.duplicate_group import DuplicateGroup
 from fides.api.models.privacy_request.execution_log import (
     COMPLETED_EXECUTION_LOG_STATUSES,
     EXITED_EXECUTION_LOG_STATUSES,
@@ -225,7 +233,16 @@ class PrivacyRequest(
     location = Column(String, nullable=True)
 
     # Duplicate detection - group ID to link duplicates together
-    duplicate_request_group_id = Column(String, index=True, nullable=True)
+    duplicate_request_group_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("duplicate_group.id"),
+        nullable=True,
+    )
+    duplicate_group = relationship(
+        DuplicateGroup,
+        back_populates="privacy_requests",
+        uselist=False,
+    )
 
     # A PrivacyRequest can be soft deleted, so we store when it was deleted
     deleted_at = Column(DateTime(timezone=True), nullable=True)
@@ -364,6 +381,29 @@ class PrivacyRequest(
 
         return super().create(db=db, data=data, check_name=check_name)
 
+    @classmethod
+    def query_without_large_columns(cls, db: Session) -> Query:
+        """
+        Returns a query for PrivacyRequest with large columns deferred to prevent OOM errors.
+
+        This excludes the large columns (_filtered_final_upload and access_result_urls)
+        which can be very large (MBs per request). Use this method for list views, bulk
+        operations, and background tasks that don't need the filtered results.
+
+        The large columns can still be accessed later if needed by accessing them directly
+        on individual PrivacyRequest instances.
+
+        Args:
+            db: Database session
+
+        Returns:
+            Query with large columns deferred
+        """
+        return db.query(cls).options(
+            defer(cls._filtered_final_upload),
+            defer(cls.access_result_urls),
+        )
+
     def to_safe_dict(self) -> Dict[str, Any]:
         """
         Return a dict representation of the PrivacyRequest, excluding any fields
@@ -604,7 +644,10 @@ class PrivacyRequest(
         """Verify the identification code supplied by the user
         If verified, change the status of the request to "pending", and set the datetime the identity was verified.
         """
-        if not self.status == PrivacyRequestStatus.identity_unverified:
+        if not self.status in [
+            PrivacyRequestStatus.identity_unverified,
+            PrivacyRequestStatus.duplicate,
+        ]:
             raise IdentityVerificationException(
                 f"Invalid identity verification request. Privacy request '{self.id}' status = {self.status.value}."  # type: ignore # pylint: disable=no-member
             )
@@ -682,12 +725,29 @@ class PrivacyRequest(
                 },
             )
 
-    def get_cached_identity_data(self) -> Dict[str, Any]:
-        """Retrieves any identity data pertaining to this request from the cache"""
+    def identity_prefix_cache_and_keys(self) -> Tuple[str, FidesopsRedis, List[str]]:
+        """Returns the prefix and cache keys for the identity data for this request"""
         prefix = f"id-{self.id}-identity-*"
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+        return prefix, cache, keys
+
+    def verify_cache_for_identity_data(self) -> bool:
+        """Verifies if the identity data is cached for this request"""
+        _, _, keys = self.identity_prefix_cache_and_keys()
+        return len(keys) > 0
+
+    def get_cached_identity_data(self) -> Dict[str, Any]:
+        """Retrieves any identity data pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
+        prefix, cache, keys = self.identity_prefix_cache_and_keys()
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            identity = self.get_persisted_identity()
+            self.cache_identity(identity)
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:
@@ -705,10 +765,25 @@ class PrivacyRequest(
 
     def get_cached_custom_privacy_request_fields(self) -> Dict[str, Any]:
         """Retrieves any custom fields pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
         prefix = f"id-{self.id}-custom-privacy-request-field-*"
+
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            custom_privacy_request_fields = (
+                self.get_persisted_custom_privacy_request_fields()
+            )
+            self.cache_custom_privacy_request_fields(
+                {
+                    key: CustomPrivacyRequestFieldSchema(**value)
+                    for key, value in custom_privacy_request_fields.items()
+                }
+            )
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:

fides/api/models/privacy_request/provided_identity.py

@@ -5,7 +5,8 @@ from __future__ import annotations
 from enum import Enum as EnumType
 from typing import TYPE_CHECKING
 
-from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy import Column, ForeignKey, Index, String
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.orm import relationship
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
@@ -46,6 +47,26 @@ class ProvidedIdentity(HashMigrationMixin, Base):  # pylint: disable=R0904
     creation time.
     """
 
+    @declared_attr
+    def __table_args__(cls) -> tuple:  # type: ignore[override]
+        """Define table-level constructs including indexes."""
+        return (
+            # Index on foreign key for better join performance and N+1 prevention
+            # Created via migration 3ff6449c099e
+            Index(
+                "ix_providedidentity_privacy_request_id",
+                "privacy_request_id",
+            ),
+            # Hash migration tracking index from HashMigrationMixin
+            Index(
+                "idx_providedidentity_unmigrated",
+                "is_hash_migrated",
+                postgresql_where=cls.is_hash_migrated.is_(  # type: ignore[attr-defined]  # pylint: disable=no-member
+                    False
+                ),
+            ),
+        )
+
     privacy_request_id = Column(
         String,
         ForeignKey("privacyrequest.id", ondelete="CASCADE", onupdate="CASCADE"),

fides/api/models/saas_template_dataset.py

@@ -0,0 +1,63 @@
+from typing import Dict
+
+from sqlalchemy import Column, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session
+
+from fides.api.db.base_class import Base
+
+
+class SaasTemplateDataset(Base):
+    """
+    Stores the latest template dataset for each SaaS connection type.
+
+    This table maintains a record of the template datasets that come with
+    Fides for each SaaS connector type. It's used as a baseline for
+    detecting customer modifications to datasets.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "saas_template_dataset"
+
+    connection_type = Column(String, nullable=False, unique=True, index=True)
+    dataset_json = Column(JSONB, nullable=False)
+
+    def __repr__(self) -> str:
+        return f"<SaasTemplateDataset(connection_type='{self.connection_type}')>"
+
+    @classmethod
+    def get_or_create(  # type: ignore[override]
+        cls,
+        db: Session,
+        connector_type: str,
+        dataset_json: Dict,
+    ) -> tuple[bool, "SaasTemplateDataset"]:
+        """
+        Get existing SaasTemplateDataset by connector_type or create if it doesn't exist.
+
+        Args:
+            db: Database session
+            connector_type: The connection type identifier
+            dataset_json: The dataset JSON to use if creating
+
+        Returns:
+            Existing or newly created SaasTemplateDataset instance
+        """
+        created = False
+        template_dataset = cls.get_by(
+            db=db, field="connection_type", value=connector_type
+        )
+
+        if not template_dataset:
+            template_dataset = cls.create(
+                db=db,
+                data={
+                    "connection_type": connector_type,
+                    "dataset_json": dataset_json,
+                },
+            )
+            created = True
+
+        return created, template_dataset

fides/api/models/sql_models.py

@@ -1,5 +1,5 @@
 # type: ignore
-# pylint: disable=comparison-with-callable
+# pylint: disable=comparison-with-callable, too-many-lines
 """
 Contains all of the SqlAlchemy models for the Fides resources.
 """
@@ -15,9 +15,10 @@ from fideslang.models import DataCategory as FideslangDataCategory
 from fideslang.models import Dataset as FideslangDataset
 from fideslang.models import DatasetCollection as FideslangDatasetCollection
 from pydantic import BaseModel
-from sqlalchemy import BOOLEAN, JSON, Column
-from sqlalchemy import Enum as EnumColumn
 from sqlalchemy import (
+    BOOLEAN,
+    JSON,
+    Column,
     ForeignKey,
     Index,
     Integer,
@@ -200,6 +201,54 @@ class DataCategory(Base, FidesBase):
         remote_side=[fides_key],
     )
 
+    @classmethod
+    def create(
+        cls: Type["DataCategory"],
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DataCategory":
+        """
+        Override create to skip name uniqueness check for hierarchical taxonomy.
+
+        DataCategory is hierarchical - fides_key provides sufficient uniqueness.
+        Multiple categories can share the same name as long as their keys differ.
+        """
+        # Check if data category with same fides_key already exists
+        if "fides_key" in data and data["fides_key"]:
+            existing_by_key = (
+                db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data category with fides_key "{data["fides_key"]}" already exists.'
+                )
+        # Always skip name check - fides_key uniqueness is sufficient
+        return super().create(db=db, data=data, check_name=False)
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DataCategory":
+        """
+        Override update to check for existing data categories with the same fides_key
+        when fides_key is being changed.
+        """
+        # Check if fides_key is being changed and if it conflicts with an existing category
+        if (
+            "fides_key" in data
+            and data["fides_key"]
+            and data["fides_key"] != self.fides_key
+        ):
+            existing_by_key = (
+                db.query(DataCategory)
+                .filter(DataCategory.fides_key == data["fides_key"])
+                .first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data category with fides_key "{data["fides_key"]}" already exists.'
+                )
+        return super().update(db=db, data=data)
+
     @classmethod
     def from_fideslang_obj(
         cls, data_category: FideslangDataCategory
@@ -231,6 +280,54 @@ class DataSubject(Base, FidesBase):
     version_deprecated = Column(Text)
     replaced_by = Column(Text)
 
+    @classmethod
+    def create(
+        cls: Type["DataSubject"],
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DataSubject":
+        """
+        Override create to skip name uniqueness check for data subjects.
+
+        DataSubject allows duplicate names - fides_key provides sufficient uniqueness.
+        Multiple subjects can share the same name as long as their keys differ.
+        """
+        # Check if data subject with same fides_key already exists
+        if "fides_key" in data and data["fides_key"]:
+            existing_by_key = (
+                db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data subject with fides_key "{data["fides_key"]}" already exists.'
+                )
+        # Always skip name check - fides_key uniqueness is sufficient
+        return super().create(db=db, data=data, check_name=False)
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DataSubject":
+        """
+        Override update to check for existing data subjects with the same fides_key
+        when fides_key is being changed.
+        """
+        # Check if fides_key is being changed and if it conflicts with an existing subject
+        if (
+            "fides_key" in data
+            and data["fides_key"]
+            and data["fides_key"] != self.fides_key
+        ):
+            existing_by_key = (
+                db.query(DataSubject)
+                .filter(DataSubject.fides_key == data["fides_key"])
+                .first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data subject with fides_key "{data["fides_key"]}" already exists.'
+                )
+        return super().update(db=db, data=data)
+
 
 class DataUse(Base, FidesBase):
     """
@@ -264,6 +361,52 @@ class DataUse(Base, FidesBase):
         remote_side=[fides_key],
     )
 
+    @classmethod
+    def create(
+        cls: Type["DataUse"],
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DataUse":
+        """
+        Override create to skip name uniqueness check for hierarchical taxonomy.
+
+        DataUse is hierarchical - fides_key provides sufficient uniqueness.
+        Multiple uses can share the same name as long as their keys differ.
+        """
+        # Check if data use with same fides_key already exists
+        if "fides_key" in data and data["fides_key"]:
+            existing_by_key = (
+                db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data use with fides_key "{data["fides_key"]}" already exists.'
+                )
+        # Always skip name check - fides_key uniqueness is sufficient
+        return super().create(db=db, data=data, check_name=False)
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DataUse":
+        """
+        Override update to check for existing data uses with the same fides_key
+        when fides_key is being changed.
+        """
+        # Check if fides_key is being changed and if it conflicts with an existing use
+        if (
+            "fides_key" in data
+            and data["fides_key"]
+            and data["fides_key"] != self.fides_key
+        ):
+            existing_by_key = (
+                db.query(DataUse).filter(DataUse.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'DataUse with fides_key "{data["fides_key"]}" already exists.'
+                )
+        return super().update(db=db, data=data)
+
     @staticmethod
     def get_parent_uses_from_key(data_use_key: str) -> Set[str]:
         """
@@ -311,7 +454,7 @@ class Dataset(Base, FidesBase):
         Duplicate names are allowed.
         """
         # Check if dataset with same fides_key already exists
-        if "fides_key" in data:
+        if "fides_key" in data and data["fides_key"]:
             existing_by_key = (
                 db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
             )
@@ -775,11 +918,11 @@ class CustomFieldDefinition(Base):
     name = Column(String, index=True, nullable=False)
     description = Column(String)
     field_type = Column(
-        EnumColumn(AllowedTypes),
+        String,
         nullable=False,
     )
     allow_list_id = Column(String, ForeignKey(CustomFieldValueList.id), nullable=True)
-    resource_type = Column(EnumColumn(ResourceTypes), nullable=False)
+    resource_type = Column(String, nullable=False)
     field_definition = Column(String)
     custom_field = relationship(
         "CustomField",
@@ -843,7 +986,7 @@ class CustomField(Base):
 
     __tablename__ = "plus_custom_field"
 
-    resource_type = Column(EnumColumn(ResourceTypes), nullable=False)
+    resource_type = Column(String, nullable=False)
     resource_id = Column(String, index=True, nullable=False)
     custom_field_definition_id = Column(
         String, ForeignKey(CustomFieldDefinition.id), nullable=False