ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/models/privacy_experience.py

@@ -17,6 +17,7 @@ from fides.api.models import (
     dry_update_data,
     update_if_modified,
 )
+from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.location_regulation_selections import PrivacyNoticeRegion
 from fides.api.models.privacy_notice import PrivacyNotice
 from fides.api.models.property import Property
@@ -48,6 +49,7 @@ class Layer1ButtonOption(Enum):
     ACKNOWLEDGE = "acknowledge"
     OPT_IN_OPT_OUT = "opt_in_opt_out"
     OPT_IN_ONLY = "opt_in_only"
+    GPC_CONDITIONAL = "gpc_conditional"
 
 
 class RejectAllMechanism(Enum):
@@ -101,6 +103,11 @@ class PrivacyExperienceConfigBase:
         nullable=True,
         default=True,
     )  # base is nullable for privacy experience config history
+    cookie_deletion_based_on_host_domain = Column(
+        Boolean,
+        nullable=True,
+        default=True,
+    )  # base is nullable for privacy experience config history
 
     disabled = Column(Boolean, nullable=False, default=True)
 
@@ -108,6 +115,10 @@ class PrivacyExperienceConfigBase:
 
     show_layer1_notices = Column(Boolean, nullable=True, default=False)
 
+    # Vendor/Asset disclosure configuration
+    allow_vendor_asset_disclosure = Column(Boolean, nullable=True, default=False)
+    asset_disclosure_include_types = Column(ARRAY(String), nullable=True)
+
     @declared_attr
     def layer1_button_options(cls) -> Column:
         return Column(
@@ -139,9 +150,15 @@ class ExperienceConfigTemplate(PrivacyExperienceConfigBase, Base):
     auto_subdomain_cookie_deletion = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    cookie_deletion_based_on_host_domain = Column(
+        Boolean, nullable=False, default=True, server_default="t"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     dismissable = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    allow_vendor_asset_disclosure = Column(
+        Boolean, nullable=False, default=False, server_default="f"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     name = Column(
         String, nullable=False
     )  # Overriding PrivacyExperienceConfigBase to make non-nullable
@@ -210,12 +227,18 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
     auto_subdomain_cookie_deletion = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    cookie_deletion_based_on_host_domain = Column(
+        Boolean, nullable=False, default=True, server_default="t"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     disabled = Column(
         Boolean, nullable=False, default=True, index=True
     )  # Overridding PrivacyExperienceConfigBase to index
     dismissable = Column(
         Boolean, nullable=False, default=True, server_default="t"
     )  # Overrides PrivacyExperienceConfigBase to make non-nullable
+    allow_vendor_asset_disclosure = Column(
+        Boolean, nullable=False, default=False, server_default="f"
+    )  # Overrides PrivacyExperienceConfigBase to make non-nullable
     name = Column(
         String, nullable=False
     )  # Overriding PrivacyExperienceConfigBase to make non-nullable
@@ -250,6 +273,7 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
         secondary="experiencenotices",
         backref="experience_configs",
         lazy="selectin",
+        order_by="ExperienceNotices.display_order.asc().nullslast()",
     )
 
     translations: RelationshipProperty[List[ExperienceTranslation]] = relationship(
@@ -727,15 +751,19 @@ def link_notices_to_experience_config(
 ) -> List[PrivacyNotice]:
     """
     Link supplied Notices to ExperienceConfig and unlink any notices not supplied.
+
+    Preserves the order of notices by setting display_order on the junction table records.
     """
     new_notices: Query = db.query(PrivacyNotice).filter(
         PrivacyNotice.id.in_(notice_ids)
     )
 
+    # Add new notices
     for notice in new_notices:
         if notice not in experience_config.privacy_notices:
             experience_config.privacy_notices.append(notice)
 
+    # Remove notices that are no longer in the list
     to_remove: Set[PrivacyNotice] = set(
         notice for notice in experience_config.privacy_notices
     ) - set(notice for notice in new_notices)
@@ -744,6 +772,19 @@ def link_notices_to_experience_config(
         experience_config.privacy_notices.remove(privacy_notice)
 
     experience_config.save(db)
+
+    # Update display_order on the junction table records based on the order in notice_ids
+    for index, notice_id in enumerate(notice_ids):
+        db.query(ExperienceNotices).filter(
+            ExperienceNotices.notice_id == notice_id,
+            ExperienceNotices.experience_config_id == experience_config.id,
+        ).update({"display_order": index})
+
+    db.commit()
+
+    # Expire the relationship to force reload with correct ordering
+    db.expire(experience_config, ["privacy_notices"])
+
     return experience_config.privacy_notices
 
 

fides/api/models/privacy_notice.py

@@ -1,17 +1,29 @@
 from __future__ import annotations
 
+import itertools
 import re
 from enum import Enum
+from functools import cached_property
 from typing import Any, Dict, List, Optional, Set, Type
 
 from fideslang.validation import FidesKey, validate_fides_key
 from sqlalchemy import Boolean, Column
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import Float, ForeignKey, String, UniqueConstraint, or_, text
+from sqlalchemy import (
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    UniqueConstraint,
+    false,
+    or_,
+    text,
+)
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlalchemy.ext.mutable import MutableList
-from sqlalchemy.orm import RelationshipProperty, Session, relationship
+from sqlalchemy.orm import RelationshipProperty, Session, relationship, selectinload
 from sqlalchemy.orm.dynamic import AppenderQuery
+from sqlalchemy.sql.elements import ColumnElement
 from sqlalchemy.util import hybridproperty
 
 from fides.api.db.base_class import Base, FidesBase
@@ -41,7 +53,7 @@ class UserConsentPreference(Enum):
     tcf = "tcf"  # Overall preference set for TCF where there are numerous preferences under the single notice
 
 
-class ConsentMechanism(Enum):
+class ConsentMechanism(str, Enum):
     """
     Enum is not formalized in the DB because it may be subject to frequent change
     """
@@ -159,11 +171,18 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         ForeignKey("privacynotice.id", ondelete="SET NULL"),
         nullable=True,
     )
+    display_order = Column(
+        Integer,
+        nullable=True,
+        doc="Optional ordering for child notices within a parent. Lower values appear first.",
+    )
+
     children: "RelationshipProperty[List[PrivacyNotice]]" = relationship(
         "PrivacyNotice",
         back_populates="parent",
         cascade="all",
         passive_deletes=True,
+        order_by=lambda: PrivacyNotice.display_order.asc().nullslast(),
     )
     parent: "RelationshipProperty[Optional[PrivacyNotice]]" = relationship(
         "PrivacyNotice",
@@ -189,22 +208,150 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
 
         raise Exception("Invalid notice consent mechanism.")
 
-    @property
-    def cookies(self) -> List[Asset]:
-        """Return relevant assets of type 'cookie' (via the data use)"""
-        db = Session.object_session(self)
-        or_queries = [
-            f"array_to_string(data_uses, ',') ILIKE '{data_use}%'"
-            for data_use in self.data_uses
+    @staticmethod
+    def _get_cookie_filter_for_data_uses(data_uses: List[str]) -> ColumnElement:
+        """
+        Returns the SQLAlchemy filter clause to find cookies for the given data uses.
+        This is a helper method to keep the query logic consistent and safe.
+        """
+        if not data_uses:
+            return false()
+
+        # Use array overlap operator (&&) for exact matches - GIN index friendly
+        exact_matches_condition = Asset.data_uses.op("&&")(data_uses)
+
+        # For hierarchical children, we still need to check individual elements with LIKE
+        # They have to match the data_use and the period separator, so we know it's a hierarchical descendant
+        hierarchical_conditions = [
+            text(
+                f"EXISTS(SELECT 1 FROM unnest(data_uses) AS data_use WHERE data_use LIKE :pattern_{i})"
+            ).bindparams(**{f"pattern_{i}": f"{data_use}.%"})
+            for i, data_use in enumerate(data_uses)
         ]
 
-        query = db.query(Asset).filter(
-            Asset.asset_type == "Cookie",
-            or_(*[text(query) for query in or_queries]),
+        return or_(exact_matches_condition, *hierarchical_conditions)
+
+    @classmethod
+    def _query_cookie_assets_for_data_uses(
+        cls,
+        db: Session,
+        data_uses: Set[str],
+        exclude_cookies_from_systems: Optional[Set[str]] = None,
+    ) -> List[Asset]:
+        """
+        Query cookie Assets for the given set of data uses using the shared filter logic.
+        Applies optional exclusion by `System.fides_key` and eagerly loads the `system` relationship.
+        """
+        if not data_uses:
+            return []
+
+        cookie_filter = cls._get_cookie_filter_for_data_uses(list(data_uses))
+        query = (
+            db.query(Asset)
+            .options(selectinload("system"))
+            .filter(
+                Asset.asset_type == "Cookie",
+                cookie_filter,
+            )
         )
+        if exclude_cookies_from_systems:
+            query = query.outerjoin(System).filter(
+                or_(
+                    Asset.system_id.is_(None),
+                    System.fides_key.not_in(exclude_cookies_from_systems),
+                )
+            )
 
         return query.all()
 
+    @staticmethod
+    def _group_cookies_by_data_use(cookies: List[Asset]) -> Dict[str, List[Asset]]:
+        """Build a mapping of data_use -> list of cookie Assets."""
+        cookies_by_data_use: Dict[str, List[Asset]] = {}
+        for cookie in cookies:
+            for data_use in cookie.data_uses or []:
+                cookies_by_data_use.setdefault(data_use, []).append(cookie)
+        return cookies_by_data_use
+
+    @staticmethod
+    def _select_cookies_for_notice_data_uses(
+        notice_data_uses: List[str],
+        cookies_by_data_use: Dict[str, List[Asset]],
+    ) -> List[Asset]:
+        """
+        Return cookies that match the notice data uses either exactly or as hierarchical descendants.
+        Deduplicate by object identity; ordering is not guaranteed.
+        """
+        unique_cookies_by_id: Dict[int, Asset] = {}
+
+        for notice_data_use in notice_data_uses or []:
+            # Exact matches
+            for cookie in cookies_by_data_use.get(notice_data_use, []):
+                unique_cookies_by_id[id(cookie)] = cookie
+
+            # Hierarchical descendants: e.g., "analytics" matches "analytics.reporting"
+            prefix = f"{notice_data_use}."
+            for du_key, cookie_list in cookies_by_data_use.items():
+                if du_key.startswith(prefix):
+                    for cookie in cookie_list:
+                        unique_cookies_by_id[id(cookie)] = cookie
+
+        return list(unique_cookies_by_id.values())
+
+    @cached_property
+    def cookies(self) -> List[Asset]:
+        """
+        Return relevant assets of type 'cookie' (via the data use)
+
+        Cookies are matched to the privacy notice if they have at least one data use
+        that is either an exact match or a hierarchical descendant of a one of the
+        data uses in the privacy notice.
+
+        This is a cached_property, so the database query is only executed
+        once per instance, and the result is cached for subsequent accesses.
+        """
+        db = Session.object_session(self)
+        cookie_filter = self._get_cookie_filter_for_data_uses(self.data_uses)
+
+        return (
+            db.query(Asset)
+            .filter(
+                Asset.asset_type == "Cookie",
+                cookie_filter,
+            )
+            .all()
+        )
+
+    @classmethod
+    def load_cookie_data_for_notices(
+        cls,
+        db: Session,
+        notices: List["PrivacyNotice"],
+        exclude_cookies_from_systems: Optional[Set[str]] = None,
+    ) -> None:
+        """
+        An efficient method to bulk-load cookie data for a list of PrivacyNotice objects.
+        This prevents the "N+1" query problem by pre-populating the `cookies`
+        cached_property for each notice.
+        """
+        if not notices:
+            return
+
+        all_data_uses = set(itertools.chain.from_iterable(n.data_uses for n in notices))
+        all_relevant_cookies = cls._query_cookie_assets_for_data_uses(
+            db, all_data_uses, exclude_cookies_from_systems
+        )
+        cookies_by_data_use = cls._group_cookies_by_data_use(all_relevant_cookies)
+
+        for notice in notices:
+            matching_cookies = cls._select_cookies_for_notice_data_uses(
+                notice.data_uses, cookies_by_data_use
+            )
+
+            # Pre-populate the cache of the 'cookies' cached_property.
+            # This directly sets the attribute that the decorator would otherwise compute.
+            setattr(notice, "cookies", matching_cookies)
+
     @property
     def calculated_systems_applicable(self) -> bool:
         """Convenience property to return if any systems overlap with this notice's data uses
@@ -244,6 +391,26 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         res = [result for result in experience_regions_cursor_result]
         return res[0]["regions"] if res else []
 
+    @staticmethod
+    def _set_child_notice_display_order(
+        child_notices: List[Dict[str, Any]],
+        existing_notices: List[PrivacyNotice],
+    ) -> None:
+        """
+        Sets display_order on child notice objects based on their position in the child_notices list.
+
+        Args:
+            child_notices: List of dictionaries containing notice IDs in the desired order
+            existing_notices: List of PrivacyNotice objects to update
+        """
+        notice_id_to_index = {
+            child_notice["id"]: index
+            for index, child_notice in enumerate(child_notices)
+        }
+        for child_notice_obj in existing_notices:
+            if child_notice_obj.id in notice_id_to_index:
+                child_notice_obj.display_order = notice_id_to_index[child_notice_obj.id]
+
     @classmethod
     def fetch_and_validate_notices(
         cls,
@@ -311,6 +478,10 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
 
         # automatically links and unlinks at the DB level with this assignment
         created.children = existing_notices
+
+        # Set display_order on children based on their position in the ORIGINAL child_notices list
+        cls._set_child_notice_display_order(child_notices, existing_notices)
+
         db.commit()
 
         data.pop(
@@ -350,6 +521,9 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         # automatically links and unlinks at the DB level with this assignment
         self.children = existing_notices
 
+        # Set display_order on children based on their position in the ORIGINAL child_notices list
+        self._set_child_notice_display_order(child_notices, existing_notices)
+
         # Performs a patch update of the base privacy notice
         base_notice_updated: bool = update_if_modified(self, db=db, data=data)
 
@@ -399,6 +573,10 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         # to prevent the dry update from being added to Session.new
         updated_attributes.pop("translations", [])
         updated_attributes.pop("children", [])
+        # The source PrivacyNotice may have cached/computed attributes (e.g., @cached_property)
+        # stored on the instance dict. These should not be included in historical payloads.
+        # For example, 'cookies' is cached on the PrivacyNotice instance when accessed.
+        updated_attributes.pop("cookies", None)
 
         # create a new object with the updated attribute data to keep this
         # ORM object (i.e., `self`) pristine
@@ -545,6 +723,11 @@ def create_historical_record_for_notice_and_translation(
     history_data: dict = create_historical_data_from_record(privacy_notice)
     history_data.pop("translations", None)
     history_data.pop("parent_id", None)
+    history_data.pop("display_order", None)
+    # The source PrivacyNotice may have cached/computed attributes (e.g., @cached_property)
+    # stored on the instance dict. These should not be included in historical payloads.
+    # For example, 'cookies' is cached on the PrivacyNotice instance when accessed.
+    history_data.pop("cookies", None)
 
     updated_translation_data: dict = create_historical_data_from_record(
         notice_translation

fides/api/models/privacy_preference.py

@@ -44,6 +44,7 @@ class RequestOrigin(Enum):
     banner_and_modal = "banner_and_modal"
     api = "api"
     tcf_overlay = "tcf_overlay"
+    headless = "headless"
 
 
 class ConsentMethod(Enum):
@@ -58,6 +59,7 @@ class ConsentMethod(Enum):
     gpc = "gpc"
     individual_notice = "individual_notice"
     acknowledge = "acknowledge"
+    external_provider = "external_provider"
 
 
 class ServingComponent(Enum):

fides/api/models/privacy_request/duplicate_group.py

@@ -0,0 +1,84 @@
+import hashlib
+import json
+import uuid
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import Boolean, Column, Text
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base  # type: ignore[attr-defined]
+from fides.config.duplicate_detection_settings import DuplicateDetectionSettings
+
+if TYPE_CHECKING:
+    from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+
+
+def generate_deterministic_uuid(rule_version: str, dedup_key: str) -> uuid.UUID:
+    """Overrides the base class method to generate a deterministic group id.
+
+    The actual rule version is a hash of the duplicate detection settings config,
+    using simple examples here for illustration:
+    - under rule "email", duplicate@example.com → group A,
+    - under rule "email|phone", duplicate@example.com|1234567890 → group B.
+    - ... and so on.
+
+    No collisions, no overlap.
+    """
+    # Combine the rule version and the key into a stable hash
+    hash_input = f"{rule_version}:{dedup_key}".encode("utf-8")
+    return uuid.UUID(hashlib.md5(hash_input).hexdigest())
+
+
+def generate_rule_version(config: DuplicateDetectionSettings) -> str:
+    """Generate a stable short hash for the dedup rule config."""
+    normalized = json.dumps(config.model_dump(), sort_keys=True)
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8]
+
+
+class DuplicateGroup(Base):
+    """
+    A table for storing duplicate request group information.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "duplicate_group"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    rule_version = Column(Text, nullable=False)
+    # TODO: The is_active column is not used yet since we are defaulting to just email for
+    # duplicate detection. When we allow users to configure the duplicate detection settings,
+    # we will need to add a way to activate/deactivate duplicate groups.
+    # This should be done by getting the previous rule version and deactivating all groups with
+    # that rule version. New groups will be created with the new rule version.
+    is_active = Column(Boolean, default=True, nullable=False)
+    dedup_key = Column(Text, nullable=False)
+    privacy_requests = relationship(
+        "PrivacyRequest",
+        back_populates="duplicate_group",
+        lazy="dynamic",
+        primaryjoin="and_(DuplicateGroup.id == PrivacyRequest.duplicate_request_group_id)",
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "DuplicateGroup":
+        """Create a new duplicate group.
+        Override the base class method to generate a deterministic group id.
+        If the group already exists, update the is_active column to True and return the group.
+        """
+        group_id = generate_deterministic_uuid(data["rule_version"], data["dedup_key"])
+        # If the group already exists, update the is_active column to True and return the group
+        duplicate_group = (
+            db.query(DuplicateGroup).filter(DuplicateGroup.id == group_id).one_or_none()
+        )
+        if duplicate_group:
+            duplicate_group.update(db, data={"is_active": True})
+            return duplicate_group
+
+        # If the group does not exist, create a new one
+        data["id"] = group_id
+        return super().create(db, data=data, check_name=check_name)