ethyca-fides 2.58.2b3__py2.py3-none-any.whl → 2.58.2b5__py2.py3-none-any.whl

fides/api/models/privacy_experience.py

@@ -20,6 +20,7 @@ from fides.api.models import (
 from fides.api.models.location_regulation_selections import PrivacyNoticeRegion
 from fides.api.models.privacy_notice import PrivacyNotice
 from fides.api.models.property import Property
+from fides.api.models.tcf_publisher_restrictions import TCFConfiguration
 from fides.api.schemas.language import SupportedLanguage
 
 
@@ -228,6 +229,13 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
         EnumColumn(RejectAllMechanism),
         nullable=True,
     )
+    # Optional FK to a TCF Configuration
+
+    tcf_configuration_id = Column(
+        String,
+        ForeignKey(TCFConfiguration.id_field_path, ondelete="SET NULL"),
+        nullable=True,
+    )
 
     # Relationships
     experiences = relationship(
@@ -258,6 +266,12 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
         lazy="selectin",
     )
 
+    tcf_configuration: RelationshipProperty[Optional[TCFConfiguration]] = relationship(
+        "TCFConfiguration",
+        back_populates="privacy_experience_configs",
+        lazy="selectin",
+    )
+
     @property
     def regions(self) -> List[PrivacyNoticeRegion]:
         """Return the regions using this experience config"""
@@ -548,6 +562,17 @@ class PrivacyExperienceConfigHistory(
         EnumColumn(RejectAllMechanism),
         nullable=True,
     )
+    # Optional FK to a TCF Configuration
+    tcf_configuration_id = Column(
+        String,
+        ForeignKey(TCFConfiguration.id_field_path, ondelete="SET NULL"),
+        nullable=True,
+    )
+
+    tcf_configuration: RelationshipProperty[Optional[TCFConfiguration]] = relationship(
+        "TCFConfiguration",
+        lazy="selectin",
+    )
 
     version = Column(Float, nullable=False, default=1.0)
 
@@ -611,6 +636,7 @@ class PrivacyExperience(Base):
     tcf_special_features: List = []
     tcf_system_consents: List = []
     tcf_system_legitimate_interests: List = []
+    tcf_publisher_restrictions: List = []
     gvl: Optional[Dict] = {}
     # TCF Developer-Friendly Meta added at runtime as the result of build_tc_data_for_mobile
     meta: Dict = {}

fides/api/models/tcf_publisher_restrictions.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from pydantic import BaseModel, Field, ValidationError, model_validator
 from sqlalchemy import Column
@@ -8,10 +8,13 @@ from sqlalchemy import ForeignKey, Index, Integer, String, insert, select, updat
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.declarative import declared_attr
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
 
+if TYPE_CHECKING:
+    from fides.api.models.privacy_experience import PrivacyExperienceConfig
+
 
 class TCFRestrictionType(str, Enum):
     """Enum for TCF restriction types"""
@@ -46,14 +49,14 @@ class RangeEntry(BaseModel):
     @model_validator(mode="after")
     def validate_vendor_range(self) -> "RangeEntry":
         """Validates that end_vendor_id is greater than start_vendor_id if present."""
-        if (
-            self.end_vendor_id is not None
-            and self.end_vendor_id <= self.start_vendor_id
-        ):
-            raise ValueError("end_vendor_id must be greater than start_vendor_id")
+        if self.end_vendor_id is not None and self.end_vendor_id < self.start_vendor_id:
+            raise ValueError(
+                "end_vendor_id must be greater than or equal to start_vendor_id"
+            )
         return self
 
-    def get_end(self) -> int:
+    @property
+    def effective_end_vendor_id(self) -> int:
         """Get the effective end of the range."""
         return self.end_vendor_id or self.start_vendor_id
 
@@ -68,7 +71,7 @@ class RangeEntry(BaseModel):
         second = other if self.start_vendor_id <= other.start_vendor_id else self
 
         # If first range's end is >= second range's start, they overlap
-        return first.get_end() >= second.start_vendor_id
+        return first.effective_end_vendor_id >= second.start_vendor_id
 
 
 class TCFConfiguration(Base):
@@ -82,6 +85,12 @@ class TCFConfiguration(Base):
 
     name = Column(String, nullable=False, index=True, unique=True)
 
+    privacy_experience_configs = relationship(
+        "PrivacyExperienceConfig",
+        back_populates="tcf_configuration",
+        lazy="selectin",
+    )
+
 
 class TCFPublisherRestriction(Base):
     """
@@ -134,9 +143,20 @@ class TCFPublisherRestriction(Base):
     ) -> None:
         """
         Validates that if vendor_restriction is restrict_all_vendors, then the entries list is empty.
+        If vendor_restriction is not restrict_all_vendors, then there must be at least one entry.
         """
         if vendor_restriction == TCFVendorRestriction.restrict_all_vendors and entries:
-            raise ValueError("restrict_all_vendors cannot have any range entries")
+            raise ValueError(
+                "A restrict_all_vendors restriction cannot have any range entries"
+            )
+
+        if (
+            vendor_restriction != TCFVendorRestriction.restrict_all_vendors
+            and not entries
+        ):
+            raise ValueError(
+                f"A {vendor_restriction} restriction must have at least one range entry"
+            )
 
     @staticmethod
     def check_for_overlaps(entries: List[RangeEntry]) -> None:
@@ -186,17 +206,19 @@ class TCFPublisherRestriction(Base):
         return data
 
     @classmethod
-    async def validate_vendor_overlaps_for_purpose(
+    async def check_for_restriction_conflicts(
         cls,
+        *,
         async_db: AsyncSession,
         configuration_id: str,
         purpose_id: int,
-        new_data: dict,
+        restriction_type: TCFRestrictionType,
+        vendor_restriction: TCFVendorRestriction,
+        range_entries: Optional[list[RangeEntry]] = None,
+        restriction_id: Optional[str] = None,
     ) -> None:
         """
-        Validates that the new vendor ranges do not overlap with any existing vendor ranges for the purpose
-        in the given configuration.
-        Raises a ValueError if any vendor ranges overlap.
+        Checks that the new restriction data does not conflict with any existing restrictions for the purpose.
         """
         # First, get all the restrictions for the purpose in the given configuration
         query = (
@@ -204,28 +226,145 @@ class TCFPublisherRestriction(Base):
             .where(cls.tcf_configuration_id == configuration_id)
             .where(cls.purpose_id == purpose_id)
         )
-        restrictions = await async_db.execute(query)
-        restrictions = restrictions.scalars().all()
-
-        # If we have an existing id, we need to exclude the current restriction from the list of existing restrictions
-        # so that the new_data restrictions don't overlap with themselves
-        if "id" in new_data:
-            existing_entries = [
-                entry
-                for r in restrictions
-                for entry in r.range_entries
-                if r.id != new_data["id"]
-            ]
-        else:
-            existing_entries = [
-                entry for r in restrictions for entry in r.range_entries
-            ]
+        # If we're updating an existing restriction, exclude it from the list of
+        # restrictions so it doesn't conflict with itself
+        if restriction_id:
+            query = query.where(cls.id != restriction_id)
 
-        all_entries = [*existing_entries, *new_data.get("range_entries", [])]
-        cls.check_for_overlaps(
-            [RangeEntry.model_validate(entry) for entry in all_entries]
+        restrictions_result = await async_db.execute(query)
+        relevant_restrictions = restrictions_result.scalars().all()
+
+        if any(
+            restriction.vendor_restriction == TCFVendorRestriction.restrict_all_vendors
+            for restriction in relevant_restrictions
+        ):
+            raise ValueError(
+                f"Invalid restriction for purpose {purpose_id}: a restrict_all_vendors restriction exists for this purpose."
+            )
+
+        # If we're creating a restrict_all_vendors restriction,
+        # then there should be no other restrictions for this purpose
+        if vendor_restriction == TCFVendorRestriction.restrict_all_vendors:
+            if relevant_restrictions:
+                raise ValueError(
+                    f"Invalid restrict_all_vendors restriction for purpose {purpose_id}: other restrictions already exist for this purpose."
+                )
+
+            return None
+
+        # If we already have a restriction for that restriction type,
+        # then we raise an error
+        if any(
+            restriction.restriction_type == restriction_type
+            for restriction in relevant_restrictions
+        ):
+            raise ValueError(
+                f"Invalid {restriction_type} restriction for purpose {purpose_id}: a restriction of this type already exists for this purpose."
+            )
+
+        # We now need to check for vendor overlaps between all the restrictions.
+        # To achieve this, we need to transform allowlist-style restrictions into
+        # actual restriction ranges (rather than "allow" ranges).
+        new_range_entries = (
+            cls.transform_allowlist_restriction(range_entries or [])
+            if vendor_restriction == TCFVendorRestriction.allow_specific_vendors
+            else (range_entries or [])
         )
 
+        existing_range_entries = []
+        for restriction in relevant_restrictions:
+            range_entries = [
+                RangeEntry.model_validate(entry) for entry in restriction.range_entries
+            ]
+            transformed_range_entries = (
+                cls.transform_allowlist_restriction(range_entries)
+                if restriction.vendor_restriction
+                == TCFVendorRestriction.allow_specific_vendors
+                else range_entries
+            )
+            existing_range_entries.extend(transformed_range_entries)
+
+        all_entries = [*existing_range_entries, *new_range_entries]
+        cls.check_for_overlaps(all_entries)
+
+    @classmethod
+    def transform_allowlist_restriction(
+        cls, range_entries: list[RangeEntry]
+    ) -> list[RangeEntry]:
+        """
+        Transform allowlist-style restrictions into restriction ranges.
+        E.g if you have an "allow specific vendors" restriction with the range_entries
+        [
+            {start_vendor_id: 5, end_vendor_id: 10},
+            {start_vendor_id: 25, end_vendor_id: 40},
+            {start_vendor_id: 123 },
+            {start_vendor_id: 345, end_vendor_id: 380},
+        ],
+        the transformed restriction ranges would be:
+        [
+            {start_vendor_id: 1, end_vendor_id: 4},
+            {start_vendor_id: 11, end_vendor_id: 24},
+            {start_vendor_id: 41, end_vendor_id: 122},
+            {start_vendor_id: 124, end_vendor_id: MAX_GVL_ID},
+        ]
+        """
+        MAX_GVL_ID = 9999  # TODO: get this from the TCF spec
+
+        # First, we need to sort the range_entries by start_vendor_id
+        sorted_range_entries = sorted(range_entries, key=lambda x: x.start_vendor_id)
+
+        # Now, we need to transform the allowlist-style restrictions into
+        # actual restriction ranges.
+        transformed_range_entries: list[RangeEntry] = []
+
+        total_entries = len(sorted_range_entries)
+
+        # This shouldn't happen, but just in case
+        if total_entries == 0:
+            raise ValueError(
+                "No range entries found for allow_specific_vendors restriction"
+            )
+
+        # If the first range entry starts at a number greater than 1,
+        # we need to add a transformed range entry from 1 up to the entry's start
+        if sorted_range_entries[0].start_vendor_id > 1:
+            transformed_range_entries.append(
+                RangeEntry(
+                    start_vendor_id=1,
+                    end_vendor_id=sorted_range_entries[0].start_vendor_id - 1,
+                )
+            )
+
+        # Iterate through the sorted range_entries and transform them into restriction ranges
+        for idx, range_entry in enumerate(sorted_range_entries):
+            # For all but the last range entry, we add an entry that corresponds to the numbers
+            # between the end of the current range entry and the start of the next range entry
+            if idx < total_entries - 1:
+                # Only create a range entry if there's actually a gap between the ranges
+                next_start = sorted_range_entries[idx + 1].start_vendor_id
+                current_end = range_entry.effective_end_vendor_id
+                if next_start > current_end + 1:
+                    transformed_range_entries.append(
+                        RangeEntry(
+                            start_vendor_id=current_end + 1,
+                            end_vendor_id=next_start - 1,
+                        )
+                    )
+
+        # If the last range entry ends at a number less than MAX_GVL_ID,
+        # we need to add a transformed range entry from the last entry's end to MAX_GVL_ID
+        if sorted_range_entries[-1].effective_end_vendor_id < MAX_GVL_ID:
+            transformed_range_entries.append(
+                RangeEntry(
+                    start_vendor_id=sorted_range_entries[-1].effective_end_vendor_id
+                    + 1,
+                    end_vendor_id=MAX_GVL_ID,
+                )
+            )
+
+        # Return the transformed restriction entries
+        return transformed_range_entries
+
     @classmethod
     def create(
         cls,
@@ -249,6 +388,17 @@ class TCFPublisherRestriction(Base):
 
         data = cls.validate_publisher_restriction_data(data)
 
+        await cls.check_for_restriction_conflicts(
+            async_db=async_db,
+            configuration_id=data["tcf_configuration_id"],
+            purpose_id=data["purpose_id"],
+            restriction_type=TCFRestrictionType(data["restriction_type"]),
+            vendor_restriction=TCFVendorRestriction(data["vendor_restriction"]),
+            range_entries=[
+                RangeEntry.model_validate(entry) for entry in data["range_entries"]
+            ],
+        )
+
         values = {
             "tcf_configuration_id": data["tcf_configuration_id"],
             "purpose_id": data["purpose_id"],
@@ -257,14 +407,6 @@ class TCFPublisherRestriction(Base):
             "range_entries": data["range_entries"],
         }
 
-        # Validate that the new vendor ranges do not overlap with any existing vendor ranges for the purpose
-        await cls.validate_vendor_overlaps_for_purpose(
-            async_db=async_db,
-            configuration_id=data["tcf_configuration_id"],
-            purpose_id=data["purpose_id"],
-            new_data=values,
-        )
-
         # Insert the new restriction
         insert_stmt = insert(cls).values(values)  # type: ignore[arg-type]
         result = await async_db.execute(insert_stmt)
@@ -280,22 +422,41 @@ class TCFPublisherRestriction(Base):
         Update a TCFPublisherRestriction with the data.
         Validates the data and checks for vendor overlaps.
         """
-        # Validate the data on its own
-        data = self.validate_publisher_restriction_data(data)
+        # Create a new dict merging the existing data and the updated data
+        updated_data = {
+            "id": self.id,
+            "tcf_configuration_id": self.tcf_configuration_id,
+            "purpose_id": self.purpose_id,
+            "restriction_type": self.restriction_type,
+            "vendor_restriction": self.vendor_restriction,
+            "range_entries": self.range_entries,
+            **data,
+        }
 
-        # Validate that the new vendor ranges do not overlap with any existing vendor ranges for the purpose
-        await self.validate_vendor_overlaps_for_purpose(
+        # First validate the data on its own
+        data = self.validate_publisher_restriction_data(updated_data)
+
+        # Then check for conflicts
+        await self.check_for_restriction_conflicts(
             async_db=async_db,
             configuration_id=self.tcf_configuration_id,
             purpose_id=self.purpose_id,
-            new_data={**data, "id": self.id},  # Pass in id explicitly
+            restriction_type=TCFRestrictionType(data["restriction_type"]),
+            vendor_restriction=TCFVendorRestriction(data["vendor_restriction"]),
+            range_entries=[
+                RangeEntry.model_validate(entry) for entry in data["range_entries"]
+            ],
+            restriction_id=self.id,
         )
 
+        # Remove the id from the updated
+        updated_data.pop("id")
+
         # Finally, make the update
         update_query = (
             update(TCFPublisherRestriction)  # type: ignore[arg-type]
             .where(TCFPublisherRestriction.id == self.id)
-            .values(**data)
+            .values(**updated_data)
         )
         await async_db.execute(update_query)
         await async_db.commit()

fides/api/util/collection_util.py

@@ -122,6 +122,7 @@ def extract_key_for_address(
     return f"{dataset}:{collection}"
 
 
+# pylint: disable=too-many-branches
 def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str, Any]:
     """
     Converts a dictionary of paths/values into a nested dictionary
@@ -149,17 +150,29 @@ def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str,
         for i, current_key in enumerate(keys[:-1]):
             next_key = keys[i + 1]
             if next_key.isdigit():
-                target = target.setdefault(current_key, [])
+                if isinstance(target, dict):  # Only call setdefault on dictionaries
+                    target = target.setdefault(current_key, [])
+                elif isinstance(
+                    target, list
+                ):  # If target is a list, handle differently
+                    idx = int(current_key)
+                    while len(target) <= idx:
+                        target.append([])  # Add a list since next_key is a digit
+                    target = target[idx]
             else:
                 if isinstance(target, dict):
                     target = target.setdefault(current_key, {})
                 elif isinstance(target, list):
-                    while len(target) <= int(current_key):
+                    idx = int(current_key)
+                    while len(target) <= idx:
                         target.append({})
-                    target = target[int(current_key)]
+                    target = target[idx]
         try:
             if isinstance(target, list):
-                target.append(value)
+                idx = int(keys[-1]) if keys[-1].isdigit() else len(target)
+                while len(target) <= idx:
+                    target.append(None)
+                target[idx] = value
             else:
                 # If the value is a dictionary, add its components to the queue for processing
                 if isinstance(value, dict):
@@ -176,10 +189,12 @@ def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str,
     return output
 
 
+# pylint: disable=too-many-branches
 def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str, Any]:
     """
     Recursively flatten a dictionary or list into a flat dictionary with dot-notation keys.
     Handles nested dictionaries and arrays with proper indices.
+    Preserves empty lists and dictionaries.
 
     example:
 
@@ -191,7 +206,9 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
         "D": [
             {"E": "3"},
             {"E": "4"}
-        ]
+        ],
+        "E": [],
+        "F": {}
     }
 
     becomes
@@ -200,7 +217,9 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
         "A.B": "1",
         "A.C": "2",
         "D.0.E": "3",
-        "D.1.E": "4"
+        "D.1.E": "4",
+        "E": [],
+        "F": {}
     }
 
     Args:
@@ -211,20 +230,40 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
     Returns:
         A flattened dictionary with dot-notation keys
     """
-    items = {}
+    items: Dict[str, Any] = {}
 
     if isinstance(data, dict):
+        # Handle top-level empty dictionary case
+        if not data and not prefix:
+            return {}
+
+        # If the dictionary is empty but has a prefix, store it as is
+        if not data:
+            items[prefix] = {}
+            return items
+
         for k, v in data.items():
             new_key = f"{prefix}{separator}{k}" if prefix else k
             if isinstance(v, (dict, list)):
-                items.update(flatten_dict(v, new_key, separator))
+                if not v:  # Handle empty dict or list
+                    items[new_key] = v
+                else:
+                    items.update(flatten_dict(v, new_key, separator))
             else:
                 items[new_key] = v
     elif isinstance(data, list):
+        # If the list is empty, store it as is
+        if not data:
+            items[prefix] = []
+            return items
+
         for i, v in enumerate(data):
             new_key = f"{prefix}{separator}{i}"
             if isinstance(v, (dict, list)):
-                items.update(flatten_dict(v, new_key, separator))
+                if not v:  # Handle empty dict or list
+                    items[new_key] = v
+                else:
+                    items.update(flatten_dict(v, new_key, separator))
             else:
                 items[new_key] = v
     else:

fides/cli/commands/pull.py

@@ -5,9 +5,11 @@ from click_default_group import DefaultGroup
 
 from fides.cli.options import fides_key_argument, manifests_dir_argument
 from fides.cli.utils import with_analytics, with_server_health_check
-from fides.common.utils import echo_red
+from fides.common.utils import echo_green, echo_red
 from fides.core import parse as _parse
 from fides.core import pull as _pull
+from fides.core.api_helpers import list_server_resources
+from fides.core.pull import remove_nulls, write_manifest_file
 from fides.core.utils import git_is_dirty
 
 
@@ -60,26 +62,88 @@ def pull_all(
 
 @pull.command(name="dataset")  # type: ignore
 @click.pass_context
-@fides_key_argument
+@click.argument("fides_key", required=False)
 @manifests_dir_argument
-def dataset(
+@click.option(
+    "--all-resources",
+    "-a",
+    is_flag=True,
+    default=False,
+    help="Pull all datasets from the server.",
+)
+@click.option(
+    "--separate-files",
+    is_flag=True,
+    default=False,
+    help="Write each dataset to a separate file named after its fides_key.",
+)
+@with_analytics
+@with_server_health_check
+def pull_dataset(
     ctx: click.Context,
-    fides_key: str,
+    fides_key: Optional[str],
     manifests_dir: str,
+    all_resources: bool,
+    separate_files: bool,
 ) -> None:
     """
-    Retrieve a specific dataset from the server and update the local manifest files.
+    Retrieve datasets from the server and update the local manifest files.
+
+    If FIDES_KEY is provided, only that dataset will be pulled.
+    If --all-resources is specified, all datasets will be pulled.
+    If --separate-files is specified, each dataset will be written to a separate file.
     """
+    if not fides_key and not all_resources:
+        echo_red("Error: Either FIDES_KEY or --all-resources must be specified.")
+        raise SystemExit(1)
 
     config = ctx.obj["CONFIG"]
-    _pull.pull(
-        url=config.cli.server_url,
-        manifests_dir=manifests_dir,
-        headers=config.user.auth_header,
-        fides_key=fides_key,
-        resource_type="dataset",
-        all_resources_file=None,
-    )
+
+    # Check for unstaged git changes before proceeding
+    if git_is_dirty(manifests_dir):
+        echo_red(
+            f"There are unstaged changes in your manifest directory: '{manifests_dir}' \nAborting pull!"
+        )
+        raise SystemExit(1)
+
+    if fides_key:
+        _pull.pull(
+            url=config.cli.server_url,
+            manifests_dir=manifests_dir,
+            headers=config.user.auth_header,
+            fides_key=fides_key,
+            resource_type="dataset",
+            all_resources_file=None,
+        )
+    elif all_resources:
+        # Get all available datasets from server
+        datasets = list_server_resources(
+            url=config.cli.server_url,
+            headers=config.user.auth_header,
+            resource_type="dataset",
+            exclude_keys=[],
+        )
+
+        if not datasets:
+            echo_red("No datasets found on the server.")
+            return
+
+        # Remove null values
+        datasets = [remove_nulls(dataset) for dataset in datasets]
+
+        if separate_files:
+            # Write each dataset to a separate file
+            for dataset in datasets:
+                if "fides_key" in dataset:
+                    fides_key = dataset["fides_key"]
+                    manifest_path = f"{manifests_dir.rstrip('/')}/{fides_key}.yml"
+                    write_manifest_file(manifest_path, {"dataset": [dataset]})
+        else:
+            # Write all datasets to a single file
+            all_datasets_file = f"{manifests_dir.rstrip('/')}/datasets.yml"
+            write_manifest_file(all_datasets_file, {"dataset": datasets})
+
+        echo_green("Pull complete.")
 
 
 @pull.command(name="system")  # type: ignore

fides/core/api.py

@@ -4,7 +4,8 @@ from typing import Dict, List, Optional, Union
 
 import requests
 
-from fides.api.util.endpoint_utils import API_PREFIX
+# Not using the constant value from fides.api.util.endpoint_utils to reduce the startup time for the CLI
+API_PREFIX = "/api/v1"
 
 
 def generate_resource_url(