ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/models/sql_models.py

@@ -1,5 +1,5 @@
 # type: ignore
-# pylint: disable=comparison-with-callable
+# pylint: disable=comparison-with-callable, too-many-lines
 """
 Contains all of the SqlAlchemy models for the Fides resources.
 """
@@ -15,9 +15,10 @@ from fideslang.models import DataCategory as FideslangDataCategory
 from fideslang.models import Dataset as FideslangDataset
 from fideslang.models import DatasetCollection as FideslangDatasetCollection
 from pydantic import BaseModel
-from sqlalchemy import BOOLEAN, JSON, Column
-from sqlalchemy import Enum as EnumColumn
 from sqlalchemy import (
+    BOOLEAN,
+    JSON,
+    Column,
     ForeignKey,
     Index,
     Integer,
@@ -184,6 +185,9 @@ class DataCategory(Base, FidesBase):
     version_deprecated = Column(Text)
     replaced_by = Column(Text)
 
+    # LLM Classification
+    tagging_instructions = Column(Text, nullable=True)
+
     children: "RelationshipProperty[List[DataCategory]]" = relationship(
         "DataCategory",
         back_populates="parent",
@@ -197,6 +201,54 @@ class DataCategory(Base, FidesBase):
         remote_side=[fides_key],
     )
 
+    @classmethod
+    def create(
+        cls: Type["DataCategory"],
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DataCategory":
+        """
+        Override create to skip name uniqueness check for hierarchical taxonomy.
+
+        DataCategory is hierarchical - fides_key provides sufficient uniqueness.
+        Multiple categories can share the same name as long as their keys differ.
+        """
+        # Check if data category with same fides_key already exists
+        if "fides_key" in data and data["fides_key"]:
+            existing_by_key = (
+                db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data category with fides_key "{data["fides_key"]}" already exists.'
+                )
+        # Always skip name check - fides_key uniqueness is sufficient
+        return super().create(db=db, data=data, check_name=False)
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DataCategory":
+        """
+        Override update to check for existing data categories with the same fides_key
+        when fides_key is being changed.
+        """
+        # Check if fides_key is being changed and if it conflicts with an existing category
+        if (
+            "fides_key" in data
+            and data["fides_key"]
+            and data["fides_key"] != self.fides_key
+        ):
+            existing_by_key = (
+                db.query(DataCategory)
+                .filter(DataCategory.fides_key == data["fides_key"])
+                .first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data category with fides_key "{data["fides_key"]}" already exists.'
+                )
+        return super().update(db=db, data=data)
+
     @classmethod
     def from_fideslang_obj(
         cls, data_category: FideslangDataCategory
@@ -228,6 +280,54 @@ class DataSubject(Base, FidesBase):
     version_deprecated = Column(Text)
     replaced_by = Column(Text)
 
+    @classmethod
+    def create(
+        cls: Type["DataSubject"],
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DataSubject":
+        """
+        Override create to skip name uniqueness check for data subjects.
+
+        DataSubject allows duplicate names - fides_key provides sufficient uniqueness.
+        Multiple subjects can share the same name as long as their keys differ.
+        """
+        # Check if data subject with same fides_key already exists
+        if "fides_key" in data and data["fides_key"]:
+            existing_by_key = (
+                db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data subject with fides_key "{data["fides_key"]}" already exists.'
+                )
+        # Always skip name check - fides_key uniqueness is sufficient
+        return super().create(db=db, data=data, check_name=False)
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DataSubject":
+        """
+        Override update to check for existing data subjects with the same fides_key
+        when fides_key is being changed.
+        """
+        # Check if fides_key is being changed and if it conflicts with an existing subject
+        if (
+            "fides_key" in data
+            and data["fides_key"]
+            and data["fides_key"] != self.fides_key
+        ):
+            existing_by_key = (
+                db.query(DataSubject)
+                .filter(DataSubject.fides_key == data["fides_key"])
+                .first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data subject with fides_key "{data["fides_key"]}" already exists.'
+                )
+        return super().update(db=db, data=data)
+
 
 class DataUse(Base, FidesBase):
     """
@@ -261,6 +361,52 @@ class DataUse(Base, FidesBase):
         remote_side=[fides_key],
     )
 
+    @classmethod
+    def create(
+        cls: Type["DataUse"],
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "DataUse":
+        """
+        Override create to skip name uniqueness check for hierarchical taxonomy.
+
+        DataUse is hierarchical - fides_key provides sufficient uniqueness.
+        Multiple uses can share the same name as long as their keys differ.
+        """
+        # Check if data use with same fides_key already exists
+        if "fides_key" in data and data["fides_key"]:
+            existing_by_key = (
+                db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Data use with fides_key "{data["fides_key"]}" already exists.'
+                )
+        # Always skip name check - fides_key uniqueness is sufficient
+        return super().create(db=db, data=data, check_name=False)
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "DataUse":
+        """
+        Override update to check for existing data uses with the same fides_key
+        when fides_key is being changed.
+        """
+        # Check if fides_key is being changed and if it conflicts with an existing use
+        if (
+            "fides_key" in data
+            and data["fides_key"]
+            and data["fides_key"] != self.fides_key
+        ):
+            existing_by_key = (
+                db.query(DataUse).filter(DataUse.fides_key == data["fides_key"]).first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'DataUse with fides_key "{data["fides_key"]}" already exists.'
+                )
+        return super().update(db=db, data=data)
+
     @staticmethod
     def get_parent_uses_from_key(data_use_key: str) -> Set[str]:
         """
@@ -308,7 +454,7 @@ class Dataset(Base, FidesBase):
         Duplicate names are allowed.
         """
         # Check if dataset with same fides_key already exists
-        if "fides_key" in data:
+        if "fides_key" in data and data["fides_key"]:
             existing_by_key = (
                 db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
             )
@@ -461,6 +607,14 @@ class System(Base, FidesBase):
         "Asset", back_populates="system", lazy="selectin", uselist=True, viewonly=True
     )
 
+    system_groups = relationship(
+        "SystemGroup",
+        secondary="system_group_member",
+        back_populates="systems",
+        lazy="selectin",
+        viewonly=True,
+    )
+
     @classmethod
     def get_data_uses(
         cls: Type[System], systems: List[System], include_parents: bool = True
@@ -764,11 +918,11 @@ class CustomFieldDefinition(Base):
     name = Column(String, index=True, nullable=False)
     description = Column(String)
     field_type = Column(
-        EnumColumn(AllowedTypes),
+        String,
         nullable=False,
     )
     allow_list_id = Column(String, ForeignKey(CustomFieldValueList.id), nullable=True)
-    resource_type = Column(EnumColumn(ResourceTypes), nullable=False)
+    resource_type = Column(String, nullable=False)
     field_definition = Column(String)
     custom_field = relationship(
         "CustomField",
@@ -832,7 +986,7 @@ class CustomField(Base):
 
     __tablename__ = "plus_custom_field"
 
-    resource_type = Column(EnumColumn(ResourceTypes), nullable=False)
+    resource_type = Column(String, nullable=False)
     resource_id = Column(String, index=True, nullable=False)
     custom_field_definition_id = Column(
         String, ForeignKey(CustomFieldDefinition.id), nullable=False

fides/api/models/system_group.py

@@ -0,0 +1,85 @@
+from enum import Enum
+
+from citext import CIText
+from sqlalchemy import ARRAY, Column
+from sqlalchemy import Enum as EnumColumn
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.fides_user import FidesUser
+from fides.api.models.sql_models import System  # type: ignore[attr-defined]
+
+
+class CustomTaxonomyColor(str, Enum):
+    WHITE = "taxonomy_white"
+    RED = "taxonomy_red"
+    ORANGE = "taxonomy_orange"
+    YELLOW = "taxonomy_yellow"
+    GREEN = "taxonomy_green"
+    BLUE = "taxonomy_blue"
+    PURPLE = "taxonomy_purple"
+    SANDSTONE = "sandstone"
+    MINOS = "minos"
+
+
+class SystemGroup(Base):
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "system_group"
+
+    fides_key = Column(
+        String,
+        ForeignKey("taxonomy_element.fides_key", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+        index=True,
+    )
+    label_color = Column(
+        EnumColumn(CustomTaxonomyColor, values_callable=lambda x: [i.value for i in x]),
+        nullable=True,
+    )
+    data_steward_username = Column(
+        CIText,
+        ForeignKey(FidesUser.username, ondelete="SET NULL"),
+        index=True,
+        nullable=True,
+    )
+    data_uses = Column(
+        ARRAY(String),
+        nullable=False,
+        server_default="{}",
+        default=list,
+    )  # a list of `fides_key`s of `DataUse` records
+    data_steward = relationship(
+        "FidesUser", foreign_keys=[data_steward_username], lazy="selectin"
+    )
+
+    systems = relationship(
+        System,
+        secondary="system_group_member",
+        back_populates="system_groups",
+        viewonly=True,
+    )
+
+
+class SystemGroupMember(Base):
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "system_group_member"
+
+    system_group_id = Column(
+        String(255), ForeignKey("system_group.id", ondelete="CASCADE"), index=True
+    )
+    system_id = Column(
+        String(255), ForeignKey("ctl_systems.id", ondelete="CASCADE"), index=True
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "system_group_id",
+            "system_id",
+            name="uq_system_group_member_group_system",
+        ),
+    )

fides/api/models/taxonomy.py

@@ -17,11 +17,19 @@ from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import RelationshipProperty, Session, relationship
 
-from fides.api.common_exceptions import ValidationError
+from fides.api.common_exceptions import KeyOrNameAlreadyExists, ValidationError
 from fides.api.db.base_class import Base
 from fides.api.models.sql_models import FidesBase  # type: ignore[attr-defined]
 
-LEGACY_TAXONOMIES = {"data_categories", "data_uses", "data_subjects"}
+# Legacy Fideslang taxonomy keys
+LEGACY_TAXONOMY_KEYS = {
+    "data_category",
+    "data_use",
+    "data_subject",
+}
+
+# Taxonomies that are managed by Fides (legacy taxonomies and system group)
+MANAGED_TAXONOMY_KEYS = {"data_category", "data_use", "data_subject", "system_group"}
 
 
 class TargetType(str, Enum):
@@ -81,14 +89,23 @@ class Taxonomy(Base, FidesBase):
         """Create a new Taxonomy with proper handling of applies_to."""
         # Disallow creating taxonomies that represent legacy types
         fides_key = data.get("fides_key")
-        if fides_key in LEGACY_TAXONOMIES:
+        # Specifically exclude system_group to allow for creation during tests
+        # We already have unique constraints on fides_key to prevent duplicates on DB level
+        if fides_key in MANAGED_TAXONOMY_KEYS:
             raise ValidationError(
                 f"Cannot create taxonomy '{fides_key}'. This is a taxonomy managed by the system."
             )
-        applies_to = data.pop("applies_to", [])
+        applies_to = data.get("applies_to", [])
+
+        # Create a copy of data without applies_to to avoid modifying the input
+        data_without_applies_to = {
+            key: value for key, value in data.items() if key != "applies_to"
+        }
 
         # Create the taxonomy
-        taxonomy: Taxonomy = super().create(db=db, data=data, check_name=check_name)
+        taxonomy: Taxonomy = super().create(
+            db=db, data=data_without_applies_to, check_name=check_name
+        )
 
         # Reconcile allowed usages if applies_to was provided
         if applies_to:
@@ -98,10 +115,15 @@ class Taxonomy(Base, FidesBase):
 
     def update(self, db: Session, *, data: Dict[str, Any]) -> "Taxonomy":
         """Update a Taxonomy with proper handling of applies_to."""
-        applies_to = data.pop("applies_to", None)
+        applies_to = data.get("applies_to", None)
+
+        # Create a copy of data without applies_to to avoid modifying the input
+        data_without_applies_to = {
+            key: value for key, value in data.items() if key != "applies_to"
+        }
 
         # Update the base fields
-        super().update(db=db, data=data)
+        super().update(db=db, data=data_without_applies_to)
 
         # If applies_to was provided, reconcile allowed usages
         if applies_to is not None:
@@ -154,7 +176,7 @@ class TaxonomyAllowedUsage(Base):
 
     target_type can be either:
     - A generic type: "system", "privacy_declaration", "taxonomy"
-    - A taxonomy key: "data_categories", "data_uses", etc.
+    - A taxonomy key: "data_category", "data_use", etc.
     """
 
     @declared_attr
@@ -181,7 +203,7 @@ class TaxonomyAllowedUsage(Base):
     )
     target_type: Column[str] = Column(
         String, primary_key=True
-    )  # Can be "system", "dataset", OR a taxonomy key like "data_categories"
+    )  # Can be "system", "dataset", OR a taxonomy key like "data_category"
 
 
 class TaxonomyElement(Base, FidesBase):
@@ -232,6 +254,66 @@ class TaxonomyElement(Base, FidesBase):
         remote_side="TaxonomyElement.fides_key",
     )
 
+    @classmethod
+    def create(
+        cls: Type["TaxonomyElement"],
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = True,
+    ) -> "TaxonomyElement":
+        """
+        Override create to skip name uniqueness check for hierarchical taxonomy.
+
+        TaxonomyElement is hierarchical - fides_key provides sufficient uniqueness.
+        Multiple elements can share the same name as long as their keys differ.
+        """
+        # Check if taxonomy element with same fides_key already exists within the same taxonomy_type
+        if "fides_key" in data and data["fides_key"] and "taxonomy_type" in data:
+            existing_by_key = (
+                db.query(cls)
+                .filter(
+                    cls.fides_key == data["fides_key"],
+                    cls.taxonomy_type == data["taxonomy_type"],
+                )
+                .first()
+            )
+            if existing_by_key:
+                raise KeyOrNameAlreadyExists(
+                    f'Taxonomy element with fides_key "{data["fides_key"]}" already exists in taxonomy "{data["taxonomy_type"]}".'
+                )
+        # Always skip name check - fides_key uniqueness is sufficient
+        return super().create(db=db, data=data, check_name=False)
+
+    def update(self, db: Session, *, data: dict[str, Any]) -> "TaxonomyElement":
+        """
+        Override update to check for existing taxonomy elements with the same fides_key
+        within the same taxonomy_type when fides_key is being changed.
+        """
+        # Check if fides_key is being changed and if it conflicts with an existing element
+        if (
+            "fides_key" in data
+            and data["fides_key"]
+            and data["fides_key"] != self.fides_key
+        ):
+            # Ensure taxonomy_type is available (either in data or from self)
+            taxonomy_type = data.get("taxonomy_type") or self.taxonomy_type
+            if taxonomy_type:
+                existing_by_key = (
+                    db.query(TaxonomyElement)
+                    .filter(
+                        TaxonomyElement.fides_key == data["fides_key"],
+                        TaxonomyElement.taxonomy_type == taxonomy_type,
+                        TaxonomyElement.id != self.id,  # Exclude current element
+                    )
+                    .first()
+                )
+                if existing_by_key:
+                    raise KeyOrNameAlreadyExists(
+                        f'Taxonomy element with fides_key "{data["fides_key"]}" already exists in taxonomy "{taxonomy_type}".'
+                    )
+        return super().update(db=db, data=data)  # type: ignore[return-value]
+
 
 class TaxonomyUsage(Base):
     """

fides/api/models/v3/privacy_preferences.py

@@ -0,0 +1,85 @@
+"""SQLAlchemy model for the v3 privacy_preferences table."""
+
+from sqlalchemy import (  # type: ignore[attr-defined]
+    BigInteger,
+    Boolean,
+    Column,
+    DateTime,
+    Identity,
+    Text,
+    text,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy_utils.types.encrypted.encrypted_type import (
+    AesGcmEngine,
+    StringEncryptedType,
+)
+
+from fides.api.cryptography.cryptographic_util import hash_value_with_salt
+from fides.api.cryptography.identity_salt import get_identity_salt
+from fides.api.db.base_class import Base
+from fides.api.schemas.redis_cache import MultiValue
+from fides.config import CONFIG
+
+
+class PrivacyPreferences(Base):
+    """
+    Model for the v3 privacy_preferences table.
+    This is used for the v3 privacy preferences API.
+
+    This table stores privacy preference records with a partitioning scheme
+    based on the is_latest flag, splitting records into current and historic partitions.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "privacy_preferences"
+
+    # Override the default id column from Base (which is a String UUID)
+    # with a BigInteger identity column for this table
+    id = Column(
+        BigInteger,  # type: ignore[arg-type]
+        Identity(start=1, increment=1, always=True),
+        primary_key=True,
+    )
+
+    # Searchable/queryable data stored as JSONB
+    search_data = Column(JSONB, nullable=True)
+
+    # Full record data stored as encrypted text (contains PII)
+    record_data = Column(
+        StringEncryptedType(
+            type_in=Text(),
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+        nullable=True,
+    )
+
+    # Partition key - determines if record goes to _current or _historic partition
+    is_latest = Column(
+        Boolean, nullable=False, server_default=text("false"), primary_key=True
+    )
+
+    # Override base class timestamp columns to match migration
+    created_at = Column(
+        DateTime(timezone=True), nullable=False, server_default=text("now()")
+    )
+    updated_at = Column(DateTime(timezone=True), nullable=True)
+
+    @classmethod
+    def hash_value(
+        cls,
+        value: MultiValue,
+        encoding: str = "UTF-8",
+    ) -> str:
+        """Utility function to hash the value with a generated salt"""
+        salt = get_identity_salt()
+        value_str = str(value)
+        hashed_value = hash_value_with_salt(
+            value_str.encode(encoding),
+            salt.encode(encoding),
+        )
+        return hashed_value

fides/api/models/worker_task.py

@@ -17,6 +17,14 @@ class ExecutionLogStatus(enum.Enum):
     awaiting_processing = "paused"  # "paused" in the database to avoid a migration, but use "awaiting_processing" in the app
     retrying = "retrying"
     skipped = "skipped"
+    polling = "polling"
+
+
+# Statuses that can be resumed
+RESUMABLE_EXECUTION_LOG_STATUSES = [
+    ExecutionLogStatus.pending,
+    ExecutionLogStatus.polling,
+]
 
 
 class WorkerTask:

fides/api/oauth/roles.py

@@ -16,6 +16,7 @@ from fides.common.api.scope_registry import (
     DATA_USE_READ,
     DATASET_READ,
     EVALUATION_READ,
+    HEAP_DUMP_EXEC,
     MASKING_EXEC,
     MASKING_READ,
     MESSAGING_CREATE_OR_UPDATE,
@@ -26,6 +27,7 @@ from fides.common.api.scope_registry import (
     PRIVACY_EXPERIENCE_READ,
     PRIVACY_NOTICE_READ,
     PRIVACY_REQUEST_CALLBACK_RESUME,
+    PRIVACY_REQUEST_CREATE,
     PRIVACY_REQUEST_DELETE,
     PRIVACY_REQUEST_EMAIL_INTEGRATIONS_SEND,
     PRIVACY_REQUEST_MANUAL_STEPS_RESPOND,
@@ -88,6 +90,7 @@ approver_scopes = [
     PRIVACY_REQUEST_UPLOAD_DATA,
     PRIVACY_REQUEST_VIEW_DATA,
     PRIVACY_REQUEST_DELETE,
+    PRIVACY_REQUEST_CREATE,  # allows approvers to create new privacy requests
     USER_READ,  # allows approver to view user management table and update their own password
     PRIVACY_REQUEST_MANUAL_STEPS_REVIEW,  # allows approvers to see all manual steps
 ]
@@ -143,6 +146,7 @@ not_contributor_scopes = [
     PRIVACY_REQUEST_NOTIFICATIONS_CREATE_OR_UPDATE,
     PRIVACY_REQUEST_EMAIL_INTEGRATIONS_SEND,
     USER_PERMISSION_ASSIGN_OWNERS,
+    HEAP_DUMP_EXEC,
 ]
 
 ROLES_TO_SCOPES_MAPPING: Dict[str, List] = {

fides/api/schemas/application_config.py

@@ -53,8 +53,11 @@ class NotificationApplicationConfig(FidesSchema):
 
     @field_validator("notification_service_type", mode="before")
     @classmethod
-    def validate_notification_service_type(cls, value: str) -> Optional[str]:
+    def validate_notification_service_type(cls, value: Optional[str]) -> Optional[str]:
         """Ensure the provided type is a valid value."""
+        if value is None:
+            return None  # Allow None values for disabling messaging
+
         value = value.lower()  # force lowercase for safety
         try:
             MessagingServiceType[value]
@@ -98,6 +101,31 @@ class ConsentConfig(FidesSchema):
     model_config = ConfigDict(extra="forbid")
 
 
+class DuplicateDetectionApplicationConfig(FidesSchema):
+    """
+    API model - configuration settings for duplicate privacy request detection.
+    """
+
+    enabled: Optional[bool] = Field(
+        default=False,
+        description="Whether duplicate detection is enabled. Disabled by default.",
+    )
+
+    time_window_days: Optional[int] = Field(
+        default=365,
+        description="Time window in days for duplicate detection. Default is 1 year.",
+        ge=1,
+        le=3650,  # Max 10 years
+    )
+
+    match_identity_fields: Optional[List[str]] = Field(
+        default=["email"],
+        description="Identity field names to match on for duplicate detection (e.g., 'email', 'phone_number'). Default is email only.",
+    )
+
+    model_config = ConfigDict(extra="forbid")
+
+
 class SecurityApplicationConfig(FidesSchema):
     # only valid URLs should be set as cors_origins
     # for advanced usage of non-URLs, e.g. wildcards (`*`), the related
@@ -126,6 +154,9 @@ class ApplicationConfig(FidesSchema):
     consent: Optional[ConsentConfig] = None
     admin_ui: Optional[AdminUIConfig] = None
     privacy_center: Optional[PrivacyCenterConfig] = None
+    privacy_request_duplicate_detection: Optional[
+        DuplicateDetectionApplicationConfig
+    ] = None
 
     @model_validator(mode="before")
     @classmethod