ethyca-fides 2.73.0rc4__py2.py3-none-any.whl → 2.73.1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-24T16:42:20-0700",
+ "date": "2025-10-30T16:26:15-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "ca872f4fade7f59c7c49c558ecb610e1f0b41128",
- "version": "2.73.0rc4"
+ "full-revisionid": "34dad3dac5891167e4a1b6cd743199971220da57",
+ "version": "2.73.1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/alembic.ini

@@ -5,7 +5,11 @@
 script_location = migrations
 
 # template used to generate migration files
-# file_template = %%(rev)s_%%(slug)s
+# Adding the prefix xx_ make all new migrations (after changing the file template to date-based)
+# show up below the existing mirations when files are sorted
+file_template = xx_%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+
+
 
 # sys.path path, will be prepended to sys.path if present.
 # defaults to the current working directory.

fides/api/alembic/migrations/env.py

@@ -3,6 +3,7 @@ from logging.config import fileConfig
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 
+from fides.api.db.database import include_object
 from fides.api.util.logger import setup as setup_fidesapi_logger
 from fides.config import CONFIG
 
@@ -48,6 +49,7 @@ def run_migrations_offline():
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        include_object=include_object,
     )
 
     with context.begin_transaction():
@@ -70,7 +72,11 @@ def run_migrations_online():
     )
 
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_object=include_object,
+        )
 
         with context.begin_transaction():
             context.run_migrations()

fides/api/alembic/migrations/versions/c09e76282dd1_add_privacy_request_duplication_cols.py

@@ -0,0 +1,64 @@
+"""add privacy request duplication cols
+
+Revision ID: c09e76282dd1
+Revises: 67f0f2f4748e
+Create Date: 2025-10-21 21:46:38.006328
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c09e76282dd1"
+down_revision = "67f0f2f4748e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Add duplicate group tracking column
+    op.add_column(
+        "privacyrequest",
+        sa.Column("duplicate_request_group_id", sa.String(), nullable=True),
+    )
+    op.create_index(
+        op.f("ix_privacyrequest_duplicate_request_group_id"),
+        "privacyrequest",
+        ["duplicate_request_group_id"],
+        unique=False,
+    )
+
+    # Add 'duplicate' status to enum
+    op.execute("alter type privacyrequeststatus add value 'duplicate'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Drop group tracking column and index
+    op.drop_index(
+        op.f("ix_privacyrequest_duplicate_request_group_id"),
+        table_name="privacyrequest",
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+
+    # Remove 'duplicate' status from enum
+    # First delete any requests with duplicate status
+    op.execute("UPDATE privacyrequest SET status = 'error' WHERE status = 'duplicate'")
+
+    # Recreate enum without 'duplicate'
+    op.execute("alter type privacyrequeststatus rename to privacyrequeststatus_old")
+    op.execute(
+        "create type privacyrequeststatus as enum("
+        "'identity_unverified', 'requires_input', 'pending', 'approved', 'denied', "
+        "'in_processing', 'complete', 'paused', 'awaiting_email_send', "
+        "'requires_manual_finalization', 'canceled', 'error')"
+    )
+    op.execute(
+        "alter table privacyrequest alter column status type privacyrequeststatus "
+        "using status::text::privacyrequeststatus"
+    )
+    op.execute("drop type privacyrequeststatus_old")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/xx_2025_10_17_1603_5093e92e2a5a_add_consent_data_v3_to_the_database.py

@@ -0,0 +1,72 @@
+"""Add consent data v3 to the database
+
+Revision ID: 5093e92e2a5a
+Revises: a55e12c2c2df
+Create Date: 2025-10-17 16:03:37.933367
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import BigInteger, Identity
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "5093e92e2a5a"
+down_revision = "a55e12c2c2df"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "privacy_preferences",
+        # Use a big integer for the primary key to ensure we have enough space for all the records
+        # and also force the ID to be generated, never allowing it to be overridden without expressly using `OVERRIDING SYSTEM VALUE`
+        sa.Column(
+            "id",
+            BigInteger,
+            Identity(start=1, increment=1, always=True),
+            primary_key=True,
+        ),
+        sa.Column("search_data", postgresql.JSONB),
+        sa.Column("record_data", postgresql.TEXT),
+        # If we have a primary key we need to use a composite key because the primary key must be unique across _all_
+        # partitions, so we can't just use `id`, we have to use `is_latest` as well as part of the primary key. This
+        # should not be problematic as we will not allow for manually providing the ID so (unless someone deliberately does it)
+        # we should not end up with duplicate IDs having different `is_latest` values.
+        sa.Column(
+            "is_latest",
+            postgresql.BOOLEAN,
+            nullable=False,
+            server_default="f",
+            primary_key=True,
+        ),
+        sa.Column(
+            "created_at",
+            postgresql.TIMESTAMP(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        postgresql_partition_by="LIST (is_latest)",
+    )
+    # Create our partition tables using _current and _historic as suffixes
+    op.execute(
+        """
+            CREATE TABLE privacy_preferences_current
+            PARTITION OF privacy_preferences
+            FOR VALUES IN (true);
+            """
+    )
+    op.execute(
+        """
+            CREATE TABLE privacy_preferences_historic
+            PARTITION OF privacy_preferences
+            FOR VALUES IN (false);
+            """
+    )
+
+
+def downgrade():
+    op.drop_table("privacy_preferences")

fides/api/alembic/migrations/versions/xx_2025_10_27_1834_67f0f2f4748e_adding_identity_definition_model.py

@@ -0,0 +1,45 @@
+"""adding identity definition model
+
+Revision ID: 67f0f2f4748e
+Revises: 5093e92e2a5a
+Create Date: 2025-10-27 18:34:38.326608
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "67f0f2f4748e"
+down_revision = "5093e92e2a5a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "identity_definition",
+        sa.Column("id", sa.String(length=255), nullable=False, unique=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("identity_key", sa.String(length=255), nullable=False),
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("type", sa.String(length=255), nullable=False),
+        sa.Column("created_by", sa.String(length=255), nullable=True),
+        sa.PrimaryKeyConstraint("identity_key"),
+    )
+
+
+def downgrade():
+    op.drop_table("identity_definition")

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -144,6 +144,7 @@ from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_APPROVE,
     PRIVACY_REQUEST_AUTHENTICATED,
     PRIVACY_REQUEST_BATCH_EMAIL_SEND,
+    PRIVACY_REQUEST_BULK_FINALIZE,
     PRIVACY_REQUEST_BULK_RETRY,
     PRIVACY_REQUEST_BULK_SOFT_DELETE,
     PRIVACY_REQUEST_DENY,
@@ -1971,6 +1972,32 @@ def resume_privacy_request_from_requires_input(
     return privacy_request  # type: ignore[return-value]
 
 
+@router.post(
+    PRIVACY_REQUEST_BULK_FINALIZE,
+    status_code=HTTP_200_OK,
+    response_model=BulkReviewResponse,
+)
+def bulk_finalize_privacy_requests(
+    *,
+    client: ClientDetail = Security(
+        verify_oauth_client,
+        scopes=[PRIVACY_REQUEST_REVIEW],
+    ),
+    privacy_request_service: PrivacyRequestService = Depends(
+        get_privacy_request_service
+    ),
+    privacy_requests: ReviewPrivacyRequestIds,
+) -> BulkReviewResponse:
+    """
+    Bulk finalize privacy requests that are in requires_manual_finalization status.
+    Each request will be moved from the 'requires_finalization' state to 'complete'.
+    Returns an object with the list of successfully finalized privacy requests and the list of failed finalizations.
+    """
+    return privacy_request_service.finalize_privacy_requests(
+        privacy_requests.request_ids, user_id=client.user_id
+    )
+
+
 @router.post(
     PRIVACY_REQUEST_FINALIZE,
     status_code=HTTP_200_OK,

fides/api/db/base.py

@@ -33,6 +33,7 @@ from fides.api.models.fides_user_permissions import FidesUserPermissions
 from fides.api.models.fides_user_respondent_email_verification import (
     FidesUserRespondentEmailVerification,
 )
+from fides.api.models.identity_definition import IdentityDefinition
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
 from fides.api.models.manual_task import (
@@ -95,3 +96,4 @@ from fides.api.models.tcf_publisher_restrictions import (
     TCFPublisherRestriction,
 )
 from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride
+from fides.api.models.v3.privacy_preferences import PrivacyPreferences

fides/api/db/database.py

@@ -9,7 +9,9 @@ from alembic import command, script
 from alembic.config import Config
 from alembic.runtime import migration
 from loguru import logger as log
+from sqlalchemy import text
 from sqlalchemy.orm import Session
+from sqlalchemy.schema import SchemaItem
 from sqlalchemy_utils.functions import create_database, database_exists
 from sqlalchemy_utils.types.encrypted.encrypted_type import InvalidCiphertextError
 
@@ -20,6 +22,25 @@ from fides.api.util.errors import get_full_exception_name
 
 DatabaseHealth = Literal["healthy", "unhealthy", "needs migration"]
 
+# Tables to exclude from migration auto-generation (e.g., tables without SQLAlchemy models)
+EXCLUDED_TABLES = {
+    "privacy_preferences_current",
+    "privacy_preferences_historic",
+}
+
+
+def include_object(
+    object: SchemaItem,  # pylint: disable=redefined-builtin
+    name: str,
+    type_: str,
+    reflected: bool,
+    compare_to: SchemaItem,
+) -> bool:
+    """Filter out excluded tables from migration comparison and auto-generation."""
+    if type_ == "table" and name in EXCLUDED_TABLES:
+        return False
+    return True
+
 
 def get_alembic_config(database_url: str) -> Config:
     """
@@ -91,6 +112,10 @@ def reset_db(database_url: str) -> None:
         log.info("Dropping tables...")
         Base.metadata.drop_all(connection)
 
+        log.info("Dropping excluded tables without models...")
+        for table_name in EXCLUDED_TABLES:
+            connection.execute(text(f"DROP TABLE IF EXISTS {table_name} CASCADE"))
+
         log.info("Dropping Alembic table...")
         migration_context = migration.MigrationContext.configure(connection)
         version = migration_context._version  # pylint: disable=protected-access
@@ -151,13 +176,18 @@ def check_missing_migrations(database_url: str) -> None:
     Tries to autogenerate migrations, returns True if a migration
     was generated.
     """
-
     engine = get_db_engine(database_uri=database_url)
     connection = engine.connect()
 
-    migration_context = migration.MigrationContext.configure(connection)
+    migration_context = migration.MigrationContext.configure(
+        connection, opts={"include_object": include_object}
+    )
     result = command.autogen.compare_metadata(migration_context, Base.metadata)  # type: ignore[attr-defined]
 
     if result:
+        log.error("Migration needs to be generated!")
+        log.error("Detected the following schema differences:")
+        for item in result:
+            log.error(f"  - {item}")
         raise SystemExit("Migration needs to be generated!")
     print("No migrations need to be generated.")

fides/api/models/identity_definition.py

@@ -0,0 +1,65 @@
+from enum import Enum
+
+from sqlalchemy import Column, String, Text
+from sqlalchemy.ext.declarative import declared_attr
+
+from fides.api.db.base_class import Base, FidesBase
+from fides.api.db.util import EnumColumn
+
+
+class IdentityDefinitionType(str, Enum):
+    """Enum for the type of identity"""
+
+    EMAIL = "email"
+    PHONE_NUMBER = "phone_number"
+    UUID = "uuid"
+    STRING = "string"
+    INTEGER = "integer"
+
+
+class IdentityDefinition(Base):
+    """
+    Model for identity definitions in Fides. This isn't for specific identity values,
+    but for the types of identities that can be used in Fides.
+
+    For example:
+    ```json
+    {
+        "identity_key": "customer_id",
+        "name": "Customer ID",
+        "description": "The unique identifier for the customer",
+        "type": "string"
+    }
+    ```
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "identity_definition"
+
+    # Overriding the id definition from Base so we don't treat this as the primary key
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    # Primary key
+    identity_key = Column(String(255), primary_key=True)
+
+    # Schema definition
+    name = Column(String(255), nullable=False)
+    description = Column(Text, nullable=True)
+    type = Column(
+        EnumColumn(
+            IdentityDefinitionType,
+            native_enum=False,
+            values_callable=lambda x: [
+                i.value for i in x
+            ],  # allows enum _values_ to be stored rather than name
+        ),
+        nullable=False,
+    )
+    created_by = Column(String(255), nullable=True)

fides/api/models/privacy_request/privacy_request.py

@@ -224,6 +224,9 @@ class PrivacyRequest(
     source = Column(EnumColumn(PrivacyRequestSource), nullable=True)
     location = Column(String, nullable=True)
 
+    # Duplicate detection - group ID to link duplicates together
+    duplicate_request_group_id = Column(String, index=True, nullable=True)
+
     # A PrivacyRequest can be soft deleted, so we store when it was deleted
     deleted_at = Column(DateTime(timezone=True), nullable=True)
     # and who deleted it

fides/api/models/v3/privacy_preferences.py

@@ -0,0 +1,85 @@
+"""SQLAlchemy model for the v3 privacy_preferences table."""
+
+from sqlalchemy import (  # type: ignore[attr-defined]
+    BigInteger,
+    Boolean,
+    Column,
+    DateTime,
+    Identity,
+    Text,
+    text,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy_utils.types.encrypted.encrypted_type import (
+    AesGcmEngine,
+    StringEncryptedType,
+)
+
+from fides.api.cryptography.cryptographic_util import hash_value_with_salt
+from fides.api.cryptography.identity_salt import get_identity_salt
+from fides.api.db.base_class import Base
+from fides.api.schemas.redis_cache import MultiValue
+from fides.config import CONFIG
+
+
+class PrivacyPreferences(Base):
+    """
+    Model for the v3 privacy_preferences table.
+    This is used for the v3 privacy preferences API.
+
+    This table stores privacy preference records with a partitioning scheme
+    based on the is_latest flag, splitting records into current and historic partitions.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "privacy_preferences"
+
+    # Override the default id column from Base (which is a String UUID)
+    # with a BigInteger identity column for this table
+    id = Column(
+        BigInteger,  # type: ignore[arg-type]
+        Identity(start=1, increment=1, always=True),
+        primary_key=True,
+    )
+
+    # Searchable/queryable data stored as JSONB
+    search_data = Column(JSONB, nullable=True)
+
+    # Full record data stored as encrypted text (contains PII)
+    record_data = Column(
+        StringEncryptedType(
+            type_in=Text(),
+            key=CONFIG.security.app_encryption_key,
+            engine=AesGcmEngine,
+            padding="pkcs5",
+        ),
+        nullable=True,
+    )
+
+    # Partition key - determines if record goes to _current or _historic partition
+    is_latest = Column(
+        Boolean, nullable=False, server_default=text("false"), primary_key=True
+    )
+
+    # Override base class timestamp columns to match migration
+    created_at = Column(
+        DateTime(timezone=True), nullable=False, server_default=text("now()")
+    )
+    updated_at = Column(DateTime(timezone=True), nullable=True)
+
+    @classmethod
+    def hash_value(
+        cls,
+        value: MultiValue,
+        encoding: str = "UTF-8",
+    ) -> str:
+        """Utility function to hash the value with a generated salt"""
+        salt = get_identity_salt()
+        value_str = str(value)
+        hashed_value = hash_value_with_salt(
+            value_str.encode(encoding),
+            salt.encode(encoding),
+        )
+        return hashed_value

fides/api/schemas/application_config.py

@@ -101,6 +101,31 @@ class ConsentConfig(FidesSchema):
     model_config = ConfigDict(extra="forbid")
 
 
+class DuplicateDetectionApplicationConfig(FidesSchema):
+    """
+    API model - configuration settings for duplicate privacy request detection.
+    """
+
+    enabled: Optional[bool] = Field(
+        default=False,
+        description="Whether duplicate detection is enabled. Disabled by default.",
+    )
+
+    time_window_days: Optional[int] = Field(
+        default=365,
+        description="Time window in days for duplicate detection. Default is 1 year.",
+        ge=1,
+        le=3650,  # Max 10 years
+    )
+
+    match_identity_fields: Optional[List[str]] = Field(
+        default=["email"],
+        description="Identity field names to match on for duplicate detection (e.g., 'email', 'phone_number'). Default is email only.",
+    )
+
+    model_config = ConfigDict(extra="forbid")
+
+
 class SecurityApplicationConfig(FidesSchema):
     # only valid URLs should be set as cors_origins
     # for advanced usage of non-URLs, e.g. wildcards (`*`), the related
@@ -129,6 +154,9 @@ class ApplicationConfig(FidesSchema):
     consent: Optional[ConsentConfig] = None
     admin_ui: Optional[AdminUIConfig] = None
     privacy_center: Optional[PrivacyCenterConfig] = None
+    privacy_request_duplicate_detection: Optional[
+        DuplicateDetectionApplicationConfig
+    ] = None
 
     @model_validator(mode="before")
     @classmethod

fides/api/schemas/privacy_request.py

@@ -306,6 +306,7 @@ class PrivacyRequestStatus(str, EnumType):
     requires_manual_finalization = "requires_manual_finalization"
     canceled = "canceled"
     error = "error"
+    duplicate = "duplicate"  # Request identified as duplicate of another request
 
 
 class PrivacyRequestResponse(FidesSchema):
@@ -342,6 +343,7 @@ class PrivacyRequestResponse(FidesSchema):
     deleted_by: Optional[str] = None
     finalized_at: Optional[datetime] = None
     finalized_by: Optional[str] = None
+    duplicate_request_group_id: Optional[str] = None
 
     model_config = ConfigDict(from_attributes=True, use_enum_values=True)
 

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -314,7 +314,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         collection_values: Dict[str, Row],
         identity_data: Dict[str, Any],
         input_data: Optional[Dict[str, List[Any]]] = None,
-        param_values: Dict[str, Any] = {},
+        param_values: Optional[Dict[str, Any]] = None,
     ) -> None:
         """
         Process param_values for update generation (erasure requests).
@@ -326,6 +326,9 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             identity_data: Identity data for parameter resolution
             input_data: Optional upstream data from other collections
         """
+        if param_values is None:
+            param_values = {}
+
         for param_value in request.param_values or []:
             if param_value.references:
                 # we resolve the param reference here for consistency,

fides/api/service/connectors/saas_connector.py

@@ -787,11 +787,10 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
 
         else:
             # follow the basic (global opt-in/out) SaaS consent flow
-            (
-                should_opt_in,
-                filtered_preferences,
-            ) = build_user_consent_and_filtered_preferences_for_service(
-                self.configuration.system, privacy_request, session, False
+            should_opt_in, filtered_preferences = (
+                build_user_consent_and_filtered_preferences_for_service(
+                    self.configuration.system, privacy_request, session, False
+                )
             )
 
             if should_opt_in is None:

fides/common/api/v1/urn_registry.py

@@ -84,6 +84,7 @@ PRIVACY_REQUEST_BATCH_EMAIL_SEND = (
     "/privacy-request/administrate/process-awaiting-email-send"
 )
 PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated"
+PRIVACY_REQUEST_BULK_FINALIZE = "/privacy-request/bulk/finalize"
 PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry"
 PRIVACY_REQUEST_BULK_SOFT_DELETE = "/privacy-request/bulk/soft-delete"
 PRIVACY_REQUEST_DENY = "/privacy-request/administrate/deny"

fides/config/__init__.py

@@ -24,6 +24,7 @@ from .cli_settings import CLISettings
 from .consent_settings import ConsentSettings
 from .credentials_settings import merge_credentials_environment
 from .database_settings import DatabaseSettings
+from .duplicate_detection_settings import DuplicateDetectionSettings
 from .execution_settings import ExecutionSettings
 from .fides_settings import FidesSettings
 from .helpers import handle_deprecated_env_variables, handle_deprecated_fields
@@ -79,6 +80,7 @@ class FidesConfig(FidesSettings):
         description="This is a special section that is used to store arbitrary key/value pairs to be used as credentials."
     )
     database: DatabaseSettings
+    privacy_request_duplicate_detection: DuplicateDetectionSettings
     execution: ExecutionSettings
     logging: LoggingSettings
     notifications: NotificationSettings
@@ -161,6 +163,7 @@ def build_config(config_dict: Dict[str, Any]) -> FidesConfig:
         "consent": ConsentSettings,
         "cli": CLISettings,
         "database": DatabaseSettings,
+        "privacy_request_duplicate_detection": DuplicateDetectionSettings,
         "execution": ExecutionSettings,
         "logging": LoggingSettings,
         "notifications": NotificationSettings,