ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-01T18:54:14+0200",
+ "date": "2025-11-05T17:23:50-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "ed0d3226e0f8af1ecab0ac2d2dabff1ee209680d",
- "version": "2.71.1b1"
+ "full-revisionid": "dd2f04e2b4b8386e585ebaf9cd935173dc333b11",
+ "version": "2.74.0rc1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/alembic.ini

@@ -5,7 +5,11 @@
 script_location = migrations
 
 # template used to generate migration files
-# file_template = %%(rev)s_%%(slug)s
+# Adding the prefix xx_ make all new migrations (after changing the file template to date-based)
+# show up below the existing mirations when files are sorted
+file_template = xx_%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+
+
 
 # sys.path path, will be prepended to sys.path if present.
 # defaults to the current working directory.

fides/api/alembic/migrations/env.py

@@ -3,6 +3,7 @@ from logging.config import fileConfig
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 
+from fides.api.db.database import include_object
 from fides.api.util.logger import setup as setup_fidesapi_logger
 from fides.config import CONFIG
 
@@ -48,6 +49,7 @@ def run_migrations_offline():
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        include_object=include_object,
     )
 
     with context.begin_transaction():
@@ -70,7 +72,11 @@ def run_migrations_online():
     )
 
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_object=include_object,
+        )
 
         with context.begin_transaction():
             context.run_migrations()

fides/api/alembic/migrations/versions/4bfbeff34611_add_polling_status.py

@@ -0,0 +1,68 @@
+"""add polling status
+
+Revision ID: 4bfbeff34611
+Revises: 7db29f9cd77b
+Create Date: 2025-09-20 23:02:45.550170
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4bfbeff34611"
+down_revision = "7db29f9cd77b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TYPE executionlogstatus ADD VALUE 'polling'")
+
+
+def downgrade():
+    # Remove any records that have the 'polling' enum value before dropping it
+    # Fallback to 'paused' (awaiting_processing) for backward compatibility
+
+    op.execute("UPDATE requesttask SET status = 'paused' WHERE status = 'polling'")
+    op.execute("UPDATE executionlog SET status = 'paused' WHERE status = 'polling'")
+    op.execute(
+        "UPDATE digest_task_execution SET status = 'paused' WHERE status = 'polling'"
+    )
+    op.execute("UPDATE monitortask SET status = 'paused' WHERE status = 'polling'")
+    op.execute(
+        "UPDATE monitortaskexecutionlog SET status = 'paused' WHERE status = 'polling'"
+    )
+
+    # Recreate the enum without the 'polling' value
+    op.execute("ALTER TYPE executionlogstatus RENAME TO executionlogstatus_old")
+    op.execute(
+        "CREATE TYPE executionlogstatus AS ENUM ("
+        "'in_processing', 'pending', 'complete', 'error', 'paused', 'retrying', 'skipped'"
+        ")"
+    )
+
+    # Update all tables that use the enum type
+    op.execute(
+        "ALTER TABLE requesttask ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE executionlog ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE digest_task_execution ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE monitortask ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE monitortaskexecutionlog ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+
+    # Drop the old enum type
+    op.execute("DROP TYPE executionlogstatus_old")

fides/api/alembic/migrations/versions/65a1bc82ae09_adds_experience_config_for_delete_.py

@@ -0,0 +1,53 @@
+"""adds experience config for delete cookie based on host domain
+
+Revision ID: 65a1bc82ae09
+Revises: 4bfbeff34611
+Create Date: 2025-10-07 11:57:39.614312
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "65a1bc82ae09"
+down_revision = "4bfbeff34611"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "experienceconfigtemplate",
+        sa.Column(
+            "cookie_deletion_based_on_host_domain",
+            sa.Boolean(),
+            server_default="t",
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfig",
+        sa.Column(
+            "cookie_deletion_based_on_host_domain",
+            sa.Boolean(),
+            server_default="t",
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfighistory",
+        sa.Column("cookie_deletion_based_on_host_domain", sa.Boolean(), nullable=True),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column(
+        "privacyexperienceconfighistory", "cookie_deletion_based_on_host_domain"
+    )
+    op.drop_column("privacyexperienceconfig", "cookie_deletion_based_on_host_domain")
+    op.drop_column("experienceconfigtemplate", "cookie_deletion_based_on_host_domain")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/7db29f9cd77b_create_new_sub_request_table.py

@@ -0,0 +1,95 @@
+"""Create new Sub Request Table
+
+Revision ID: 7db29f9cd77b
+Revises: b97e92b038d2
+Create Date: 2025-09-16 14:00:16.282996
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy_utils.types.encrypted.encrypted_type import (
+    AesGcmEngine,
+    StringEncryptedType,
+)
+
+from fides.api.db.base_class import JSONTypeOverride
+from fides.config import CONFIG
+
+# revision identifiers, used by Alembic.
+revision = "7db29f9cd77b"
+down_revision = "b97e92b038d2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "request_task_sub_request",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("request_task_id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "param_values",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=False,
+        ),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column(
+            "access_data",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=True,
+        ),
+        sa.Column("rows_masked", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["request_task_id"],
+            ["requesttask.id"],
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_id"),
+        "request_task_sub_request",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        "request_task_sub_request",
+        ["request_task_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        table_name="request_task_sub_request",
+    )
+    op.drop_index(
+        op.f("ix_request_task_sub_request_id"), table_name="request_task_sub_request"
+    )
+    op.drop_table("request_task_sub_request")

fides/api/alembic/migrations/versions/a55e12c2c2df_add_tagging_instructions_to_data_.py

@@ -0,0 +1,29 @@
+"""add_tagging_instructions_to_data_category
+
+Revision ID: a55e12c2c2df
+Revises: 65a1bc82ae09
+Create Date: 2025-10-13 21:14:00.723464
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a55e12c2c2df"
+down_revision = "65a1bc82ae09"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Add tagging_instructions column to ctl_data_categories table."""
+    op.add_column(
+        "ctl_data_categories",
+        sa.Column("tagging_instructions", sa.Text(), nullable=True),
+    )
+
+
+def downgrade():
+    """Remove tagging_instructions column from ctl_data_categories table."""
+    op.drop_column("ctl_data_categories", "tagging_instructions")

fides/api/alembic/migrations/versions/b97e92b038d2_add_digest_execution_model.py

@@ -0,0 +1,117 @@
+"""add digest execution model
+
+Revision ID: b97e92b038d2
+Revises: 3efe14d4469a
+Create Date: 2025-10-01 16:42:41.900651
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b97e92b038d2"
+down_revision = "3efe14d4469a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "digest_task_execution",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("action_type", sa.String(), nullable=False),
+        sa.Column("digest_config_id", sa.String(), nullable=False),
+        sa.Column("celery_task_id", sa.String(), nullable=True),
+        sa.Column(
+            "status",
+            postgresql.ENUM(name="executionlogstatus", create_type=False),
+            nullable=False,
+        ),
+        sa.Column("total_recipients", sa.Integer(), nullable=True),
+        sa.Column("processed_recipients", sa.Integer(), nullable=False),
+        sa.Column("successful_communications", sa.Integer(), nullable=False),
+        sa.Column("failed_communications", sa.Integer(), nullable=False),
+        sa.Column(
+            "execution_state", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column(
+            "processed_user_ids", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_checkpoint_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["digest_config_id"], ["digest_config.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_action_type"),
+        "digest_task_execution",
+        ["action_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        "digest_task_execution",
+        ["celery_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        "digest_task_execution",
+        ["digest_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_id"),
+        "digest_task_execution",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_status"),
+        "digest_task_execution",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_digest_task_execution_status"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_id"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_action_type"), table_name="digest_task_execution"
+    )
+    op.drop_table("digest_task_execution")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/c09e76282dd1_add_privacy_request_duplication_cols.py

@@ -0,0 +1,64 @@
+"""add privacy request duplication cols
+
+Revision ID: c09e76282dd1
+Revises: 67f0f2f4748e
+Create Date: 2025-10-21 21:46:38.006328
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c09e76282dd1"
+down_revision = "67f0f2f4748e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Add duplicate group tracking column
+    op.add_column(
+        "privacyrequest",
+        sa.Column("duplicate_request_group_id", sa.String(), nullable=True),
+    )
+    op.create_index(
+        op.f("ix_privacyrequest_duplicate_request_group_id"),
+        "privacyrequest",
+        ["duplicate_request_group_id"],
+        unique=False,
+    )
+
+    # Add 'duplicate' status to enum
+    op.execute("alter type privacyrequeststatus add value 'duplicate'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Drop group tracking column and index
+    op.drop_index(
+        op.f("ix_privacyrequest_duplicate_request_group_id"),
+        table_name="privacyrequest",
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+
+    # Remove 'duplicate' status from enum
+    # First delete any requests with duplicate status
+    op.execute("UPDATE privacyrequest SET status = 'error' WHERE status = 'duplicate'")
+
+    # Recreate enum without 'duplicate'
+    op.execute("alter type privacyrequeststatus rename to privacyrequeststatus_old")
+    op.execute(
+        "create type privacyrequeststatus as enum("
+        "'identity_unverified', 'requires_input', 'pending', 'approved', 'denied', "
+        "'in_processing', 'complete', 'paused', 'awaiting_email_send', "
+        "'requires_manual_finalization', 'canceled', 'error')"
+    )
+    op.execute(
+        "alter table privacyrequest alter column status type privacyrequeststatus "
+        "using status::text::privacyrequeststatus"
+    )
+    op.execute("drop type privacyrequeststatus_old")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/xx_2025_10_17_1603_5093e92e2a5a_add_consent_data_v3_to_the_database.py

@@ -0,0 +1,72 @@
+"""Add consent data v3 to the database
+
+Revision ID: 5093e92e2a5a
+Revises: a55e12c2c2df
+Create Date: 2025-10-17 16:03:37.933367
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import BigInteger, Identity
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "5093e92e2a5a"
+down_revision = "a55e12c2c2df"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "privacy_preferences",
+        # Use a big integer for the primary key to ensure we have enough space for all the records
+        # and also force the ID to be generated, never allowing it to be overridden without expressly using `OVERRIDING SYSTEM VALUE`
+        sa.Column(
+            "id",
+            BigInteger,
+            Identity(start=1, increment=1, always=True),
+            primary_key=True,
+        ),
+        sa.Column("search_data", postgresql.JSONB),
+        sa.Column("record_data", postgresql.TEXT),
+        # If we have a primary key we need to use a composite key because the primary key must be unique across _all_
+        # partitions, so we can't just use `id`, we have to use `is_latest` as well as part of the primary key. This
+        # should not be problematic as we will not allow for manually providing the ID so (unless someone deliberately does it)
+        # we should not end up with duplicate IDs having different `is_latest` values.
+        sa.Column(
+            "is_latest",
+            postgresql.BOOLEAN,
+            nullable=False,
+            server_default="f",
+            primary_key=True,
+        ),
+        sa.Column(
+            "created_at",
+            postgresql.TIMESTAMP(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        postgresql_partition_by="LIST (is_latest)",
+    )
+    # Create our partition tables using _current and _historic as suffixes
+    op.execute(
+        """
+            CREATE TABLE privacy_preferences_current
+            PARTITION OF privacy_preferences
+            FOR VALUES IN (true);
+            """
+    )
+    op.execute(
+        """
+            CREATE TABLE privacy_preferences_historic
+            PARTITION OF privacy_preferences
+            FOR VALUES IN (false);
+            """
+    )
+
+
+def downgrade():
+    op.drop_table("privacy_preferences")

fides/api/alembic/migrations/versions/xx_2025_10_27_1834_67f0f2f4748e_adding_identity_definition_model.py

@@ -0,0 +1,45 @@
+"""adding identity definition model
+
+Revision ID: 67f0f2f4748e
+Revises: 5093e92e2a5a
+Create Date: 2025-10-27 18:34:38.326608
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "67f0f2f4748e"
+down_revision = "5093e92e2a5a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "identity_definition",
+        sa.Column("id", sa.String(length=255), nullable=False, unique=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("identity_key", sa.String(length=255), nullable=False),
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("type", sa.String(length=255), nullable=False),
+        sa.Column("created_by", sa.String(length=255), nullable=True),
+        sa.PrimaryKeyConstraint("identity_key"),
+    )
+
+
+def downgrade():
+    op.drop_table("identity_definition")

fides/api/alembic/migrations/versions/xx_2025_10_29_1659_80d28dea3b6b_added_duplicate_group_table.py

@@ -0,0 +1,79 @@
+"""added duplicate group table
+
+Revision ID: 80d28dea3b6b
+Revises: c09e76282dd1
+Create Date: 2025-10-29 16:59:09.551676
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "80d28dea3b6b"
+down_revision = "c09e76282dd1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "duplicate_group",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("rule_version", sa.Text(), nullable=False),
+        sa.Column("is_active", sa.Boolean(), server_default="t", nullable=False),
+        sa.Column("dedup_key", sa.Text(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.drop_index(
+        "ix_privacyrequest_duplicate_request_group_id", table_name="privacyrequest"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column(
+            "duplicate_request_group_id", postgresql.UUID(as_uuid=True), nullable=True
+        ),
+    )
+    op.create_foreign_key(
+        "fk_privacyrequest_duplicate_group",
+        "privacyrequest",
+        "duplicate_group",
+        ["duplicate_request_group_id"],
+        ["id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "fk_privacyrequest_duplicate_group", "privacyrequest", type_="foreignkey"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column("duplicate_request_group_id", sa.String(), nullable=True),
+    )
+    op.create_index(
+        "ix_privacyrequest_duplicate_request_group_id",
+        "privacyrequest",
+        ["duplicate_request_group_id"],
+        unique=False,
+    )
+    op.drop_table("duplicate_group")
+    # ### end Alembic commands ###