ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/b97e92b038d2_add_digest_execution_model.py

@@ -0,0 +1,117 @@
+"""add digest execution model
+
+Revision ID: b97e92b038d2
+Revises: 3efe14d4469a
+Create Date: 2025-10-01 16:42:41.900651
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b97e92b038d2"
+down_revision = "3efe14d4469a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "digest_task_execution",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("action_type", sa.String(), nullable=False),
+        sa.Column("digest_config_id", sa.String(), nullable=False),
+        sa.Column("celery_task_id", sa.String(), nullable=True),
+        sa.Column(
+            "status",
+            postgresql.ENUM(name="executionlogstatus", create_type=False),
+            nullable=False,
+        ),
+        sa.Column("total_recipients", sa.Integer(), nullable=True),
+        sa.Column("processed_recipients", sa.Integer(), nullable=False),
+        sa.Column("successful_communications", sa.Integer(), nullable=False),
+        sa.Column("failed_communications", sa.Integer(), nullable=False),
+        sa.Column(
+            "execution_state", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column(
+            "processed_user_ids", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_checkpoint_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["digest_config_id"], ["digest_config.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_action_type"),
+        "digest_task_execution",
+        ["action_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        "digest_task_execution",
+        ["celery_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        "digest_task_execution",
+        ["digest_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_id"),
+        "digest_task_execution",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_status"),
+        "digest_task_execution",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_digest_task_execution_status"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_id"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_action_type"), table_name="digest_task_execution"
+    )
+    op.drop_table("digest_task_execution")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/b9bfa12c167b_add_not_applicable_to_asset_consentstatus.py

@@ -0,0 +1,46 @@
+"""Add not_applicable to asset ConsentStatus
+
+Revision ID: b9bfa12c167b
+Revises: 30369bb8ae01
+Create Date: 2025-09-09 16:44:23.146325
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b9bfa12c167b"
+down_revision = "30369bb8ae01"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("ALTER TYPE consentstatus ADD VALUE 'not_applicable'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Remove any records that have the new enum value before dropping it
+    op.execute(
+        "UPDATE asset SET consent_status = 'unknown' WHERE consent_status = 'not_applicable'"
+    )
+
+    # Recreate the enum without the new value
+    op.execute("ALTER TYPE consentstatus RENAME TO consentstatus_old")
+    op.execute(
+        "CREATE TYPE consentstatus AS ENUM ('cmp_error', 'pre_consent', 'with_consent', 'without_consent', 'exempt', 'unknown')"
+    )
+
+    # Drop default, change type, and set default back
+    op.execute("ALTER TABLE asset ALTER COLUMN consent_status DROP DEFAULT")
+    op.execute(
+        "ALTER TABLE asset ALTER COLUMN consent_status TYPE consentstatus USING "
+        "consent_status::text::consentstatus"
+    )
+    op.execute("ALTER TABLE asset ALTER COLUMN consent_status SET DEFAULT 'unknown'")
+    op.execute("DROP TYPE consentstatus_old")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/c09e76282dd1_add_privacy_request_duplication_cols.py

@@ -0,0 +1,64 @@
+"""add privacy request duplication cols
+
+Revision ID: c09e76282dd1
+Revises: 67f0f2f4748e
+Create Date: 2025-10-21 21:46:38.006328
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c09e76282dd1"
+down_revision = "67f0f2f4748e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Add duplicate group tracking column
+    op.add_column(
+        "privacyrequest",
+        sa.Column("duplicate_request_group_id", sa.String(), nullable=True),
+    )
+    op.create_index(
+        op.f("ix_privacyrequest_duplicate_request_group_id"),
+        "privacyrequest",
+        ["duplicate_request_group_id"],
+        unique=False,
+    )
+
+    # Add 'duplicate' status to enum
+    op.execute("alter type privacyrequeststatus add value 'duplicate'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Drop group tracking column and index
+    op.drop_index(
+        op.f("ix_privacyrequest_duplicate_request_group_id"),
+        table_name="privacyrequest",
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+
+    # Remove 'duplicate' status from enum
+    # First delete any requests with duplicate status
+    op.execute("UPDATE privacyrequest SET status = 'error' WHERE status = 'duplicate'")
+
+    # Recreate enum without 'duplicate'
+    op.execute("alter type privacyrequeststatus rename to privacyrequeststatus_old")
+    op.execute(
+        "create type privacyrequeststatus as enum("
+        "'identity_unverified', 'requires_input', 'pending', 'approved', 'denied', "
+        "'in_processing', 'complete', 'paused', 'awaiting_email_send', "
+        "'requires_manual_finalization', 'canceled', 'error')"
+    )
+    op.execute(
+        "alter table privacyrequest alter column status type privacyrequeststatus "
+        "using status::text::privacyrequeststatus"
+    )
+    op.execute("drop type privacyrequeststatus_old")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/cd8649be3a2b_add_classifications_and_user_assigned_.py

@@ -0,0 +1,74 @@
+"""add classifications and user_assigned_data_categories indices
+
+Revision ID: cd8649be3a2b
+Revises: a8e0c016afd
+Create Date: 2025-09-19 13:49:02.762672
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "cd8649be3a2b"
+down_revision = "a8e0c016afd"
+branch_labels = None
+depends_on = None
+
+"""
+WARNING - Conditional migration based on table size
+"""
+
+
+def upgrade():
+    # Check stagedresource table size
+    connection = op.get_bind()
+
+    # Check stagedresource table size to decide if we should create indexes immediately
+    table_size = connection.execute(
+        sa.text("SELECT COUNT(*) FROM stagedresource")
+    ).scalar()
+
+    if table_size < 1000000:
+        logger.info(
+            f"stagedresource table has {table_size} rows, creating GIN indices directly"
+        )
+
+        # Create GIN index for user_assigned_data_categories array operations (&&, @>, <@)
+        op.execute(
+            "CREATE INDEX idx_stagedresource_user_categories_gin "
+            "ON stagedresource USING GIN (user_assigned_data_categories)"
+        )
+
+        # Create GIN index for classifications array operations (&&, @>, <@)
+        op.execute(
+            "CREATE INDEX idx_stagedresource_classifications_gin "
+            "ON stagedresource USING GIN (classifications)"
+        )
+
+        logger.info("GIN indices created successfully")
+
+    else:
+        logger.warning(
+            "The stagedresource table has more than 1 million rows, "
+            "skipping index creation. Indexes will be created during application startup "
+            "via post_upgrade_index_creation.py"
+        )
+
+
+def downgrade():
+    # Drop indices in reverse order
+    logger.info("Dropping GIN indices for stagedresource arrays")
+
+    try:
+        op.drop_index("idx_stagedresource_classifications_gin", "stagedresource")
+        logger.info("Dropped classifications GIN index")
+    except Exception as e:
+        logger.warning(f"Could not drop classifications index: {e}")
+
+    try:
+        op.drop_index("idx_stagedresource_user_categories_gin", "stagedresource")
+        logger.info("Dropped user_assigned_data_categories GIN index")
+    except Exception as e:
+        logger.warning(f"Could not drop user_categories index: {e}")

fides/api/alembic/migrations/versions/e2cda8d6abc3_add_dismissed_in_activity_view_to_.py

@@ -0,0 +1,29 @@
+"""add_dismissed_to_monitor_task
+
+Revision ID: e2cda8d6abc3
+Revises: 0eef0016cf06
+Create Date: 2025-09-17 23:37:49.678071
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "e2cda8d6abc3"
+down_revision = "0eef0016cf06"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add the dismissed column to monitortask table
+    op.add_column(
+        "monitortask",
+        sa.Column("dismissed", sa.Boolean(), nullable=False, server_default="f"),
+    )
+
+
+def downgrade():
+    # Remove the dismissed column from monitortask table
+    op.drop_column("monitortask", "dismissed")

fides/api/alembic/migrations/versions/f108fa05c579_adds_optional_duration_field_to_assets.py

@@ -0,0 +1,28 @@
+"""adds optional duration field to assets
+
+Revision ID: f108fa05c579
+Revises: 9caf76161e55
+Create Date: 2025-08-22 12:47:31.374493
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f108fa05c579"
+down_revision = "9caf76161e55"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("asset", sa.Column("duration", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("asset", "duration")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/f36ce1bde293_add_system_groups.py

@@ -0,0 +1,132 @@
+"""add system groups
+
+Revision ID: f36ce1bde293
+Revises: 78dbe23d8204
+Create Date: 2025-08-18 21:36:19.279969
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from citext import CIText
+
+# revision identifiers, used by Alembic.
+revision = "f36ce1bde293"
+down_revision = "78dbe23d8204"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "system_group",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("fides_key", sa.String(), nullable=False),
+        sa.Column(
+            "label_color",
+            sa.Enum(
+                "taxonomy_white",
+                "taxonomy_red",
+                "taxonomy_orange",
+                "taxonomy_yellow",
+                "taxonomy_green",
+                "taxonomy_blue",
+                "taxonomy_purple",
+                "sandstone",
+                "minos",
+                name="customtaxonomycolor",
+            ),
+            nullable=True,
+        ),
+        sa.Column("data_steward_username", CIText(), nullable=True),
+        sa.Column(
+            "data_uses", sa.ARRAY(sa.String()), server_default="{}", nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["data_steward_username"], ["fidesuser.username"], ondelete="SET NULL"
+        ),
+        sa.ForeignKeyConstraint(
+            ["fides_key"], ["taxonomy_element.fides_key"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_system_group_data_steward_username"),
+        "system_group",
+        ["data_steward_username"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_system_group_fides_key"), "system_group", ["fides_key"], unique=True
+    )
+    op.create_index(op.f("ix_system_group_id"), "system_group", ["id"], unique=False)
+    op.create_table(
+        "system_group_member",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("system_group_id", sa.String(length=255), nullable=True),
+        sa.Column("system_id", sa.String(length=255), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["system_group_id"], ["system_group.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(["system_id"], ["ctl_systems.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "system_group_id", "system_id", name="uq_system_group_member_group_system"
+        ),
+    )
+    op.create_index(
+        op.f("ix_system_group_member_id"), "system_group_member", ["id"], unique=False
+    )
+    op.create_index(
+        op.f("ix_system_group_member_system_group_id"),
+        "system_group_member",
+        ["system_group_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_system_group_member_system_id"),
+        "system_group_member",
+        ["system_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_system_group_member_system_id"), table_name="system_group_member"
+    )
+    op.drop_index(
+        op.f("ix_system_group_member_system_group_id"), table_name="system_group_member"
+    )
+    op.drop_index(op.f("ix_system_group_member_id"), table_name="system_group_member")
+    op.drop_table("system_group_member")
+    op.drop_index(op.f("ix_system_group_id"), table_name="system_group")
+    op.drop_index(op.f("ix_system_group_fides_key"), table_name="system_group")
+    op.drop_index(
+        op.f("ix_system_group_data_steward_username"), table_name="system_group"
+    )
+    op.drop_table("system_group")

fides/api/alembic/migrations/versions/fd7571bef683_adds_new_fields_to_messagingconfig_for_.py

@@ -0,0 +1,31 @@
+"""Adds new fields to MessagingConfig for test info
+
+Revision ID: fd7571bef683
+Revises: b9bfa12c167b
+Create Date: 2024-10-09 23:08:01.901592
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "fd7571bef683"
+down_revision = "b9bfa12c167b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "messagingconfig",
+        sa.Column("last_test_timestamp", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column(
+        "messagingconfig", sa.Column("last_test_succeeded", sa.Boolean(), nullable=True)
+    )
+
+
+def downgrade():
+    op.drop_column("messagingconfig", "last_test_succeeded")
+    op.drop_column("messagingconfig", "last_test_timestamp")

fides/api/alembic/migrations/versions/xx_2025_10_17_1603_5093e92e2a5a_add_consent_data_v3_to_the_database.py

@@ -0,0 +1,72 @@
+"""Add consent data v3 to the database
+
+Revision ID: 5093e92e2a5a
+Revises: a55e12c2c2df
+Create Date: 2025-10-17 16:03:37.933367
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import BigInteger, Identity
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "5093e92e2a5a"
+down_revision = "a55e12c2c2df"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "privacy_preferences",
+        # Use a big integer for the primary key to ensure we have enough space for all the records
+        # and also force the ID to be generated, never allowing it to be overridden without expressly using `OVERRIDING SYSTEM VALUE`
+        sa.Column(
+            "id",
+            BigInteger,
+            Identity(start=1, increment=1, always=True),
+            primary_key=True,
+        ),
+        sa.Column("search_data", postgresql.JSONB),
+        sa.Column("record_data", postgresql.TEXT),
+        # If we have a primary key we need to use a composite key because the primary key must be unique across _all_
+        # partitions, so we can't just use `id`, we have to use `is_latest` as well as part of the primary key. This
+        # should not be problematic as we will not allow for manually providing the ID so (unless someone deliberately does it)
+        # we should not end up with duplicate IDs having different `is_latest` values.
+        sa.Column(
+            "is_latest",
+            postgresql.BOOLEAN,
+            nullable=False,
+            server_default="f",
+            primary_key=True,
+        ),
+        sa.Column(
+            "created_at",
+            postgresql.TIMESTAMP(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        postgresql_partition_by="LIST (is_latest)",
+    )
+    # Create our partition tables using _current and _historic as suffixes
+    op.execute(
+        """
+            CREATE TABLE privacy_preferences_current
+            PARTITION OF privacy_preferences
+            FOR VALUES IN (true);
+            """
+    )
+    op.execute(
+        """
+            CREATE TABLE privacy_preferences_historic
+            PARTITION OF privacy_preferences
+            FOR VALUES IN (false);
+            """
+    )
+
+
+def downgrade():
+    op.drop_table("privacy_preferences")

fides/api/alembic/migrations/versions/xx_2025_10_27_1834_67f0f2f4748e_adding_identity_definition_model.py

@@ -0,0 +1,45 @@
+"""adding identity definition model
+
+Revision ID: 67f0f2f4748e
+Revises: 5093e92e2a5a
+Create Date: 2025-10-27 18:34:38.326608
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "67f0f2f4748e"
+down_revision = "5093e92e2a5a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "identity_definition",
+        sa.Column("id", sa.String(length=255), nullable=False, unique=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("identity_key", sa.String(length=255), nullable=False),
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("type", sa.String(length=255), nullable=False),
+        sa.Column("created_by", sa.String(length=255), nullable=True),
+        sa.PrimaryKeyConstraint("identity_key"),
+    )
+
+
+def downgrade():
+    op.drop_table("identity_definition")