ethyca-fides 2.73.2b0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-30T20:40:29+0000",
+ "date": "2025-12-05T12:28:42-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "b17dcbb16dfac3ce994a9bd97fd39d126ec66d9f",
- "version": "2.73.2b0"
+ "full-revisionid": "5d4b1e4aa628b612ac705b8f9cd2b41218cddc59",
+ "version": "2.76.0rc2"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/303287c70600_migrate_resource_type_from_enum_to_.py

@@ -0,0 +1,99 @@
+"""migrate resource type from enum to string to account for custom taxonomies
+
+Revision ID: 303287c70600
+Revises: 3ff6449c099e
+Create Date: 2025-10-22 15:31:12.980243
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "303287c70600"
+down_revision = "3ff6449c099e"
+branch_labels = None
+depends_on = None
+
+# Original enum values used by ResourceTypes
+RESOURCE_TYPES = (
+    "system",
+    "data use",
+    "data category",
+    "data subject",
+    "privacy declaration",
+)
+
+# Name for the PostgreSQL enum type that backs ResourceTypes
+PG_ENUM_NAME = "resourcetypes"
+
+# Mapping from old enum values (with underscores) to new string values (with spaces)
+# The old PostgreSQL enum used underscores but the Python ResourceTypes enum expects spaces
+RESOURCE_TYPE_MAPPING = {
+    "data_use": "data use",
+    "data_category": "data category",
+    "data_subject": "data subject",
+    "privacy_declaration": "privacy declaration",
+}
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Drop unique functional index that references resource_type
+    op.drop_index(
+        "ix_plus_custom_field_definition_unique_lowername_resourcetype",
+        table_name="plus_custom_field_definition",
+    )
+
+    # Alter resource_type columns from Enum -> String
+    # Use a PostgreSQL USING clause to safely cast enum -> text
+    op.alter_column(
+        "plus_custom_field_definition",
+        "resource_type",
+        existing_type=sa.Enum(*RESOURCE_TYPES, name=PG_ENUM_NAME),
+        type_=sa.String(),
+        postgresql_using="resource_type::text",
+        nullable=False,
+    )
+    op.alter_column(
+        "plus_custom_field",
+        "resource_type",
+        existing_type=sa.Enum(*RESOURCE_TYPES, name=PG_ENUM_NAME),
+        type_=sa.String(),
+        postgresql_using="resource_type::text",
+        nullable=False,
+    )
+
+    # Data migration: Convert old enum values (with underscores) to new string values (with spaces)
+    # The old PostgreSQL enum used underscores (e.g., 'privacy_declaration', 'data_use') but the Python
+    # ResourceTypes enum expects spaces (e.g., 'privacy declaration', 'data use')
+    for old_value, new_value in RESOURCE_TYPE_MAPPING.items():
+        for table_name in ["plus_custom_field_definition", "plus_custom_field"]:
+            op.execute(
+                f"""
+                UPDATE {table_name}
+                SET resource_type = '{new_value}'
+                WHERE resource_type = '{old_value}'
+                """
+            )
+
+    # Recreate unique functional index on (resource_type, lower(name))
+    # Note: Use a raw expression for lower(name)
+    op.create_index(
+        "ix_plus_custom_field_definition_unique_lowername_resourcetype",
+        "plus_custom_field_definition",
+        ["resource_type", sa.text("lower(name)")],
+        unique=True,
+    )
+
+    # Drop the unused enum type after all columns have been converted
+    op.execute(f'DROP TYPE IF EXISTS "{PG_ENUM_NAME}"')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    """
+    This migration does not support downgrades.
+    """
+    logger.info("Conversion of str to enum ResourceType is unsupported.")

fides/api/alembic/migrations/versions/795f46f656c0_migrate_field_type_from_enum_to_string.py

@@ -0,0 +1,61 @@
+"""migrate field type from enum to string to support custom taxonomies
+
+Revision ID: 795f46f656c0
+Revises: 303287c70600
+Create Date: 2025-11-18 18:30:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "795f46f656c0"
+down_revision = "303287c70600"
+branch_labels = None
+depends_on = None
+
+# Original enum values used by AllowedTypes
+ALLOWED_TYPES = (
+    "string",
+    "string[]",
+)
+
+# Name for the PostgreSQL enum type that backs AllowedTypes
+PG_ENUM_NAME = "allowedtypes"
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Alter field_type column from Enum -> String
+    # Use a PostgreSQL USING clause to safely cast enum -> text
+    op.alter_column(
+        "plus_custom_field_definition",
+        "field_type",
+        existing_type=sa.Enum(*ALLOWED_TYPES, name=PG_ENUM_NAME),
+        type_=sa.String(),
+        postgresql_using="field_type::text",
+        nullable=False,
+    )
+
+    # Data migration: Convert old enum value 'string_list' to 'string[]'
+    op.execute(
+        """
+        UPDATE plus_custom_field_definition
+        SET field_type = 'string[]'
+        WHERE field_type = 'string_list'
+        """
+    )
+
+    # Drop the unused enum type after the column has been converted
+    op.execute(f'DROP TYPE IF EXISTS "{PG_ENUM_NAME}"')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    """
+    This migration does not support downgrades.
+    Converting string back to enum would fail if taxonomy keys are present.
+    """
+    logger.info("Conversion of str to enum AllowedTypes is unsupported.")

fides/api/alembic/migrations/versions/xx_2025_10_29_1659_80d28dea3b6b_added_duplicate_group_table.py

@@ -0,0 +1,79 @@
+"""added duplicate group table
+
+Revision ID: 80d28dea3b6b
+Revises: c09e76282dd1
+Create Date: 2025-10-29 16:59:09.551676
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "80d28dea3b6b"
+down_revision = "c09e76282dd1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "duplicate_group",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("rule_version", sa.Text(), nullable=False),
+        sa.Column("is_active", sa.Boolean(), server_default="t", nullable=False),
+        sa.Column("dedup_key", sa.Text(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.drop_index(
+        "ix_privacyrequest_duplicate_request_group_id", table_name="privacyrequest"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column(
+            "duplicate_request_group_id", postgresql.UUID(as_uuid=True), nullable=True
+        ),
+    )
+    op.create_foreign_key(
+        "fk_privacyrequest_duplicate_group",
+        "privacyrequest",
+        "duplicate_group",
+        ["duplicate_request_group_id"],
+        ["id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "fk_privacyrequest_duplicate_group", "privacyrequest", type_="foreignkey"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column("duplicate_request_group_id", sa.String(), nullable=True),
+    )
+    op.create_index(
+        "ix_privacyrequest_duplicate_request_group_id",
+        "privacyrequest",
+        ["duplicate_request_group_id"],
+        unique=False,
+    )
+    op.drop_table("duplicate_group")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/xx_2025_11_05_0200_f1a2b3c4d5e6_create_staged_resource_error_table.py

@@ -0,0 +1,82 @@
+"""Create staged resource error table
+
+Revision ID: f1a2b3c4d5e6
+Revises: 80d28dea3b6b
+Create Date: 2025-11-05 02:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f1a2b3c4d5e6"
+down_revision = "80d28dea3b6b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Create the stagedresourceerror table
+    op.create_table(
+        "stagedresourceerror",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "staged_resource_urn",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("message", sa.String(), nullable=False),
+        sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("error_type", sa.String(), nullable=False),
+        sa.Column("diff_status", sa.String(), nullable=False),
+        sa.Column("task_id", sa.String(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["staged_resource_urn"],
+            ["stagedresource.urn"],
+            name="stagedresourceerror_staged_resource_urn_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+    op.create_index(
+        "ix_stagedresourceerror_staged_resource_urn",
+        "stagedresourceerror",
+        ["staged_resource_urn"],
+        unique=False,
+    )
+
+    op.create_index(
+        "ix_stagedresourceerror_task_id",
+        "stagedresourceerror",
+        ["task_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    # Drop the indices
+    op.drop_index(
+        "ix_stagedresourceerror_task_id",
+        table_name="stagedresourceerror",
+    )
+    op.drop_index(
+        "ix_stagedresourceerror_staged_resource_urn",
+        table_name="stagedresourceerror",
+    )
+
+    # Drop the stagedresourceerror table
+    op.drop_table("stagedresourceerror")

fides/api/alembic/migrations/versions/xx_2025_11_07_1709_56fe6fad2d89_add_notice_display_order.py

@@ -0,0 +1,47 @@
+"""add_notice_display_order
+
+Revision ID: 56fe6fad2d89
+Revises: b2c3d4e5f6a7
+Create Date: 2025-11-07 17:09:24.917188
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "56fe6fad2d89"
+down_revision = "b2c3d4e5f6a7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """
+    Add display_order column to experiencenotices and privacynotice tables.
+
+    This allows:
+    1. Experiences to control the order in which notices are displayed in the API response
+    2. Parent notices to control the order in which child notices are displayed
+
+    Both enable drag-and-drop reordering in the Admin UI.
+    """
+    # Add display_order to experiencenotices for ordering notices within an experience
+    op.add_column(
+        "experiencenotices",
+        sa.Column("display_order", sa.Integer(), nullable=True),
+    )
+
+    # Add display_order to privacynotice for ordering child notices within a parent
+    op.add_column(
+        "privacynotice",
+        sa.Column("display_order", sa.Integer(), nullable=True),
+    )
+
+
+def downgrade():
+    """
+    Remove display_order column from experiencenotices and privacynotice tables.
+    """
+    op.drop_column("experiencenotices", "display_order")
+    op.drop_column("privacynotice", "display_order")

fides/api/alembic/migrations/versions/xx_2025_11_10_1200_a1b2c3d4e5f6_add_test_datastore_to_connectiontype.py

@@ -0,0 +1,53 @@
+"""Add test_datastore to ConnectionType
+
+Revision ID: a1b2c3d4e5f6
+Revises: f1a2b3c4d5e6
+Create Date: 2025-11-10 12:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a1b2c3d4e5f6"
+down_revision = "f1a2b3c4d5e6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add test_datastore to ConnectionType enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('attentive_email', 'bigquery', 'datahub', 'dynamodb', 'fides', 'generic_consent_email', 'generic_erasure_email', 'dynamic_erasure_email', 'google_cloud_sql_mysql', 'google_cloud_sql_postgres', 'https', 'manual', 'manual_webhook', 'manual_task', 'mariadb', 'mongodb', 'mssql', 'mysql', 'okta', 'postgres', 'rds_mysql', 'rds_postgres', 'redshift', 's3', 'saas', 'scylla', 'snowflake', 'sovrn', 'test_datastore', 'test_website', 'timescale', 'website')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")
+
+
+def downgrade():
+    # First remove the ConnectionConfigs with connection_type = 'test_datastore'
+    op.execute(
+        "delete from connectionconfig where connection_type in ('test_datastore')"
+    )
+    # Then remove 'test_datastore' from ConnectionType enum by renaming old enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    # Create a new enum without 'test_datastore'
+    op.execute(
+        "create type connectiontype as enum('attentive_email', 'bigquery', 'datahub', 'dynamodb', 'fides', 'generic_consent_email', 'generic_erasure_email', 'dynamic_erasure_email', 'google_cloud_sql_mysql', 'google_cloud_sql_postgres', 'https', 'manual', 'manual_webhook', 'manual_task', 'mariadb', 'mongodb', 'mssql', 'mysql', 'okta', 'postgres', 'rds_mysql', 'rds_postgres', 'redshift', 's3', 'saas', 'scylla', 'snowflake', 'sovrn', 'test_website', 'timescale', 'website')"
+    )
+    # Alter the connectionconfig table to use the new enum
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    # Drop the old enum
+    op.execute("drop type connectiontype_old")

fides/api/alembic/migrations/versions/xx_2025_11_11_1317_7d82c8fc4c34_store_saas_template_datasets.py

@@ -0,0 +1,68 @@
+"""Store SaaS Template Datasets
+
+Revision ID: 7d82c8fc4c34
+Revises: a1b2c3d4e5f6
+Create Date: 2025-11-11 13:17:36.602477
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "7d82c8fc4c34"
+down_revision = "a1b2c3d4e5f6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "saas_template_dataset",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("connection_type", sa.String(), nullable=False),
+        sa.Column(
+            "dataset_json", postgresql.JSONB(astext_type=sa.Text()), nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_saas_template_dataset_connection_type"),
+        "saas_template_dataset",
+        ["connection_type"],
+        unique=True,
+    )
+    op.create_index(
+        op.f("ix_saas_template_dataset_id"),
+        "saas_template_dataset",
+        ["id"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_saas_template_dataset_id"), table_name="saas_template_dataset"
+    )
+    op.drop_index(
+        op.f("ix_saas_template_dataset_connection_type"),
+        table_name="saas_template_dataset",
+    )
+    op.drop_table("saas_template_dataset")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/xx_2025_11_12_1430_b2c3d4e5f6a7_add_default_identity_definitions.py

@@ -0,0 +1,81 @@
+"""add default identity definitions
+
+Revision ID: b2c3d4e5f6a7
+Revises: 7d82c8fc4c34
+Create Date: 2024-11-11 18:27:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b2c3d4e5f6a7"
+down_revision = "7d82c8fc4c34"
+branch_labels = None
+depends_on = None
+
+# Default identity keys to create
+IDENTITY_KEYS = ["email", "phone_number", "fides_user_device_id", "external_id"]
+
+# Privacy preferences partition tables
+PARTITIONS = ["current", "historic"]
+
+
+def upgrade():
+    # Add is_default column to identity_definition table
+    op.add_column(
+        "identity_definition",
+        sa.Column("is_default", sa.Boolean(), nullable=False, server_default="false"),
+    )
+
+    # Delete any existing entries for these identity keys
+    # This handles cases where the API was already released and users may have created these
+    op.execute(
+        """
+        DELETE FROM identity_definition
+        WHERE identity_key IN ('email', 'phone_number', 'fides_user_device_id', 'external_id')
+        """
+    )
+
+    # Insert default identity definitions
+    op.execute(
+        """
+        INSERT INTO identity_definition (id, created_at, updated_at, identity_key, name, description, type, created_by, is_default)
+        VALUES
+          ('ide_' || gen_random_uuid(), now(), now(), 'email', 'Email', 'Email address', 'email', NULL, true),
+          ('ide_' || gen_random_uuid(), now(), now(), 'phone_number', 'Phone number', 'Phone number', 'phone_number', NULL, true),
+          ('ide_' || gen_random_uuid(), now(), now(), 'fides_user_device_id', 'Fides user device ID', 'Unique device identifier for Fides', 'uuid', NULL, true),
+          ('ide_' || gen_random_uuid(), now(), now(), 'external_id', 'External ID', 'External identifier from another system', 'string', NULL, true)
+        """
+    )
+
+    # Create indexes on privacy_preferences partitions for each identity key
+    # These indexes enable efficient lookups on search_data->'identity'->>'{identity_key}'
+    for identity_key in IDENTITY_KEYS:
+        for partition in PARTITIONS:
+            index_name = f"idx_identity_{identity_key}_{partition}"
+            table_name = f"privacy_preferences_{partition}"
+            op.execute(
+                f"CREATE INDEX {index_name} "
+                f"ON {table_name} ((search_data->'identity'->>{repr(identity_key)}))"
+            )
+
+
+def downgrade():
+    # Drop indexes from privacy_preferences partitions
+    for identity_key in IDENTITY_KEYS:
+        for partition in PARTITIONS:
+            index_name = f"idx_identity_{identity_key}_{partition}"
+            op.execute(f"DROP INDEX IF EXISTS {index_name}")
+
+    # Remove default identity definitions
+    op.execute(
+        """
+        DELETE FROM identity_definition
+        WHERE identity_key IN ('email', 'phone_number', 'fides_user_device_id', 'external_id')
+        """
+    )
+
+    # Drop is_default column from identity_definition table
+    op.drop_column("identity_definition", "is_default")

fides/api/alembic/migrations/versions/xx_2025_11_25_1854_3ff6449c099e_add_index_on_providedidentity_privacy_.py

@@ -0,0 +1,50 @@
+"""add index on providedidentity.privacy_request_id
+
+Revision ID: 3ff6449c099e
+Revises: 56fe6fad2d89
+Create Date: 2025-11-25 18:54:50.135782
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+from fides.api.migrations.post_upgrade_index_creation import INDEX_ROW_COUNT_THRESHOLD
+
+# revision identifiers, used by Alembic.
+revision = "3ff6449c099e"
+down_revision = "56fe6fad2d89"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    connection = op.get_bind()
+
+    # Check providedidentity table size to decide if we should create index immediately
+    providedidentity_count = connection.execute(
+        sa.text("SELECT COUNT(*) FROM providedidentity")
+    ).scalar()
+
+    if providedidentity_count < INDEX_ROW_COUNT_THRESHOLD:
+        op.create_index(
+            op.f("ix_providedidentity_privacy_request_id"),
+            "providedidentity",
+            ["privacy_request_id"],
+            unique=False,
+        )
+    else:
+        logger.warning(
+            "The providedidentity table has more than 1 million rows, "
+            "skipping index creation. Indexes will be created during application startup "
+            "via post_upgrade_index_creation.py"
+        )
+
+
+def downgrade():
+    # Drop index if it exists (it may not exist if it was deferred)
+    # Note: If you downgrade this migration, you should also remove the
+    # "providedidentity" entry from TABLE_OBJECT_MAP in post_upgrade_index_creation.py
+    # to prevent the index from being automatically recreated on app startup
+    op.execute("DROP INDEX IF EXISTS ix_providedidentity_privacy_request_id")

fides/api/api/v1/api.py

@@ -2,6 +2,7 @@ from fides.api.api.v1.endpoints import (
     config_endpoints,
     connection_endpoints,
     connection_type_endpoints,
+    connector_template_endpoints,
     consent_request_endpoints,
     dataset_config_endpoints,
     drp_endpoints,
@@ -31,6 +32,7 @@ api_router = APIRouter()
 api_router.include_router(config_endpoints.router)
 api_router.include_router(connection_type_endpoints.router)
 api_router.include_router(connection_endpoints.router)
+api_router.include_router(connector_template_endpoints.router)
 api_router.include_router(consent_request_endpoints.router)
 api_router.include_router(dataset_config_endpoints.router)
 api_router.include_router(drp_endpoints.router)