ethyca-fides 2.63.0rc0__py2.py3-none-any.whl → 2.63.0rc2__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/bf713b5a021d_staged_resource_ancestor_link_data_.py

@@ -0,0 +1,250 @@
+"""
+Staged resource ancestor link data migration.
+
+Indexes and constraints are created on this table _after_ data is populated.
+If > 1,000,000 stagedresourceancestor records are created as part
+of the data migration, the migration will skip creating the indexes
+and constraints, and will instead log a message
+indicating that index migration should be performed as part of app runtime.
+
+The data migration has the following steps:
+- Query for all staged resources and their children
+- Build a list in-memory of all ancestor-descendant pairs to insert by recursively
+    processing each resource's children
+- Write the ancestor links to a CSV string in-memory
+- Copy the CSV string into the stagedresourceancestor table via a PostgreSQL COPY
+    command
+- Create the indexes and constraints on the stagedresourceancestor table if
+    < 1,000,000 ancestor links are created
+
+
+Revision ID: bf713b5a021d
+Revises: 5474a47c77da
+Create Date: 2025-06-03 09:44:58.769535
+
+"""
+
+import csv
+import uuid
+from io import StringIO
+from pathlib import Path
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "bf713b5a021d"
+down_revision = "5474a47c77da"
+
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    logger.info("Populating staged resource ancestor links...")
+
+    conn = op.get_bind()
+
+    # Get all resources and their children in batches
+    BATCH_SIZE = 500000
+
+    # Query resources in batches using yield_per
+    resources_query = sa.text(
+        """
+        SELECT urn, children
+        FROM stagedresource
+    """
+    )
+    resource_children = {}
+
+    # process resources and populate resource_children map in batches
+    # to limit memory usage
+    for batch in (
+        conn.execution_options(stream_results=True)
+        .execute(resources_query)
+        .yield_per(BATCH_SIZE)
+        .partitions()
+    ):
+        logger.info(f"Processing batch of {BATCH_SIZE} resources")
+
+        # Build resource -> children map for this batch
+        for result in batch:
+            urn = result.urn
+            children = result.children
+            if children:
+                resource_children[urn] = children
+
+    # Build list of ancestor-descendant pairs
+    ancestor_links = []
+
+    def process_children(ancestor_urn, children, visited=None):
+        """Recursively process children and collect ancestor links"""
+        if visited is None:
+            visited = set()
+
+        for child_urn in children:
+            if child_urn not in visited:
+                visited.add(child_urn)
+                # Add direct ancestor link
+                ancestor_links.append(
+                    {
+                        "id": f"srl_{uuid.uuid4()}",
+                        "ancestor_urn": ancestor_urn,
+                        "descendant_urn": child_urn,
+                    }
+                )
+
+                # Recursively process this child's children
+                if child_urn in resource_children:
+                    process_children(
+                        ancestor_urn, resource_children[child_urn], visited
+                    )
+
+    logger.info(
+        f"Recursively processing {len(resource_children)} resources for ancestor links in current batch"
+    )
+
+    # Process each resource's children recursively
+    for ancestor_urn, children in resource_children.items():
+        process_children(ancestor_urn, children)
+
+    # remove the resource_children map from memory
+    del resource_children
+
+    ancestor_links_count = len(ancestor_links)
+    logger.info(f"Found {ancestor_links_count} ancestor links in current batch")
+
+    if ancestor_links_count > 0:
+        # Create temporary CSV file
+        temp_csv_path = Path("staged_resource_ancestors.csv")
+        with open(temp_csv_path, "w", newline="") as csv_file:
+            writer = csv.DictWriter(
+                csv_file, fieldnames=["id", "ancestor_urn", "descendant_urn"]
+            )
+            writer.writeheader()
+            logger.info(f"Writing {ancestor_links_count} ancestor links to CSV file")
+            writer.writerows(ancestor_links)
+
+        del ancestor_links
+
+        # Copy all data from CSV file into table
+        logger.info(
+            f"Copying {ancestor_links_count} ancestor links from CSV file into stagedresourceancestor table..."
+        )
+        with open(temp_csv_path, "r") as csv_file:
+            copy_query = """
+                COPY stagedresourceancestor (id, ancestor_urn, descendant_urn)
+                FROM STDIN
+                WITH (FORMAT CSV, HEADER TRUE)
+            """
+            conn.connection.cursor().copy_expert(copy_query, csv_file)
+
+        # Clean up temp file
+        temp_csv_path.unlink()
+
+        logger.info(
+            f"Completed copying all ancestor links. Total ancestor links created: {ancestor_links_count}"
+        )
+
+    if ancestor_links_count < 1000000:
+
+        logger.info("Creating primary key index on stagedresourceancestor table...")
+
+        op.create_index(
+            "ix_staged_resource_ancestor_pkey",
+            "stagedresourceancestor",
+            ["id"],
+            unique=True,
+        )
+
+        logger.info("Completed creating primary key index")
+
+        logger.info("Creating foreign key constraints stagedresourceancestor table...")
+
+        op.create_foreign_key(
+            "fk_staged_resource_ancestor_ancestor",
+            "stagedresourceancestor",
+            "stagedresource",
+            ["ancestor_urn"],
+            ["urn"],
+            ondelete="CASCADE",
+        )
+
+        op.create_foreign_key(
+            "fk_staged_resource_ancestor_descendant",
+            "stagedresourceancestor",
+            "stagedresource",
+            ["descendant_urn"],
+            ["urn"],
+            ondelete="CASCADE",
+        )
+
+        logger.info("Completed creating foreign key constraints")
+
+        logger.info("Creating unique constraint on stagedresourceancestor table...")
+
+        op.create_unique_constraint(
+            "uq_staged_resource_ancestor",
+            "stagedresourceancestor",
+            ["ancestor_urn", "descendant_urn"],
+        )
+
+        logger.info("Completed creating unique constraint")
+
+        logger.info("Creating indexes on stagedresourceancestor table...")
+
+        op.create_index(
+            "ix_staged_resource_ancestor_ancestor",
+            "stagedresourceancestor",
+            ["ancestor_urn"],
+            unique=False,
+        )
+        op.create_index(
+            "ix_staged_resource_ancestor_descendant",
+            "stagedresourceancestor",
+            ["descendant_urn"],
+            unique=False,
+        )
+
+        logger.info("Completed creating indexes on stagedresourceancestor table")
+    else:
+        logger.info(
+            "Skipping creation of primary key index, foreign key constraints, unique constraint, and indexes on stagedresourceancestor table because there are more than 1,000,000 ancestor links. These will be populated as part of the `post_upgrade_index_creation.py` task kicked off during application startup."
+        )
+
+
+def downgrade():
+    logger.info(
+        "Downgrading staged resource ancestor link data migration, populating child_diff_statuses..."
+    )
+
+    # Get child diff statuses for each ancestor
+    conn = op.get_bind()
+    child_diff_statuses_query = """
+        UPDATE stagedresource
+        SET child_diff_statuses = child_statuses.status_map
+        FROM (
+            SELECT
+                stagedresourceancestor.ancestor_urn,
+                jsonb_object_agg(distinct(stagedresource.diff_status), True) as status_map
+            FROM stagedresourceancestor
+            JOIN stagedresource ON stagedresourceancestor.descendant_urn = stagedresource.urn
+            GROUP BY stagedresourceancestor.ancestor_urn
+        ) AS child_statuses
+        WHERE stagedresource.urn = child_statuses.ancestor_urn
+    """
+    result = conn.execute(child_diff_statuses_query)
+    updated_rows = result.rowcount
+
+    logger.info(
+        f"Downgraded staged resource ancestor link data migration, completed populating child_diff_statuses for {updated_rows} rows"
+    )
+
+    # Intentionally not dropping the stagedresourceancestor indexes here because they
+    # may not have been created as part of the data migration above. If they were not created,
+    # then the downgrade would fail trying to drop the non-existent indexes.
+
+    # The indexes and constraints will be dropped as part of the overall table drop that's done
+    # as part of the downgrade in the `5474a47c77da_create_staged_resource_ancestor_link_table.py`
+    # migration.

fides/api/alembic/migrations/versions/c586a56c25e7_remove_child_diff_statuses.py

@@ -0,0 +1,40 @@
+"""
+Removes the  stagedresource.child_diff_statuses column.
+
+This migration is run _after_ the data migration
+that populates the staged resource ancestor link table,
+to prevent unintended data loss.
+
+Revision ID: c586a56c25e7
+Revises: bf713b5a021d
+Create Date: 2025-06-03 09:47:11.389652
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "c586a56c25e7"
+down_revision = "bf713b5a021d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # remove the StagedResource.child_diff_statuses column as it's no longer needed
+    op.drop_column("stagedresource", "child_diff_statuses")
+
+
+def downgrade():
+    # re-add the StagedResource.child_diff_statuses column
+    op.add_column(
+        "stagedresource",
+        sa.Column(
+            "child_diff_statuses",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+    )

fides/api/main.py

@@ -33,6 +33,9 @@ from fides.api.common_exceptions import MalisciousUrlException
 from fides.api.cryptography.identity_salt import get_identity_salt
 from fides.api.middleware import handle_audit_log_resource
 from fides.api.migrations.hash_migration_job import initiate_bcrypt_migration_task
+from fides.api.migrations.post_upgrade_index_creation import (
+    initiate_post_upgrade_index_creation,
+)
 from fides.api.schemas.analytics import Event, ExtraData
 
 # pylint: disable=wildcard-import, unused-wildcard-import
@@ -97,6 +100,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
     initiate_bcrypt_migration_task()
+    initiate_post_upgrade_index_creation()
 
     logger.debug("Sending startup analytics events...")
     # Avoid circular imports

fides/api/migrations/post_upgrade_index_creation.py

@@ -0,0 +1,269 @@
+import json
+from typing import Dict, List
+
+from loguru import logger
+from redis.lock import Lock
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from fides.api.db.session import get_db_session
+from fides.api.tasks.scheduled.scheduler import scheduler
+from fides.api.util.lock import redis_lock
+from fides.config import CONFIG
+
+"""
+This utility is used to detect indices or constraints that were deferred as part
+of a standard Fides/Alembic migration.
+
+The standard approach for creating indices or constraints on large tables is a blocking operation,
+meaning reads and write can't run while the indices and constraints are being created.
+A non-blocking approach is to use the CONCURRENTLY keyword for index creation then adding
+the constraints with ADD CONSTRAINT USING INDEX.
+
+However these approaches cannot run in a transaction, which is what Fides/Alembic uses to
+apply migrations. The approach here is to run these commands after the application has started
+up and commit the changes immediately, outside of a transaction.
+
+The commands that need to run are denoted in the `TABLE_OBJECT_MAP`. Indices that are used
+to create constraints also include the `constraint_name` value since either the presence of
+the index or the constraint can be used to determine if an index has executed.
+When using the ADD CONSTRAINT USING INDEX syntax, the index specified in the command is
+automatically renamed to match the name of the constraint being added.
+This means that after the constraint is created, the index will have the same name as the constraint.
+"""
+
+POST_UPGRADE_INDEX_CREATION = "post_upgrade_index_creation"
+
+TABLE_OBJECT_MAP: Dict[str, List[Dict[str, str]]] = {
+    "currentprivacypreferencev2": [
+        {
+            "name": "ix_currentprivacypreferencev2_email_property_id",
+            "statement": "CREATE UNIQUE INDEX CONCURRENTLY ix_currentprivacypreferencev2_email_property_id ON currentprivacypreferencev2 (email, property_id)",
+            "type": "index",
+            "constraint_name": "last_saved_for_email_per_property_id",
+        },
+        {
+            "name": "ix_currentprivacypreferencev2_external_id_property_id",
+            "statement": "CREATE UNIQUE INDEX CONCURRENTLY ix_currentprivacypreferencev2_external_id_property_id ON currentprivacypreferencev2 (external_id, property_id)",
+            "type": "index",
+            "constraint_name": "last_saved_for_external_id_per_property_id",
+        },
+        {
+            "name": "ix_currentprivacypreferencev2_fides_user_device_property_id",
+            "statement": "CREATE UNIQUE INDEX CONCURRENTLY ix_currentprivacypreferencev2_fides_user_device_property_id ON currentprivacypreferencev2 (fides_user_device, property_id)",
+            "type": "index",
+            "constraint_name": "last_saved_for_fides_user_device_per_property_id",
+        },
+        {
+            "name": "ix_currentprivacypreferencev2_phone_number_property_id",
+            "statement": "CREATE UNIQUE INDEX CONCURRENTLY ix_currentprivacypreferencev2_phone_number_property_id ON currentprivacypreferencev2 (phone_number, property_id)",
+            "type": "index",
+            "constraint_name": "last_saved_for_phone_number_per_property_id",
+        },
+        {
+            "name": "ix_currentprivacypreferencev2_hashed_external_id",
+            "statement": "CREATE INDEX CONCURRENTLY ix_currentprivacypreferencev2_hashed_external_id ON currentprivacypreferencev2 (hashed_external_id)",
+            "type": "index",
+        },
+        {
+            "name": "last_saved_for_email_per_property_id",
+            "statement": "ALTER TABLE currentprivacypreferencev2 ADD CONSTRAINT last_saved_for_email_per_property_id UNIQUE USING INDEX ix_currentprivacypreferencev2_email_property_id",
+            "type": "constraint",
+        },
+        {
+            "name": "last_saved_for_external_id_per_property_id",
+            "statement": "ALTER TABLE currentprivacypreferencev2 ADD CONSTRAINT last_saved_for_external_id_per_property_id UNIQUE USING INDEX ix_currentprivacypreferencev2_external_id_property_id",
+            "type": "constraint",
+        },
+        {
+            "name": "last_saved_for_fides_user_device_per_property_id",
+            "statement": "ALTER TABLE currentprivacypreferencev2 ADD CONSTRAINT last_saved_for_fides_user_device_per_property_id UNIQUE USING INDEX ix_currentprivacypreferencev2_fides_user_device_property_id",
+            "type": "constraint",
+        },
+        {
+            "name": "last_saved_for_phone_number_per_property_id",
+            "statement": "ALTER TABLE currentprivacypreferencev2 ADD CONSTRAINT last_saved_for_phone_number_per_property_id UNIQUE USING INDEX ix_currentprivacypreferencev2_phone_number_property_id",
+            "type": "constraint",
+        },
+    ],
+    "privacypreferencehistory": [
+        {
+            "name": "ix_privacypreferencehistory_hashed_external_id",
+            "statement": "CREATE INDEX CONCURRENTLY ix_privacypreferencehistory_hashed_external_id ON privacypreferencehistory (hashed_external_id)",
+            "type": "index",
+        },
+    ],
+    "servednoticehistory": [
+        {
+            "name": "ix_servednoticehistory_hashed_external_id",
+            "statement": "CREATE INDEX CONCURRENTLY ix_servednoticehistory_hashed_external_id ON servednoticehistory (hashed_external_id)",
+            "type": "index",
+        },
+    ],
+    "stagedresourceancestor": [
+        # primary key index
+        {
+            "name": "ix_staged_resource_ancestor_pkey",
+            "statement": "CREATE UNIQUE INDEX CONCURRENTLY ix_staged_resource_ancestor_pkey ON stagedresourceancestor (id)",
+            "type": "index",
+        },
+        # unique constraint + index
+        {
+            "name": "ix_staged_resource_ancestor_unique",
+            "statement": "CREATE UNIQUE INDEX CONCURRENTLY ix_staged_resource_ancestor_unique ON stagedresourceancestor (ancestor_urn, descendant_urn)",
+            "type": "index",
+            "constraint_name": "uq_staged_resource_ancestor",
+        },
+        {
+            "name": "uq_staged_resource_ancestor",
+            "statement": "ALTER TABLE stagedresourceancestor ADD CONSTRAINT uq_staged_resource_ancestor UNIQUE USING INDEX ix_staged_resource_ancestor_unique",
+            "type": "constraint",
+        },
+        # foreign key constraints
+        {
+            "name": "fk_staged_resource_ancestor_ancestor",
+            "statement": "ALTER TABLE stagedresourceancestor ADD CONSTRAINT fk_staged_resource_ancestor_ancestor FOREIGN KEY (ancestor_urn) REFERENCES stagedresource (urn) ON DELETE CASCADE",
+            "type": "constraint",
+        },
+        {
+            "name": "fk_staged_resource_ancestor_descendant",
+            "statement": "ALTER TABLE stagedresourceancestor ADD CONSTRAINT fk_staged_resource_ancestor_descendant FOREIGN KEY (descendant_urn) REFERENCES stagedresource (urn) ON DELETE CASCADE",
+            "type": "constraint",
+        },
+        # column indexes
+        {
+            "name": "ix_staged_resource_ancestor_ancestor",
+            "statement": "CREATE INDEX CONCURRENTLY ix_staged_resource_ancestor_ancestor ON stagedresourceancestor (ancestor_urn)",
+            "type": "index",
+        },
+        {
+            "name": "ix_staged_resource_ancestor_descendant",
+            "statement": "CREATE INDEX CONCURRENTLY ix_staged_resource_ancestor_descendant ON stagedresourceancestor (descendant_urn)",
+            "type": "index",
+        },
+    ],
+}
+
+
+def check_object_exists(db: Session, object_name: str) -> bool:
+    """Checks Postgres' system catalogs to verify the existence of a given index or constraint in a specific table."""
+    object_query = text(
+        """
+        SELECT EXISTS (
+            SELECT 1
+            FROM pg_indexes
+            WHERE indexname = :object_name
+        ) OR EXISTS (
+            SELECT 1
+            FROM pg_constraint
+            WHERE conname = :object_name
+        )
+        """
+    )
+    result = db.execute(
+        object_query,
+        {
+            "object_name": object_name,
+        },
+    ).scalar()
+    return result
+
+
+def create_object(db: Session, object_statement: str, object_name: str) -> None:
+    """Executes the index or constraint creation statement."""
+    logger.info(f"Creating index/constraint object: '{object_name}'...")
+    with db.bind.connect().execution_options(
+        isolation_level="AUTOCOMMIT"
+    ) as connection:
+        connection.execute(text(object_statement))
+    logger.info(f"Successfully created index/constraint object: '{object_name}'")
+
+
+def check_and_create_objects(
+    db: Session, table_object_map: Dict[str, List[Dict[str, str]]], lock: Lock
+) -> Dict[str, str]:
+    """Returns a dictionary of any indices or constraints that are in the process of being created."""
+    object_info: Dict[str, str] = {}
+    for _, objects in table_object_map.items():
+        for object_data in objects:
+            object_name = object_data["name"]
+            object_statement = object_data["statement"]
+            object_type = object_data["type"]
+            object_exists = check_object_exists(db, object_name)
+
+            if not object_exists:
+                if object_type == "index":
+                    constraint_name = object_data.get("constraint_name")
+                    if constraint_name:
+                        constraint_exists = check_object_exists(db, constraint_name)
+                        if constraint_exists:
+                            logger.debug(
+                                f"Constraint {constraint_name} already exists, skipping index creation for {object_name}"
+                            )
+                            continue
+
+                create_object(db, object_statement, object_name)
+                object_info[object_name] = "in progress"
+            else:
+                logger.debug(
+                    f"Object {object_name} already exists, skipping index/constraint creation"
+                )
+            lock.reacquire()
+
+    return object_info
+
+
+# Lock key for the post upgrade index creation
+POST_UPGRADE_INDEX_CREATION_LOCK = "post_upgrade_index_creation_lock"
+# The timeout of the lock for the post upgrade index creation, in seconds
+POST_UPGRADE_INDEX_CREATION_LOCK_TIMEOUT_SECONDS = 600  # 10 minutes
+
+
+def post_upgrade_index_creation_task() -> None:
+    """
+    Task for creating indices and constraints that were deferred
+    as part of a standard Fides/Alembic migration.
+
+    This task is to be kicked off as a background task during application startup,
+    after standard migrations have been applied.
+
+    If all specified indexes and constraints are created, the task is effectively
+    a no-op, as it checks for the presence of the objects in the database before
+    creating them.
+    """
+    with redis_lock(
+        POST_UPGRADE_INDEX_CREATION_LOCK,
+        POST_UPGRADE_INDEX_CREATION_LOCK_TIMEOUT_SECONDS,
+    ) as lock:
+        if not lock:
+            return
+
+        SessionLocal = get_db_session(CONFIG)
+        with SessionLocal() as db:
+            object_info: Dict[str, str] = check_and_create_objects(
+                db, TABLE_OBJECT_MAP, lock
+            )
+            if object_info:
+                logger.info(
+                    f"Post upgrade index creation output: {json.dumps(object_info)}"
+                )
+            else:
+                logger.debug("All indices and constraints created")
+
+
+def initiate_post_upgrade_index_creation() -> None:
+    """Initiates scheduler to migrate all non-credential tables using a bcrypt hash to use a SHA-256 hash"""
+
+    if CONFIG.test_mode:
+        logger.debug("Skipping post upgrade index creation in test mode")
+        return
+
+    assert (
+        scheduler.running
+    ), "Scheduler is not running! Cannot migrate tables with bcrypt hashes."
+
+    logger.info("Initiating scheduler for hash migration")
+    scheduler.add_job(
+        func=post_upgrade_index_creation_task,
+        id=POST_UPGRADE_INDEX_CREATION,
+    )