ethyca-fides 2.63.0rc0__tar.gz → 2.63.0rc2__tar.gz

{ethyca_fides-2.63.0rc0/src/ethyca_fides.egg-info → ethyca_fides-2.63.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ethyca_fides
-Version: 2.63.0rc0
+Version: 2.63.0rc2
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.63.0rc0 → ethyca_fides-2.63.0rc2/src/ethyca_fides.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ethyca_fides
-Version: 2.63.0rc0
+Version: 2.63.0rc2
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.63.0rc0 → ethyca_fides-2.63.0rc2}/src/ethyca_fides.egg-info/SOURCES.txt

@@ -119,6 +119,7 @@ src/fides/api/alembic/migrations/versions/52b2cfd1af47_fix_messaging_template_in
 src/fides/api/alembic/migrations/versions/5307999c0dac_remove_deprecated_data_uses_for_.py
 src/fides/api/alembic/migrations/versions/530fb8533ca4_test.py
 src/fides/api/alembic/migrations/versions/54102bba36de_add_attentive_connector.py
+src/fides/api/alembic/migrations/versions/5474a47c77da_create_staged_resource_ancestor_link_table.py
 src/fides/api/alembic/migrations/versions/548a1ac26771_fideslang_3.0_remove_references.py
 src/fides/api/alembic/migrations/versions/55bedede956d_requesttask.py
 src/fides/api/alembic/migrations/versions/55d61eb8ed12_add_default_policies.py
@@ -230,11 +231,13 @@ src/fides/api/alembic/migrations/versions/bab75915670a_add_finished_audit_log.py
 src/fides/api/alembic/migrations/versions/bd875a8b5d96_add_attachments_and_.py
 src/fides/api/alembic/migrations/versions/bde646a6f51e_add_execution_timeframe.py
 src/fides/api/alembic/migrations/versions/be432bd23596_add_fidesctl_meta_field_to_dataset.py
+src/fides/api/alembic/migrations/versions/bf713b5a021d_staged_resource_ancestor_link_data_.py
 src/fides/api/alembic/migrations/versions/c1885270b3cc_add_html_format_for_storageconfig.py
 src/fides/api/alembic/migrations/versions/c2f7a29c4780_email_connection_config.py
 src/fides/api/alembic/migrations/versions/c3472d75c80e_connectionconfig_disabled.py
 src/fides/api/alembic/migrations/versions/c4df5d585029_data_use_unique_together.py
 src/fides/api/alembic/migrations/versions/c5336b841d70_add_policy_webhooks.py
+src/fides/api/alembic/migrations/versions/c586a56c25e7_remove_child_diff_statuses.py
 src/fides/api/alembic/migrations/versions/c5a218831820_additional_tcf_fields_system.py
 src/fides/api/alembic/migrations/versions/c61fd9d4f73e_emailconfig.py
 src/fides/api/alembic/migrations/versions/c7cc36820d4b_rename_user_name_tables_and_indexes.py
@@ -377,6 +380,7 @@ src/fides/api/migrations/__init__.py
 src/fides/api/migrations/hash_migration_job.py
 src/fides/api/migrations/hash_migration_mixin.py
 src/fides/api/migrations/hash_migration_tracker.py
+src/fides/api/migrations/post_upgrade_index_creation.py
 src/fides/api/models/__init__.py
 src/fides/api/models/application_config.py
 src/fides/api/models/asset.py
@@ -684,6 +688,7 @@ src/fides/api/util/errors.py
 src/fides/api/util/filter_utils.py
 src/fides/api/util/fuzzy_search_utils.py
 src/fides/api/util/identity_verification.py
+src/fides/api/util/lock.py
 src/fides/api/util/logger.py
 src/fides/api/util/logger_context_utils.py
 src/fides/api/util/matching_queue.py
@@ -819,8 +824,8 @@ src/fides/ui-build/static/admin/properties.html
 src/fides/ui-build/static/admin/systems.html
 src/fides/ui-build/static/admin/taxonomy.html
 src/fides/ui-build/static/admin/user-management.html
-src/fides/ui-build/static/admin/_next/static/LY2HTf0jAydL8pMpld53D/_buildManifest.js
-src/fides/ui-build/static/admin/_next/static/LY2HTf0jAydL8pMpld53D/_ssgManifest.js
+src/fides/ui-build/static/admin/_next/static/Fb70i-8GI-owNAvgEJWhA/_buildManifest.js
+src/fides/ui-build/static/admin/_next/static/Fb70i-8GI-owNAvgEJWhA/_ssgManifest.js
 src/fides/ui-build/static/admin/_next/static/chunks/1099-667e84655846e78c.js
 src/fides/ui-build/static/admin/_next/static/chunks/1100-0b01f500b1eaf60d.js
 src/fides/ui-build/static/admin/_next/static/chunks/1817-951f58ddd5d46081.js
@@ -900,7 +905,7 @@ src/fides/ui-build/static/admin/_next/static/chunks/pages/dataset-e3b2d8e3980ab0
 src/fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-84eb56f5a07bdf4f.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/fides-js-docs-80b241bf6cddb72e.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/index-773182b5e35d0045.js
-src/fides/ui-build/static/admin/_next/static/chunks/pages/integrations-b582e348423e5bf4.js
+src/fides/ui-build/static/admin/_next/static/chunks/pages/integrations-781808bca01f8048.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/login-0cb0eb44bf2dd24c.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/messaging-7dc295b3ed3d2224.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-20cdb2c8a03deae1.js

{ethyca_fides-2.63.0rc0 → ethyca_fides-2.63.0rc2}/src/fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-04T15:43:31-0400",
+ "date": "2025-06-06T16:55:52-0300",
  "dirty": true,
  "error": null,
- "full-revisionid": "01dc75672a6f67a929e0aa20e2eb9e99ccadb0de",
- "version": "2.63.0rc0"
+ "full-revisionid": "20e38def347592a9f23c0e896f17fe63c4663b1e",
+ "version": "2.63.0rc2"
 }
 '''  # END VERSION_JSON
 

ethyca_fides-2.63.0rc2/src/fides/api/alembic/migrations/versions/5474a47c77da_create_staged_resource_ancestor_link_table.py

@@ -0,0 +1,72 @@
+"""
+Creates staged resource ancestor link table.
+
+Does not populate the table with data, that is handled in a separate data migration.
+
+Also does not drop the StagedResource.child_diff_statuses column,
+that is handled in a separate migration, _after_ the data migration
+that populates the staged resource ancestor link table, to prevent
+unintended data loss.
+
+Revision ID: 5474a47c77da
+Revises: 440a5b9a3493
+Create Date: 2025-05-27 19:02:43.802783
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "5474a47c77da"
+down_revision = "440a5b9a3493"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "stagedresourceancestor",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("ancestor_urn", sa.String(), nullable=False),
+        sa.Column("descendant_urn", sa.String(), nullable=False),
+        # primary key, foreign key and unique constraints are deferred
+        # until after the data migration that populates the table,
+        # to avoid slowing down the data migration
+    )
+
+    # defer creating `stagedresourceancestor` indexes until after the data migration
+    # that populates the table, to avoid slowing down the data migration
+
+    # add an index for StagedResource.diff_status to improve queries against the diff_status field
+    op.create_index(
+        op.f("ix_stagedresource_diff_status"),
+        "stagedresource",
+        ["diff_status"],
+        unique=False,
+    )
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    # drop the StagedResource.diff_status index created in the migration
+    op.drop_index(op.f("ix_stagedresource_diff_status"), table_name="stagedresource")
+
+    # drop the stagedresourceancestor table (and all its indexes and constraints)
+    op.drop_table("stagedresourceancestor")
+    # ### end Alembic commands ###

ethyca_fides-2.63.0rc2/src/fides/api/alembic/migrations/versions/bf713b5a021d_staged_resource_ancestor_link_data_.py

@@ -0,0 +1,250 @@
+"""
+Staged resource ancestor link data migration.
+
+Indexes and constraints are created on this table _after_ data is populated.
+If > 1,000,000 stagedresourceancestor records are created as part
+of the data migration, the migration will skip creating the indexes
+and constraints, and will instead log a message
+indicating that index migration should be performed as part of app runtime.
+
+The data migration has the following steps:
+- Query for all staged resources and their children
+- Build a list in-memory of all ancestor-descendant pairs to insert by recursively
+    processing each resource's children
+- Write the ancestor links to a CSV string in-memory
+- Copy the CSV string into the stagedresourceancestor table via a PostgreSQL COPY
+    command
+- Create the indexes and constraints on the stagedresourceancestor table if
+    < 1,000,000 ancestor links are created
+
+
+Revision ID: bf713b5a021d
+Revises: 5474a47c77da
+Create Date: 2025-06-03 09:44:58.769535
+
+"""
+
+import csv
+import uuid
+from io import StringIO
+from pathlib import Path
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "bf713b5a021d"
+down_revision = "5474a47c77da"
+
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    logger.info("Populating staged resource ancestor links...")
+
+    conn = op.get_bind()
+
+    # Get all resources and their children in batches
+    BATCH_SIZE = 500000
+
+    # Query resources in batches using yield_per
+    resources_query = sa.text(
+        """
+        SELECT urn, children
+        FROM stagedresource
+    """
+    )
+    resource_children = {}
+
+    # process resources and populate resource_children map in batches
+    # to limit memory usage
+    for batch in (
+        conn.execution_options(stream_results=True)
+        .execute(resources_query)
+        .yield_per(BATCH_SIZE)
+        .partitions()
+    ):
+        logger.info(f"Processing batch of {BATCH_SIZE} resources")
+
+        # Build resource -> children map for this batch
+        for result in batch:
+            urn = result.urn
+            children = result.children
+            if children:
+                resource_children[urn] = children
+
+    # Build list of ancestor-descendant pairs
+    ancestor_links = []
+
+    def process_children(ancestor_urn, children, visited=None):
+        """Recursively process children and collect ancestor links"""
+        if visited is None:
+            visited = set()
+
+        for child_urn in children:
+            if child_urn not in visited:
+                visited.add(child_urn)
+                # Add direct ancestor link
+                ancestor_links.append(
+                    {
+                        "id": f"srl_{uuid.uuid4()}",
+                        "ancestor_urn": ancestor_urn,
+                        "descendant_urn": child_urn,
+                    }
+                )
+
+                # Recursively process this child's children
+                if child_urn in resource_children:
+                    process_children(
+                        ancestor_urn, resource_children[child_urn], visited
+                    )
+
+    logger.info(
+        f"Recursively processing {len(resource_children)} resources for ancestor links in current batch"
+    )
+
+    # Process each resource's children recursively
+    for ancestor_urn, children in resource_children.items():
+        process_children(ancestor_urn, children)
+
+    # remove the resource_children map from memory
+    del resource_children
+
+    ancestor_links_count = len(ancestor_links)
+    logger.info(f"Found {ancestor_links_count} ancestor links in current batch")
+
+    if ancestor_links_count > 0:
+        # Create temporary CSV file
+        temp_csv_path = Path("staged_resource_ancestors.csv")
+        with open(temp_csv_path, "w", newline="") as csv_file:
+            writer = csv.DictWriter(
+                csv_file, fieldnames=["id", "ancestor_urn", "descendant_urn"]
+            )
+            writer.writeheader()
+            logger.info(f"Writing {ancestor_links_count} ancestor links to CSV file")
+            writer.writerows(ancestor_links)
+
+        del ancestor_links
+
+        # Copy all data from CSV file into table
+        logger.info(
+            f"Copying {ancestor_links_count} ancestor links from CSV file into stagedresourceancestor table..."
+        )
+        with open(temp_csv_path, "r") as csv_file:
+            copy_query = """
+                COPY stagedresourceancestor (id, ancestor_urn, descendant_urn)
+                FROM STDIN
+                WITH (FORMAT CSV, HEADER TRUE)
+            """
+            conn.connection.cursor().copy_expert(copy_query, csv_file)
+
+        # Clean up temp file
+        temp_csv_path.unlink()
+
+        logger.info(
+            f"Completed copying all ancestor links. Total ancestor links created: {ancestor_links_count}"
+        )
+
+    if ancestor_links_count < 1000000:
+
+        logger.info("Creating primary key index on stagedresourceancestor table...")
+
+        op.create_index(
+            "ix_staged_resource_ancestor_pkey",
+            "stagedresourceancestor",
+            ["id"],
+            unique=True,
+        )
+
+        logger.info("Completed creating primary key index")
+
+        logger.info("Creating foreign key constraints stagedresourceancestor table...")
+
+        op.create_foreign_key(
+            "fk_staged_resource_ancestor_ancestor",
+            "stagedresourceancestor",
+            "stagedresource",
+            ["ancestor_urn"],
+            ["urn"],
+            ondelete="CASCADE",
+        )
+
+        op.create_foreign_key(
+            "fk_staged_resource_ancestor_descendant",
+            "stagedresourceancestor",
+            "stagedresource",
+            ["descendant_urn"],
+            ["urn"],
+            ondelete="CASCADE",
+        )
+
+        logger.info("Completed creating foreign key constraints")
+
+        logger.info("Creating unique constraint on stagedresourceancestor table...")
+
+        op.create_unique_constraint(
+            "uq_staged_resource_ancestor",
+            "stagedresourceancestor",
+            ["ancestor_urn", "descendant_urn"],
+        )
+
+        logger.info("Completed creating unique constraint")
+
+        logger.info("Creating indexes on stagedresourceancestor table...")
+
+        op.create_index(
+            "ix_staged_resource_ancestor_ancestor",
+            "stagedresourceancestor",
+            ["ancestor_urn"],
+            unique=False,
+        )
+        op.create_index(
+            "ix_staged_resource_ancestor_descendant",
+            "stagedresourceancestor",
+            ["descendant_urn"],
+            unique=False,
+        )
+
+        logger.info("Completed creating indexes on stagedresourceancestor table")
+    else:
+        logger.info(
+            "Skipping creation of primary key index, foreign key constraints, unique constraint, and indexes on stagedresourceancestor table because there are more than 1,000,000 ancestor links. These will be populated as part of the `post_upgrade_index_creation.py` task kicked off during application startup."
+        )
+
+
+def downgrade():
+    logger.info(
+        "Downgrading staged resource ancestor link data migration, populating child_diff_statuses..."
+    )
+
+    # Get child diff statuses for each ancestor
+    conn = op.get_bind()
+    child_diff_statuses_query = """
+        UPDATE stagedresource
+        SET child_diff_statuses = child_statuses.status_map
+        FROM (
+            SELECT
+                stagedresourceancestor.ancestor_urn,
+                jsonb_object_agg(distinct(stagedresource.diff_status), True) as status_map
+            FROM stagedresourceancestor
+            JOIN stagedresource ON stagedresourceancestor.descendant_urn = stagedresource.urn
+            GROUP BY stagedresourceancestor.ancestor_urn
+        ) AS child_statuses
+        WHERE stagedresource.urn = child_statuses.ancestor_urn
+    """
+    result = conn.execute(child_diff_statuses_query)
+    updated_rows = result.rowcount
+
+    logger.info(
+        f"Downgraded staged resource ancestor link data migration, completed populating child_diff_statuses for {updated_rows} rows"
+    )
+
+    # Intentionally not dropping the stagedresourceancestor indexes here because they
+    # may not have been created as part of the data migration above. If they were not created,
+    # then the downgrade would fail trying to drop the non-existent indexes.
+
+    # The indexes and constraints will be dropped as part of the overall table drop that's done
+    # as part of the downgrade in the `5474a47c77da_create_staged_resource_ancestor_link_table.py`
+    # migration.

ethyca_fides-2.63.0rc2/src/fides/api/alembic/migrations/versions/c586a56c25e7_remove_child_diff_statuses.py

@@ -0,0 +1,40 @@
+"""
+Removes the  stagedresource.child_diff_statuses column.
+
+This migration is run _after_ the data migration
+that populates the staged resource ancestor link table,
+to prevent unintended data loss.
+
+Revision ID: c586a56c25e7
+Revises: bf713b5a021d
+Create Date: 2025-06-03 09:47:11.389652
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "c586a56c25e7"
+down_revision = "bf713b5a021d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # remove the StagedResource.child_diff_statuses column as it's no longer needed
+    op.drop_column("stagedresource", "child_diff_statuses")
+
+
+def downgrade():
+    # re-add the StagedResource.child_diff_statuses column
+    op.add_column(
+        "stagedresource",
+        sa.Column(
+            "child_diff_statuses",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+    )

{ethyca_fides-2.63.0rc0 → ethyca_fides-2.63.0rc2}/src/fides/api/main.py

@@ -33,6 +33,9 @@ from fides.api.common_exceptions import MalisciousUrlException
 from fides.api.cryptography.identity_salt import get_identity_salt
 from fides.api.middleware import handle_audit_log_resource
 from fides.api.migrations.hash_migration_job import initiate_bcrypt_migration_task
+from fides.api.migrations.post_upgrade_index_creation import (
+    initiate_post_upgrade_index_creation,
+)
 from fides.api.schemas.analytics import Event, ExtraData
 
 # pylint: disable=wildcard-import, unused-wildcard-import
@@ -97,6 +100,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
     initiate_bcrypt_migration_task()
+    initiate_post_upgrade_index_creation()
 
     logger.debug("Sending startup analytics events...")
     # Avoid circular imports