ethyca-fides 2.62.1rc0__py2.py3-none-any.whl → 2.63.0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-06T15:00:50-0400",
+ "date": "2025-06-09T12:29:37-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "4fca3c3a4592528e57449b8ee9cafed8010d6052",
- "version": "2.62.1rc0"
+ "full-revisionid": "26ed475edc54d4cc56a7060fbec3ed7c16b647d2",
+ "version": "2.63.0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/2263583b0e44_add_shared_monitor_config.py

@@ -0,0 +1,82 @@
+"""add shared monitor config
+
+Revision ID: 2263583b0e44
+Revises: d0cbfec0b2dd
+Create Date: 2025-05-12 23:44:11.809581
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "2263583b0e44"
+down_revision = "d0cbfec0b2dd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "shared_monitor_config",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("key", sa.String(), nullable=False),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column(
+            "classify_params",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("key"),
+    )
+    op.create_index(
+        op.f("ix_shared_monitor_config_id"),
+        "shared_monitor_config",
+        ["id"],
+        unique=False,
+    )
+    op.add_column(
+        "monitorconfig", sa.Column("shared_config_id", sa.String(), nullable=True)
+    )
+    op.create_index(
+        op.f("ix_monitorconfig_shared_config_id"),
+        "monitorconfig",
+        ["shared_config_id"],
+        unique=False,
+    )
+    op.create_foreign_key(
+        "fk_monitorconfig_shared_config_id",
+        "monitorconfig",
+        "shared_monitor_config",
+        ["shared_config_id"],
+        ["id"],
+        ondelete="RESTRICT",
+    )
+
+
+def downgrade():
+    op.drop_constraint(
+        "fk_monitorconfig_shared_config_id", "monitorconfig", type_="foreignkey"
+    )
+    op.drop_index(op.f("ix_monitorconfig_shared_config_id"), table_name="monitorconfig")
+    op.drop_column("monitorconfig", "shared_config_id")
+    op.drop_index(
+        op.f("ix_shared_monitor_config_id"), table_name="shared_monitor_config"
+    )
+    op.drop_table("shared_monitor_config")

fides/api/alembic/migrations/versions/440a5b9a3493_add_fides_user_respondent_email_.py

@@ -0,0 +1,80 @@
+"""Add fides_user_respondent_email_verification table
+
+Revision ID: 440a5b9a3493
+Revises: 2263583b0e44
+Create Date: 2025-05-29 15:41:59.521940
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "440a5b9a3493"
+down_revision = "2263583b0e44"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "fides_user_respondent_email_verification",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column("user_id", sa.String(), nullable=False),
+        sa.Column("access_token", sa.String(), nullable=False),
+        sa.Column(
+            "access_token_expires_at", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.Column("identity_verified_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["fidesuser.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_fides_user_respondent_email_verification_id",
+        "fides_user_respondent_email_verification",
+        ["id"],
+        unique=True,
+    )
+    op.create_index(
+        "ix_fides_user_respondent_email_verification_access_token",
+        "fides_user_respondent_email_verification",
+        ["access_token"],
+        unique=True,
+    )
+    op.create_index(
+        "ix_fides_user_respondent_email_verification_user_id",
+        "fides_user_respondent_email_verification",
+        ["user_id"],
+    )
+
+
+def downgrade():
+    op.drop_index(
+        "ix_fides_user_respondent_email_verification_user_id",
+        table_name="fides_user_respondent_email_verification",
+    )
+    op.drop_index(
+        "ix_fides_user_respondent_email_verification_access_token",
+        table_name="fides_user_respondent_email_verification",
+    )
+    op.drop_index(
+        "ix_fides_user_respondent_email_verification_id",
+        table_name="fides_user_respondent_email_verification",
+    )
+    op.drop_table("fides_user_respondent_email_verification")

fides/api/alembic/migrations/versions/5474a47c77da_create_staged_resource_ancestor_link_table.py

@@ -0,0 +1,72 @@
+"""
+Creates staged resource ancestor link table.
+
+Does not populate the table with data, that is handled in a separate data migration.
+
+Also does not drop the StagedResource.child_diff_statuses column,
+that is handled in a separate migration, _after_ the data migration
+that populates the staged resource ancestor link table, to prevent
+unintended data loss.
+
+Revision ID: 5474a47c77da
+Revises: 440a5b9a3493
+Create Date: 2025-05-27 19:02:43.802783
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "5474a47c77da"
+down_revision = "440a5b9a3493"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "stagedresourceancestor",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("ancestor_urn", sa.String(), nullable=False),
+        sa.Column("descendant_urn", sa.String(), nullable=False),
+        # primary key, foreign key and unique constraints are deferred
+        # until after the data migration that populates the table,
+        # to avoid slowing down the data migration
+    )
+
+    # defer creating `stagedresourceancestor` indexes until after the data migration
+    # that populates the table, to avoid slowing down the data migration
+
+    # add an index for StagedResource.diff_status to improve queries against the diff_status field
+    op.create_index(
+        op.f("ix_stagedresource_diff_status"),
+        "stagedresource",
+        ["diff_status"],
+        unique=False,
+    )
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    # drop the StagedResource.diff_status index created in the migration
+    op.drop_index(op.f("ix_stagedresource_diff_status"), table_name="stagedresource")
+
+    # drop the stagedresourceancestor table (and all its indexes and constraints)
+    op.drop_table("stagedresourceancestor")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/bf713b5a021d_staged_resource_ancestor_link_data_.py

@@ -0,0 +1,259 @@
+"""
+Staged resource ancestor link data migration.
+
+Indexes and constraints are created on this table _after_ data is populated.
+If > 1,000,000 stagedresourceancestor records are created as part
+of the data migration, the migration will skip creating the indexes
+and constraints, and will instead log a message
+indicating that index migration should be performed as part of app runtime.
+
+The data migration has the following steps:
+- Query for all staged resources and their children
+- Build a list in-memory of all ancestor-descendant pairs to insert by recursively
+    processing each resource's children
+- Write the ancestor links to a CSV string in-memory
+- Copy the CSV string into the stagedresourceancestor table via a PostgreSQL COPY
+    command
+- Create the indexes and constraints on the stagedresourceancestor table if
+    < 1,000,000 ancestor links are created
+
+
+Revision ID: bf713b5a021d
+Revises: 5474a47c77da
+Create Date: 2025-06-03 09:44:58.769535
+
+"""
+
+import csv
+import uuid
+from io import StringIO
+from pathlib import Path
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "bf713b5a021d"
+down_revision = "5474a47c77da"
+
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    logger.info("Populating staged resource ancestor links...")
+
+    conn = op.get_bind()
+
+    # Get all resources and their children in batches
+    BATCH_SIZE = 500000
+
+    # Query resources in batches using yield_per
+    resources_query = sa.text(
+        """
+        SELECT urn, children
+        FROM stagedresource
+    """
+    )
+    resource_children = {}
+
+    # process resources and populate resource_children map in batches
+    # to limit memory usage
+    for batch in (
+        conn.execution_options(stream_results=True)
+        .execute(resources_query)
+        .yield_per(BATCH_SIZE)
+        .partitions()
+    ):
+        logger.info(f"Processing batch of {BATCH_SIZE} resources")
+
+        # Build resource -> children map for this batch
+        for result in batch:
+            urn = result.urn
+            children = result.children
+            if children:
+                resource_children[urn] = children
+            else:
+                # even if no children, we still add the resource to the map
+                # so that we maintain a record of all resources and can
+                # check for orphaned descendant records later
+                resource_children[urn] = []
+
+    # Build list of ancestor-descendant pairs
+    ancestor_links = []
+
+    def process_children(ancestor_urn, children, visited=None):
+        """Recursively process children and collect ancestor links"""
+        if visited is None:
+            visited = set()
+
+        for child_urn in children:
+            if child_urn not in visited:
+                visited.add(child_urn)
+                if child_urn not in resource_children:
+                    logger.warning(
+                        f"Found orphaned descendant URN: {child_urn}, not adding ancestor link"
+                    )
+                else:
+                    # Add direct ancestor link
+                    ancestor_links.append(
+                        {
+                            "id": f"srl_{uuid.uuid4()}",
+                            "ancestor_urn": ancestor_urn,
+                            "descendant_urn": child_urn,
+                        }
+                    )
+
+                    # Recursively process this child's children
+                    process_children(
+                        ancestor_urn, resource_children[child_urn], visited
+                    )
+
+    logger.info(
+        f"Recursively processing {len(resource_children)} resources for ancestor links in current batch"
+    )
+
+    # Process each resource's children recursively
+    for ancestor_urn, children in resource_children.items():
+        process_children(ancestor_urn, children)
+
+    # remove the resource_children map from memory
+    del resource_children
+
+    ancestor_links_count = len(ancestor_links)
+    logger.info(f"Found {ancestor_links_count} ancestor links in current batch")
+
+    if ancestor_links_count > 0:
+        # Create temporary CSV file
+        temp_csv_path = Path("staged_resource_ancestors.csv")
+        with open(temp_csv_path, "w", newline="") as csv_file:
+            writer = csv.DictWriter(
+                csv_file, fieldnames=["id", "ancestor_urn", "descendant_urn"]
+            )
+            writer.writeheader()
+            logger.info(f"Writing {ancestor_links_count} ancestor links to CSV file")
+            writer.writerows(ancestor_links)
+
+        del ancestor_links
+
+        # Copy all data from CSV file into table
+        logger.info(
+            f"Copying {ancestor_links_count} ancestor links from CSV file into stagedresourceancestor table..."
+        )
+        with open(temp_csv_path, "r") as csv_file:
+            copy_query = """
+                COPY stagedresourceancestor (id, ancestor_urn, descendant_urn)
+                FROM STDIN
+                WITH (FORMAT CSV, HEADER TRUE)
+            """
+            conn.connection.cursor().copy_expert(copy_query, csv_file)
+
+        # Clean up temp file
+        temp_csv_path.unlink()
+
+        logger.info(
+            f"Completed copying all ancestor links. Total ancestor links created: {ancestor_links_count}"
+        )
+
+    if ancestor_links_count < 1000000:
+
+        logger.info("Creating primary key index on stagedresourceancestor table...")
+
+        op.create_index(
+            "ix_staged_resource_ancestor_pkey",
+            "stagedresourceancestor",
+            ["id"],
+            unique=True,
+        )
+
+        logger.info("Completed creating primary key index")
+
+        logger.info("Creating foreign key constraints stagedresourceancestor table...")
+
+        op.create_foreign_key(
+            "fk_staged_resource_ancestor_ancestor",
+            "stagedresourceancestor",
+            "stagedresource",
+            ["ancestor_urn"],
+            ["urn"],
+            ondelete="CASCADE",
+        )
+
+        op.create_foreign_key(
+            "fk_staged_resource_ancestor_descendant",
+            "stagedresourceancestor",
+            "stagedresource",
+            ["descendant_urn"],
+            ["urn"],
+            ondelete="CASCADE",
+        )
+
+        logger.info("Completed creating foreign key constraints")
+
+        logger.info("Creating unique constraint on stagedresourceancestor table...")
+
+        op.create_unique_constraint(
+            "uq_staged_resource_ancestor",
+            "stagedresourceancestor",
+            ["ancestor_urn", "descendant_urn"],
+        )
+
+        logger.info("Completed creating unique constraint")
+
+        logger.info("Creating indexes on stagedresourceancestor table...")
+
+        op.create_index(
+            "ix_staged_resource_ancestor_ancestor",
+            "stagedresourceancestor",
+            ["ancestor_urn"],
+            unique=False,
+        )
+        op.create_index(
+            "ix_staged_resource_ancestor_descendant",
+            "stagedresourceancestor",
+            ["descendant_urn"],
+            unique=False,
+        )
+
+        logger.info("Completed creating indexes on stagedresourceancestor table")
+    else:
+        logger.info(
+            "Skipping creation of primary key index, foreign key constraints, unique constraint, and indexes on stagedresourceancestor table because there are more than 1,000,000 ancestor links. These will be populated as part of the `post_upgrade_index_creation.py` task kicked off during application startup."
+        )
+
+
+def downgrade():
+    logger.info(
+        "Downgrading staged resource ancestor link data migration, populating child_diff_statuses..."
+    )
+
+    # Get child diff statuses for each ancestor
+    conn = op.get_bind()
+    child_diff_statuses_query = """
+        UPDATE stagedresource
+        SET child_diff_statuses = child_statuses.status_map
+        FROM (
+            SELECT
+                stagedresourceancestor.ancestor_urn,
+                jsonb_object_agg(distinct(stagedresource.diff_status), True) as status_map
+            FROM stagedresourceancestor
+            JOIN stagedresource ON stagedresourceancestor.descendant_urn = stagedresource.urn
+            GROUP BY stagedresourceancestor.ancestor_urn
+        ) AS child_statuses
+        WHERE stagedresource.urn = child_statuses.ancestor_urn
+    """
+    result = conn.execute(child_diff_statuses_query)
+    updated_rows = result.rowcount
+
+    logger.info(
+        f"Downgraded staged resource ancestor link data migration, completed populating child_diff_statuses for {updated_rows} rows"
+    )
+
+    # Intentionally not dropping the stagedresourceancestor indexes here because they
+    # may not have been created as part of the data migration above. If they were not created,
+    # then the downgrade would fail trying to drop the non-existent indexes.
+
+    # The indexes and constraints will be dropped as part of the overall table drop that's done
+    # as part of the downgrade in the `5474a47c77da_create_staged_resource_ancestor_link_table.py`
+    # migration.

fides/api/alembic/migrations/versions/c586a56c25e7_remove_child_diff_statuses.py

@@ -0,0 +1,40 @@
+"""
+Removes the  stagedresource.child_diff_statuses column.
+
+This migration is run _after_ the data migration
+that populates the staged resource ancestor link table,
+to prevent unintended data loss.
+
+Revision ID: c586a56c25e7
+Revises: bf713b5a021d
+Create Date: 2025-06-03 09:47:11.389652
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "c586a56c25e7"
+down_revision = "bf713b5a021d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # remove the StagedResource.child_diff_statuses column as it's no longer needed
+    op.drop_column("stagedresource", "child_diff_statuses")
+
+
+def downgrade():
+    # re-add the StagedResource.child_diff_statuses column
+    op.add_column(
+        "stagedresource",
+        sa.Column(
+            "child_diff_statuses",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+    )

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -120,7 +120,7 @@ from fides.api.util.collection_util import Row
 from fides.api.util.endpoint_utils import validate_start_and_end_filters
 from fides.api.util.enums import ColumnSort
 from fides.api.util.fuzzy_search_utils import get_decrypted_identities_automaton
-from fides.api.util.storage_util import storage_json_encoder
+from fides.api.util.storage_util import StorageJSONEncoder
 from fides.common.api.scope_registry import (
     PRIVACY_REQUEST_CALLBACK_RESUME,
     PRIVACY_REQUEST_CREATE,
@@ -2129,7 +2129,7 @@ def get_test_privacy_request_results(
 
     # Escape datetime and ObjectId values
     raw_data = privacy_request.get_raw_access_results()
-    escaped_json = json.dumps(raw_data, indent=2, default=storage_json_encoder)
+    escaped_json = json.dumps(raw_data, indent=2, cls=StorageJSONEncoder)
     results = json.loads(escaped_json)
 
     filtered_results: Dict[str, Any] = filter_access_results(

fides/api/db/base.py

@@ -22,6 +22,9 @@ from fides.api.models.fides_cloud import FidesCloud
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.fides_user_invite import FidesUserInvite
 from fides.api.models.fides_user_permissions import FidesUserPermissions
+from fides.api.models.fides_user_respondent_email_verification import (
+    FidesUserRespondentEmailVerification,
+)
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
 from fides.api.models.manual_webhook import AccessManualWebhook

fides/api/main.py

@@ -33,6 +33,9 @@ from fides.api.common_exceptions import MalisciousUrlException
 from fides.api.cryptography.identity_salt import get_identity_salt
 from fides.api.middleware import handle_audit_log_resource
 from fides.api.migrations.hash_migration_job import initiate_bcrypt_migration_task
+from fides.api.migrations.post_upgrade_index_creation import (
+    initiate_post_upgrade_index_creation,
+)
 from fides.api.schemas.analytics import Event, ExtraData
 
 # pylint: disable=wildcard-import, unused-wildcard-import
@@ -97,6 +100,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
     initiate_bcrypt_migration_task()
+    initiate_post_upgrade_index_creation()
 
     logger.debug("Sending startup analytics events...")
     # Avoid circular imports
@@ -240,7 +244,6 @@ async def log_request(request: Request, call_next: Callable) -> Response:
         status_code=response.status_code,
         handler_time=f"{total_time}ms",
         path=request.url.path,
-        fides_client=request.headers.get("Fides-Client", "unknown"),
     ).info("Request received")
     return response