ethyca-fides 2.71.0rc3__py2.py3-none-any.whl → 2.71.1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-27T00:39:42-0600",
+ "date": "2025-10-06T14:27:23+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "6f0d8440f6dadbb3a395d768cfd325cba333cd99",
- "version": "2.71.0rc3"
+ "full-revisionid": "8791bbbc6f37550a8dd4c6e68612f06de9ecc45a",
+ "version": "2.71.1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/3efe14d4469a_adds_new_experience_configs_for_vendor_.py

@@ -0,0 +1,79 @@
+"""adds new experience configs for vendor asset disclosure
+
+Revision ID: 3efe14d4469a
+Revises: f108fa05c579
+Create Date: 2025-09-09 11:19:24.060587
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "3efe14d4469a"
+down_revision = "f108fa05c579"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "experienceconfigtemplate",
+        sa.Column(
+            "asset_disclosure_include_types",
+            postgresql.ARRAY(sa.String()),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "experienceconfigtemplate",
+        sa.Column(
+            "allow_vendor_asset_disclosure",
+            sa.Boolean(),
+            server_default="f",
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfig",
+        sa.Column(
+            "asset_disclosure_include_types",
+            postgresql.ARRAY(sa.String()),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfig",
+        sa.Column(
+            "allow_vendor_asset_disclosure",
+            sa.Boolean(),
+            server_default="f",
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfighistory",
+        sa.Column("allow_vendor_asset_disclosure", sa.Boolean(), nullable=True),
+    )
+    op.add_column(
+        "privacyexperienceconfighistory",
+        sa.Column(
+            "asset_disclosure_include_types",
+            postgresql.ARRAY(sa.String()),
+            nullable=True,
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("privacyexperienceconfighistory", "asset_disclosure_include_types")
+    op.drop_column("privacyexperienceconfighistory", "allow_vendor_asset_disclosure")
+    op.drop_column("privacyexperienceconfig", "allow_vendor_asset_disclosure")
+    op.drop_column("privacyexperienceconfig", "asset_disclosure_include_types")
+    op.drop_column("experienceconfigtemplate", "allow_vendor_asset_disclosure")
+    op.drop_column("experienceconfigtemplate", "asset_disclosure_include_types")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/4bfbeff34611_add_polling_status.py

@@ -0,0 +1,68 @@
+"""add polling status
+
+Revision ID: 4bfbeff34611
+Revises: 7db29f9cd77b
+Create Date: 2025-09-20 23:02:45.550170
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4bfbeff34611"
+down_revision = "7db29f9cd77b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TYPE executionlogstatus ADD VALUE 'polling'")
+
+
+def downgrade():
+    # Remove any records that have the 'polling' enum value before dropping it
+    # Fallback to 'paused' (awaiting_processing) for backward compatibility
+
+    op.execute("UPDATE requesttask SET status = 'paused' WHERE status = 'polling'")
+    op.execute("UPDATE executionlog SET status = 'paused' WHERE status = 'polling'")
+    op.execute(
+        "UPDATE digest_task_execution SET status = 'paused' WHERE status = 'polling'"
+    )
+    op.execute("UPDATE monitortask SET status = 'paused' WHERE status = 'polling'")
+    op.execute(
+        "UPDATE monitortaskexecutionlog SET status = 'paused' WHERE status = 'polling'"
+    )
+
+    # Recreate the enum without the 'polling' value
+    op.execute("ALTER TYPE executionlogstatus RENAME TO executionlogstatus_old")
+    op.execute(
+        "CREATE TYPE executionlogstatus AS ENUM ("
+        "'in_processing', 'pending', 'complete', 'error', 'paused', 'retrying', 'skipped'"
+        ")"
+    )
+
+    # Update all tables that use the enum type
+    op.execute(
+        "ALTER TABLE requesttask ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE executionlog ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE digest_task_execution ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE monitortask ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+    op.execute(
+        "ALTER TABLE monitortaskexecutionlog ALTER COLUMN status TYPE executionlogstatus USING "
+        "status::text::executionlogstatus"
+    )
+
+    # Drop the old enum type
+    op.execute("DROP TYPE executionlogstatus_old")

fides/api/alembic/migrations/versions/7db29f9cd77b_create_new_sub_request_table.py

@@ -0,0 +1,95 @@
+"""Create new Sub Request Table
+
+Revision ID: 7db29f9cd77b
+Revises: b97e92b038d2
+Create Date: 2025-09-16 14:00:16.282996
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy_utils.types.encrypted.encrypted_type import (
+    AesGcmEngine,
+    StringEncryptedType,
+)
+
+from fides.api.db.base_class import JSONTypeOverride
+from fides.config import CONFIG
+
+# revision identifiers, used by Alembic.
+revision = "7db29f9cd77b"
+down_revision = "b97e92b038d2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "request_task_sub_request",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("request_task_id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "param_values",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=False,
+        ),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column(
+            "access_data",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=True,
+        ),
+        sa.Column("rows_masked", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["request_task_id"],
+            ["requesttask.id"],
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_id"),
+        "request_task_sub_request",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        "request_task_sub_request",
+        ["request_task_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        table_name="request_task_sub_request",
+    )
+    op.drop_index(
+        op.f("ix_request_task_sub_request_id"), table_name="request_task_sub_request"
+    )
+    op.drop_table("request_task_sub_request")

fides/api/alembic/migrations/versions/918aefc950c9_create_digest_conditional_dependencies.py

@@ -0,0 +1,125 @@
+"""create digest conditional dependencies
+
+Revision ID: 918aefc950c9
+Revises: e2cda8d6abc3
+Create Date: 2025-09-18 21:04:23.620675
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "918aefc950c9"
+down_revision = "e2cda8d6abc3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+
+    op.create_table(
+        "digest_condition",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column("digest_config_id", sa.String(length=255), nullable=False),
+        sa.Column("parent_id", sa.String(length=255), nullable=True),
+        sa.Column("digest_condition_type", sa.String(), nullable=False),
+        sa.Column("condition_type", sa.String(), nullable=False),
+        sa.Column("field_address", sa.String(length=255), nullable=True),
+        sa.Column("operator", sa.String(), nullable=True),
+        sa.Column("value", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.Column("logical_operator", sa.String(), nullable=True),
+        sa.Column("sort_order", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["digest_config_id"], ["digest_config.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent_id"], ["digest_condition.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+    op.create_index(
+        "ix_digest_condition_condition_type",
+        "digest_condition",
+        ["condition_type"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_digest_condition_digest_condition_type",
+        "digest_condition",
+        ["digest_condition_type"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_digest_condition_digest_config_id",
+        "digest_condition",
+        ["digest_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_digest_condition_parent_id",
+        "digest_condition",
+        ["parent_id"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_digest_condition_sort_order",
+        "digest_condition",
+        ["sort_order"],
+        unique=False,
+    )
+
+    # Unique constraint to ensure only one root condition per digest_condition_type per digest_config
+    op.create_index(
+        "ix_digest_condition_unique_root_per_type",
+        "digest_condition",
+        ["digest_config_id", "digest_condition_type"],
+        unique=True,
+        postgresql_where=sa.text("parent_id IS NULL"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+
+    op.drop_index(
+        "ix_digest_condition_unique_root_per_type",
+        table_name="digest_condition",
+    )
+    op.drop_index(
+        "ix_digest_condition_sort_order",
+        table_name="digest_condition",
+    )
+    op.drop_index(
+        "ix_digest_condition_parent_id",
+        table_name="digest_condition",
+    )
+    op.drop_index(
+        "ix_digest_condition_digest_config_id",
+        table_name="digest_condition",
+    )
+    op.drop_index(
+        "ix_digest_condition_digest_condition_type",
+        table_name="digest_condition",
+    )
+    op.drop_index(
+        "ix_digest_condition_condition_type",
+        table_name="digest_condition",
+    )
+
+    op.drop_table("digest_condition")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/9caf76161e55_make_user_assigned_data_uses_nullable_.py

@@ -0,0 +1,64 @@
+"""Update user_assigned_data_uses so it's nullable and defaults to none
+
+Revision ID: 9caf76161e55
+Revises: 918aefc950c9
+Create Date: 2025-09-29 14:14:37.837616
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "9caf76161e55"
+down_revision = "918aefc950c9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "stagedresource",
+        "user_assigned_data_uses",
+        existing_type=postgresql.ARRAY(sa.VARCHAR()),
+        nullable=True,
+        server_default=None,
+        default=None,
+        existing_server_default=sa.text("'{}'::character varying[]"),
+    )
+
+    # Update web monitor resources with empty array to null
+    op.execute(
+        """
+        UPDATE stagedresource
+        SET user_assigned_data_uses = NULL
+        WHERE
+          resource_type IN ('Cookie', 'Browser request', 'Image', 'iFrame', 'Javascript tag')
+          AND user_assigned_data_uses = '{}'
+        """
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute(
+        """
+        UPDATE stagedresource
+        SET user_assigned_data_uses = '{}'
+        WHERE user_assigned_data_uses IS NULL
+        """
+    )
+
+    op.alter_column(
+        "stagedresource",
+        "user_assigned_data_uses",
+        existing_type=postgresql.ARRAY(sa.VARCHAR()),
+        nullable=False,
+        server_default=sa.text("'{}'::character varying[]"),
+        existing_server_default=None,
+        default=dict,
+    )
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/b97e92b038d2_add_digest_execution_model.py

@@ -0,0 +1,117 @@
+"""add digest execution model
+
+Revision ID: b97e92b038d2
+Revises: 3efe14d4469a
+Create Date: 2025-10-01 16:42:41.900651
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b97e92b038d2"
+down_revision = "3efe14d4469a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "digest_task_execution",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("action_type", sa.String(), nullable=False),
+        sa.Column("digest_config_id", sa.String(), nullable=False),
+        sa.Column("celery_task_id", sa.String(), nullable=True),
+        sa.Column(
+            "status",
+            postgresql.ENUM(name="executionlogstatus", create_type=False),
+            nullable=False,
+        ),
+        sa.Column("total_recipients", sa.Integer(), nullable=True),
+        sa.Column("processed_recipients", sa.Integer(), nullable=False),
+        sa.Column("successful_communications", sa.Integer(), nullable=False),
+        sa.Column("failed_communications", sa.Integer(), nullable=False),
+        sa.Column(
+            "execution_state", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column(
+            "processed_user_ids", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_checkpoint_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["digest_config_id"], ["digest_config.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_action_type"),
+        "digest_task_execution",
+        ["action_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        "digest_task_execution",
+        ["celery_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        "digest_task_execution",
+        ["digest_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_id"),
+        "digest_task_execution",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_status"),
+        "digest_task_execution",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_digest_task_execution_status"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_id"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_action_type"), table_name="digest_task_execution"
+    )
+    op.drop_table("digest_task_execution")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/f108fa05c579_adds_optional_duration_field_to_assets.py

@@ -0,0 +1,28 @@
+"""adds optional duration field to assets
+
+Revision ID: f108fa05c579
+Revises: 9caf76161e55
+Create Date: 2025-08-22 12:47:31.374493
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f108fa05c579"
+down_revision = "9caf76161e55"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("asset", sa.Column("duration", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("asset", "duration")
+    # ### end Alembic commands ###

fides/api/common_exceptions.py

@@ -179,6 +179,10 @@ class AwaitingAsyncTask(BaseException):
     """Request Task is Awaiting Processing - Awaiting Async Task"""
 
 
+class AwaitingAsyncProcessing(BaseException):
+    """Request Task is actively being processed by external system - Fides is polling"""
+
+
 class UpstreamTasksNotReady(BaseException):
     """Privacy Request Task awaiting upstream tasks"""
 

fides/api/db/base.py

@@ -23,7 +23,7 @@ from fides.api.models.detection_discovery.monitor_task import (
     MonitorTaskExecutionLog,
 )
 from fides.api.models.detection_discovery.web_monitor import WebMonitorGroupJob
-from fides.api.models.digest import DigestConfig
+from fides.api.models.digest import DigestCondition, DigestConfig
 from fides.api.models.event_audit import EventAudit
 from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.fides_cloud import FidesCloud

fides/api/main.py

@@ -45,9 +45,9 @@ from fides.api.service.privacy_request.email_batch_service import (
     initiate_scheduled_batch_email_send,
 )
 from fides.api.service.privacy_request.request_service import (
-    initiate_async_tasks_status_polling,
     initiate_interrupted_task_requeue_poll,
     initiate_poll_for_exited_privacy_request_tasks,
+    initiate_polling_task_requeue,
     initiate_scheduled_dsr_data_removal,
 )
 
@@ -103,7 +103,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_poll_for_exited_privacy_request_tasks()
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
-    initiate_async_tasks_status_polling()
+    initiate_polling_task_requeue()
     initiate_bcrypt_migration_task()
     initiate_post_upgrade_index_creation()
 

fides/api/models/asset.py

@@ -17,7 +17,7 @@ from sqlalchemy import (
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.mutable import MutableDict
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, selectinload
 
 from fides.api.db.base_class import Base
 from fides.api.db.util import EnumColumn
@@ -47,6 +47,7 @@ class Asset(Base):
     name = Column(String, index=True, nullable=False)
     asset_type = Column(String, index=True, nullable=False)
     domain = Column(String, index=True)
+    duration = Column(String, nullable=True)
     parent = Column(ARRAY(String), server_default="{}", nullable=False)
     parent_domain = Column(String)
     locations = Column(ARRAY(String), server_default="{}", nullable=False)
@@ -190,5 +191,17 @@ class Asset(Base):
                 .where(System.fides_key == system_fides_key)
             )
 
+        # Explicitly eager load the `system` relationship to make this query's
+        # performance predictable and prevent N+1 issues.
+        query = query.options(selectinload(cls.system))  # type: ignore[attr-defined]
         result = await async_session.execute(query)
         return result.scalars().all()
+
+    # Expose related System attributes for API serialization convenience
+    @property
+    def system_name(self) -> Optional[str]:
+        return self.system.name if self.system else None
+
+    @property
+    def system_fides_key(self) -> Optional[str]:
+        return self.system.fides_key if self.system else None

fides/api/models/attachment.py

@@ -52,6 +52,7 @@ class AttachmentReferenceType(str, EnumType):
     privacy_request = "privacy_request"
     comment = "comment"
     manual_task_submission = "manual_task_submission"
+    request_task = "request_task"
 
 
 class AttachmentReference(Base):