ethyca-fides 2.71.1b0__py2.py3-none-any.whl → 2.71.1rc0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-29T16:40:09-0600",
+ "date": "2025-10-03T10:03:48-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "830e00751b37962f36c99b0d915abf68e75325a8",
- "version": "2.71.1b0"
+ "full-revisionid": "43bcd93b9c3859ad4a79f3825f1ebc789a58caf3",
+ "version": "2.71.1rc0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/3efe14d4469a_adds_new_experience_configs_for_vendor_.py

@@ -0,0 +1,79 @@
+"""adds new experience configs for vendor asset disclosure
+
+Revision ID: 3efe14d4469a
+Revises: f108fa05c579
+Create Date: 2025-09-09 11:19:24.060587
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "3efe14d4469a"
+down_revision = "f108fa05c579"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "experienceconfigtemplate",
+        sa.Column(
+            "asset_disclosure_include_types",
+            postgresql.ARRAY(sa.String()),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "experienceconfigtemplate",
+        sa.Column(
+            "allow_vendor_asset_disclosure",
+            sa.Boolean(),
+            server_default="f",
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfig",
+        sa.Column(
+            "asset_disclosure_include_types",
+            postgresql.ARRAY(sa.String()),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfig",
+        sa.Column(
+            "allow_vendor_asset_disclosure",
+            sa.Boolean(),
+            server_default="f",
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfighistory",
+        sa.Column("allow_vendor_asset_disclosure", sa.Boolean(), nullable=True),
+    )
+    op.add_column(
+        "privacyexperienceconfighistory",
+        sa.Column(
+            "asset_disclosure_include_types",
+            postgresql.ARRAY(sa.String()),
+            nullable=True,
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("privacyexperienceconfighistory", "asset_disclosure_include_types")
+    op.drop_column("privacyexperienceconfighistory", "allow_vendor_asset_disclosure")
+    op.drop_column("privacyexperienceconfig", "allow_vendor_asset_disclosure")
+    op.drop_column("privacyexperienceconfig", "asset_disclosure_include_types")
+    op.drop_column("experienceconfigtemplate", "allow_vendor_asset_disclosure")
+    op.drop_column("experienceconfigtemplate", "asset_disclosure_include_types")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/4bfbeff34611_add_polling_status.py

@@ -0,0 +1,35 @@
+"""add polling status
+
+Revision ID: 4bfbeff34611
+Revises: 7db29f9cd77b
+Create Date: 2025-09-20 23:02:45.550170
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4bfbeff34611"
+down_revision = "7db29f9cd77b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Check if value already exists
+    connection = op.get_bind()
+    result = connection.execute(
+        sa.text(
+            "SELECT 1 FROM pg_enum WHERE enumlabel = 'polling' "
+            "AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'executionlogstatus')"
+        )
+    )
+
+    if not result.fetchone():
+        op.execute("ALTER TYPE executionlogstatus ADD VALUE 'polling'")
+
+
+def downgrade():
+    # The 'polling' value will remain in the enum but won't be used by older app versions
+    pass

fides/api/alembic/migrations/versions/7db29f9cd77b_create_new_sub_request_table.py

@@ -0,0 +1,95 @@
+"""Create new Sub Request Table
+
+Revision ID: 7db29f9cd77b
+Revises: b97e92b038d2
+Create Date: 2025-09-16 14:00:16.282996
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy_utils.types.encrypted.encrypted_type import (
+    AesGcmEngine,
+    StringEncryptedType,
+)
+
+from fides.api.db.base_class import JSONTypeOverride
+from fides.config import CONFIG
+
+# revision identifiers, used by Alembic.
+revision = "7db29f9cd77b"
+down_revision = "b97e92b038d2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "request_task_sub_request",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("request_task_id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "param_values",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=False,
+        ),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column(
+            "access_data",
+            StringEncryptedType(
+                type_in=JSONTypeOverride,
+                key=CONFIG.security.app_encryption_key,
+                engine=AesGcmEngine,
+                padding="pkcs5",
+            ),
+            nullable=True,
+        ),
+        sa.Column("rows_masked", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["request_task_id"],
+            ["requesttask.id"],
+            name="request_task_sub_request_request_task_id_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_id"),
+        "request_task_sub_request",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        "request_task_sub_request",
+        ["request_task_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_request_task_sub_request_request_task_id"),
+        table_name="request_task_sub_request",
+    )
+    op.drop_index(
+        op.f("ix_request_task_sub_request_id"), table_name="request_task_sub_request"
+    )
+    op.drop_table("request_task_sub_request")

fides/api/alembic/migrations/versions/9caf76161e55_make_user_assigned_data_uses_nullable_.py

@@ -0,0 +1,64 @@
+"""Update user_assigned_data_uses so it's nullable and defaults to none
+
+Revision ID: 9caf76161e55
+Revises: 918aefc950c9
+Create Date: 2025-09-29 14:14:37.837616
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "9caf76161e55"
+down_revision = "918aefc950c9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "stagedresource",
+        "user_assigned_data_uses",
+        existing_type=postgresql.ARRAY(sa.VARCHAR()),
+        nullable=True,
+        server_default=None,
+        default=None,
+        existing_server_default=sa.text("'{}'::character varying[]"),
+    )
+
+    # Update web monitor resources with empty array to null
+    op.execute(
+        """
+        UPDATE stagedresource
+        SET user_assigned_data_uses = NULL
+        WHERE
+          resource_type IN ('Cookie', 'Browser request', 'Image', 'iFrame', 'Javascript tag')
+          AND user_assigned_data_uses = '{}'
+        """
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute(
+        """
+        UPDATE stagedresource
+        SET user_assigned_data_uses = '{}'
+        WHERE user_assigned_data_uses IS NULL
+        """
+    )
+
+    op.alter_column(
+        "stagedresource",
+        "user_assigned_data_uses",
+        existing_type=postgresql.ARRAY(sa.VARCHAR()),
+        nullable=False,
+        server_default=sa.text("'{}'::character varying[]"),
+        existing_server_default=None,
+        default=dict,
+    )
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/b97e92b038d2_add_digest_execution_model.py

@@ -0,0 +1,117 @@
+"""add digest execution model
+
+Revision ID: b97e92b038d2
+Revises: 3efe14d4469a
+Create Date: 2025-10-01 16:42:41.900651
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b97e92b038d2"
+down_revision = "3efe14d4469a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "digest_task_execution",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("action_type", sa.String(), nullable=False),
+        sa.Column("digest_config_id", sa.String(), nullable=False),
+        sa.Column("celery_task_id", sa.String(), nullable=True),
+        sa.Column(
+            "status",
+            postgresql.ENUM(name="executionlogstatus", create_type=False),
+            nullable=False,
+        ),
+        sa.Column("total_recipients", sa.Integer(), nullable=True),
+        sa.Column("processed_recipients", sa.Integer(), nullable=False),
+        sa.Column("successful_communications", sa.Integer(), nullable=False),
+        sa.Column("failed_communications", sa.Integer(), nullable=False),
+        sa.Column(
+            "execution_state", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column(
+            "processed_user_ids", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_checkpoint_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["digest_config_id"], ["digest_config.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_action_type"),
+        "digest_task_execution",
+        ["action_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        "digest_task_execution",
+        ["celery_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        "digest_task_execution",
+        ["digest_config_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_id"),
+        "digest_task_execution",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_task_execution_status"),
+        "digest_task_execution",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_digest_task_execution_status"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_id"), table_name="digest_task_execution"
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_digest_config_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_celery_task_id"),
+        table_name="digest_task_execution",
+    )
+    op.drop_index(
+        op.f("ix_digest_task_execution_action_type"), table_name="digest_task_execution"
+    )
+    op.drop_table("digest_task_execution")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/f108fa05c579_adds_optional_duration_field_to_assets.py

@@ -0,0 +1,28 @@
+"""adds optional duration field to assets
+
+Revision ID: f108fa05c579
+Revises: 9caf76161e55
+Create Date: 2025-08-22 12:47:31.374493
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f108fa05c579"
+down_revision = "9caf76161e55"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("asset", sa.Column("duration", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("asset", "duration")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/generic_overrides.py

@@ -54,7 +54,6 @@ from fides.common.api.v1.urn_registry import DATASETS_CLEAN, V1_URL_PREFIX
 from fides.service.dataset.dataset_service import (
     DatasetNotFoundException,
     DatasetService,
-    LinkedDatasetException,
 )
 from fides.service.taxonomy.taxonomy_service import TaxonomyService
 
@@ -127,7 +126,7 @@ async def update_dataset(
     except DatasetNotFoundException as e:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
-            detail=str(e),
+            detail={"message": str(e)},
         )
 
 
@@ -249,7 +248,7 @@ async def get_dataset(
     except DatasetNotFoundException as e:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
-            detail=str(e),
+            detail={"message": str(e)},
         )
 
 
@@ -276,12 +275,7 @@ async def delete_dataset(
     except DatasetNotFoundException as e:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
-            detail=str(e),
-        )
-    except LinkedDatasetException as e:
-        raise HTTPException(
-            status_code=HTTP_400_BAD_REQUEST,
-            detail=str(e),
+            detail={"message": str(e)},
         )
 
 

fides/api/common_exceptions.py

@@ -179,6 +179,10 @@ class AwaitingAsyncTask(BaseException):
     """Request Task is Awaiting Processing - Awaiting Async Task"""
 
 
+class AwaitingAsyncProcessing(BaseException):
+    """Request Task is actively being processed by external system - Fides is polling"""
+
+
 class UpstreamTasksNotReady(BaseException):
     """Privacy Request Task awaiting upstream tasks"""
 

fides/api/main.py

@@ -45,9 +45,9 @@ from fides.api.service.privacy_request.email_batch_service import (
     initiate_scheduled_batch_email_send,
 )
 from fides.api.service.privacy_request.request_service import (
-    initiate_async_tasks_status_polling,
     initiate_interrupted_task_requeue_poll,
     initiate_poll_for_exited_privacy_request_tasks,
+    initiate_polling_task_requeue,
     initiate_scheduled_dsr_data_removal,
 )
 
@@ -103,7 +103,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_poll_for_exited_privacy_request_tasks()
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
-    initiate_async_tasks_status_polling()
+    initiate_polling_task_requeue()
     initiate_bcrypt_migration_task()
     initiate_post_upgrade_index_creation()
 

fides/api/models/asset.py

@@ -17,7 +17,7 @@ from sqlalchemy import (
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.mutable import MutableDict
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, selectinload
 
 from fides.api.db.base_class import Base
 from fides.api.db.util import EnumColumn
@@ -47,6 +47,7 @@ class Asset(Base):
     name = Column(String, index=True, nullable=False)
     asset_type = Column(String, index=True, nullable=False)
     domain = Column(String, index=True)
+    duration = Column(String, nullable=True)
     parent = Column(ARRAY(String), server_default="{}", nullable=False)
     parent_domain = Column(String)
     locations = Column(ARRAY(String), server_default="{}", nullable=False)
@@ -190,5 +191,17 @@ class Asset(Base):
                 .where(System.fides_key == system_fides_key)
             )
 
+        # Explicitly eager load the `system` relationship to make this query's
+        # performance predictable and prevent N+1 issues.
+        query = query.options(selectinload(cls.system))  # type: ignore[attr-defined]
         result = await async_session.execute(query)
         return result.scalars().all()
+
+    # Expose related System attributes for API serialization convenience
+    @property
+    def system_name(self) -> Optional[str]:
+        return self.system.name if self.system else None
+
+    @property
+    def system_fides_key(self) -> Optional[str]:
+        return self.system.fides_key if self.system else None

fides/api/models/attachment.py

@@ -52,6 +52,7 @@ class AttachmentReferenceType(str, EnumType):
     privacy_request = "privacy_request"
     comment = "comment"
     manual_task_submission = "manual_task_submission"
+    request_task = "request_task"
 
 
 class AttachmentReference(Base):

fides/api/models/detection_discovery/core.py

@@ -60,6 +60,56 @@ class MonitorFrequency(Enum):
 QUARTERLY_MONTH_PATTERN = r"^\d+,\d+,\d+,\d+$"
 
 
+class StagedResourceType(str, Enum):
+    """
+    Enum representing the type of staged resource.
+    The resource_type column is a string in the DB, this is just for
+    application-level use.
+    """
+
+    # Note: If you add a new resource type, make sure to update either
+    # get_datastore_resource_types or get_website_monitor_resource_types
+
+    # Datastore staged resources
+    DATABASE = "Database"
+    SCHEMA = "Schema"
+    TABLE = "Table"
+    FIELD = "Field"
+    ENDPOINT = "Endpoint"
+    # Website monitor staged resources
+    COOKIE = "Cookie"
+    BROWSER_REQUEST = "Browser request"
+    IMAGE_BROWSER_REQUEST = "Image"
+    IFRAME_BROWSER_REQUEST = "iFrame"
+    JAVASCRIPT_BROWSER_REQUEST = "Javascript tag"
+
+    @staticmethod
+    def get_datastore_resource_types() -> List["StagedResourceType"]:
+        return [
+            StagedResourceType.DATABASE,
+            StagedResourceType.SCHEMA,
+            StagedResourceType.TABLE,
+            StagedResourceType.FIELD,
+            StagedResourceType.ENDPOINT,
+        ]
+
+    @staticmethod
+    def get_website_monitor_resource_types() -> List["StagedResourceType"]:
+        return [
+            StagedResourceType.COOKIE,
+            StagedResourceType.BROWSER_REQUEST,
+            StagedResourceType.IMAGE_BROWSER_REQUEST,
+            StagedResourceType.IFRAME_BROWSER_REQUEST,
+            StagedResourceType.JAVASCRIPT_BROWSER_REQUEST,
+        ]
+
+    def is_datastore_resource(self) -> bool:
+        return self in self.get_datastore_resource_types()
+
+    def is_website_monitor_resource(self) -> bool:
+        return self in self.get_website_monitor_resource_types()
+
+
 class SharedMonitorConfig(Base, FidesBase):
     """SQL model for shareable monitor configurations"""
 
@@ -499,11 +549,15 @@ class StagedResource(Base):
         server_default="{}",
         default=dict,
     )
+    # This field is intentionally nullable to distinguish the case when no user assigned data uses
+    # have been set (default, value is None) from the case in which they have been explicitly set
+    # as empty (value is empty array). This allows users to remove all data uses from a StagedResource,
+    # which was not possible when this field was not nullable.
     user_assigned_data_uses = Column(
         ARRAY(String),
-        nullable=False,
-        server_default="{}",
-        default=dict,
+        nullable=True,
+        server_default=None,
+        default=None,
     )
     user_assigned_system_id = Column(String, nullable=True, index=True)
 

fides/api/models/digest/__init__.py

@@ -5,10 +5,12 @@ from fides.api.models.digest.conditional_dependencies import (
     DigestConditionType,
 )
 from fides.api.models.digest.digest_config import DigestConfig, DigestType
+from fides.api.models.digest.digest_execution import DigestTaskExecution
 
 __all__ = [
     "DigestConfig",
     "DigestType",
     "DigestCondition",
     "DigestConditionType",
+    "DigestTaskExecution",
 ]

fides/api/models/digest/digest_config.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 
 from sqlalchemy import Boolean, Column, DateTime, String, Text
 from sqlalchemy.dialects.postgresql import JSONB
@@ -19,6 +19,9 @@ from fides.api.task.conditional_dependencies.schemas import (
     ConditionLeaf,
 )
 
+if TYPE_CHECKING:
+    from fides.api.models.digest.digest_execution import DigestTaskExecution
+
 
 class DigestType(str, Enum):
     """Types of digests that can be configured."""
@@ -60,6 +63,12 @@ class DigestConfig(Base):
         back_populates="digest_config",
         cascade="all, delete-orphan",
     )
+    executions = relationship(
+        "DigestTaskExecution",
+        back_populates="digest_config",
+        cascade="all, delete-orphan",
+        order_by="DigestTaskExecution.created_at.desc()",
+    )
 
     def get_receiver_condition(
         self, db: Session