ethyca-fides 2.73.2b0__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-30T20:40:29+0000",
+ "date": "2025-11-05T17:23:50-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "b17dcbb16dfac3ce994a9bd97fd39d126ec66d9f",
- "version": "2.73.2b0"
+ "full-revisionid": "dd2f04e2b4b8386e585ebaf9cd935173dc333b11",
+ "version": "2.74.0rc1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/xx_2025_10_29_1659_80d28dea3b6b_added_duplicate_group_table.py

@@ -0,0 +1,79 @@
+"""added duplicate group table
+
+Revision ID: 80d28dea3b6b
+Revises: c09e76282dd1
+Create Date: 2025-10-29 16:59:09.551676
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "80d28dea3b6b"
+down_revision = "c09e76282dd1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "duplicate_group",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("rule_version", sa.Text(), nullable=False),
+        sa.Column("is_active", sa.Boolean(), server_default="t", nullable=False),
+        sa.Column("dedup_key", sa.Text(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.drop_index(
+        "ix_privacyrequest_duplicate_request_group_id", table_name="privacyrequest"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column(
+            "duplicate_request_group_id", postgresql.UUID(as_uuid=True), nullable=True
+        ),
+    )
+    op.create_foreign_key(
+        "fk_privacyrequest_duplicate_group",
+        "privacyrequest",
+        "duplicate_group",
+        ["duplicate_request_group_id"],
+        ["id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "fk_privacyrequest_duplicate_group", "privacyrequest", type_="foreignkey"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column("duplicate_request_group_id", sa.String(), nullable=True),
+    )
+    op.create_index(
+        "ix_privacyrequest_duplicate_request_group_id",
+        "privacyrequest",
+        ["duplicate_request_group_id"],
+        unique=False,
+    )
+    op.drop_table("duplicate_group")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/xx_2025_11_05_0200_f1a2b3c4d5e6_create_staged_resource_error_table.py

@@ -0,0 +1,82 @@
+"""Create staged resource error table
+
+Revision ID: f1a2b3c4d5e6
+Revises: 80d28dea3b6b
+Create Date: 2025-11-05 02:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f1a2b3c4d5e6"
+down_revision = "80d28dea3b6b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Create the stagedresourceerror table
+    op.create_table(
+        "stagedresourceerror",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "staged_resource_urn",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("message", sa.String(), nullable=False),
+        sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("error_type", sa.String(), nullable=False),
+        sa.Column("diff_status", sa.String(), nullable=False),
+        sa.Column("task_id", sa.String(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["staged_resource_urn"],
+            ["stagedresource.urn"],
+            name="stagedresourceerror_staged_resource_urn_fkey",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+    op.create_index(
+        "ix_stagedresourceerror_staged_resource_urn",
+        "stagedresourceerror",
+        ["staged_resource_urn"],
+        unique=False,
+    )
+
+    op.create_index(
+        "ix_stagedresourceerror_task_id",
+        "stagedresourceerror",
+        ["task_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    # Drop the indices
+    op.drop_index(
+        "ix_stagedresourceerror_task_id",
+        table_name="stagedresourceerror",
+    )
+    op.drop_index(
+        "ix_stagedresourceerror_staged_resource_urn",
+        table_name="stagedresourceerror",
+    )
+
+    # Drop the stagedresourceerror table
+    op.drop_table("stagedresourceerror")

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -91,6 +91,7 @@ from fides.api.schemas.privacy_request import (
     BulkPostPrivacyRequests,
     BulkReviewResponse,
     BulkSoftDeletePrivacyRequests,
+    CancelPrivacyRequests,
     CheckpointActionRequired,
     DenyPrivacyRequests,
     ExecutionLogDetailResponse,
@@ -147,6 +148,7 @@ from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_BULK_FINALIZE,
     PRIVACY_REQUEST_BULK_RETRY,
     PRIVACY_REQUEST_BULK_SOFT_DELETE,
+    PRIVACY_REQUEST_CANCEL,
     PRIVACY_REQUEST_DENY,
     PRIVACY_REQUEST_FILTERED_RESULTS,
     PRIVACY_REQUEST_FINALIZE,
@@ -486,7 +488,7 @@ def _filter_privacy_request_queryset(
     errored_gt: Optional[datetime] = None,
     external_id: Optional[str] = None,
     location: Optional[str] = None,
-    action_type: Optional[ActionType] = None,
+    action_type: Optional[Union[ActionType, List[ActionType]]] = None,
     include_consent_webhook_requests: Optional[bool] = False,
     include_deleted_requests: Optional[bool] = False,
 ) -> Query:
@@ -676,9 +678,11 @@ def _filter_privacy_request_queryset(
             PrivacyRequest.finished_processing_at > errored_gt,
         )
     if action_type:
+        if isinstance(action_type, ActionType):
+            action_type = [action_type]
         policy_ids_for_action_type = (
             db.query(Rule)
-            .filter(Rule.action_type == action_type)
+            .filter(Rule.action_type.in_(action_type))
             .with_entities(Rule.policy_id)
             .distinct()
         )
@@ -792,7 +796,7 @@ def _shared_privacy_request_search(
     errored_gt: Optional[datetime] = None,
     external_id: Optional[str] = None,
     location: Optional[str] = None,
-    action_type: Optional[ActionType] = None,
+    action_type: Optional[Union[ActionType, List[ActionType]]] = None,
     verbose: Optional[bool] = False,
     include_identities: Optional[bool] = False,
     include_custom_privacy_request_fields: Optional[bool] = False,
@@ -1357,10 +1361,10 @@ def restart_privacy_request_from_failure(
     )
 
     # Automatically resubmit the request if the cache has expired
+
     if (
-        not privacy_request.get_cached_identity_data()
-        and privacy_request.status
-        not in [PrivacyRequestStatus.complete, PrivacyRequestStatus.pending]
+        not privacy_request.verify_cache_for_identity_data()
+        and privacy_request.status != PrivacyRequestStatus.complete
     ):
         logger.info(
             f"Cached data for privacy request {privacy_request.id} has expired, automatically resubmitting request"
@@ -1477,6 +1481,29 @@ def deny_privacy_request(
     )
 
 
+@router.patch(
+    PRIVACY_REQUEST_CANCEL,
+    status_code=HTTP_200_OK,
+    response_model=BulkReviewResponse,
+)
+def cancel_privacy_request(
+    *,
+    client: ClientDetail = Security(
+        verify_oauth_client,
+        scopes=[PRIVACY_REQUEST_REVIEW],
+    ),
+    privacy_request_service: PrivacyRequestService = Depends(
+        get_privacy_request_service
+    ),
+    privacy_requests: CancelPrivacyRequests,
+) -> BulkReviewResponse:
+    """Cancel a list of privacy requests and/or report failure"""
+
+    return privacy_request_service.cancel_privacy_requests(
+        privacy_requests.request_ids, privacy_requests.reason, user_id=client.user_id
+    )
+
+
 def _validate_privacy_request_pending_or_error(
     db: Session, privacy_request_id: str
 ) -> None:

fides/api/db/database.py

@@ -2,7 +2,10 @@
 Contains all of the logic related to the database including connections, setup, teardown, etc.
 """
 
+import os
+import subprocess
 from os import path
+from pathlib import Path
 from typing import Literal, Optional, Tuple
 
 from alembic import command, script
@@ -191,3 +194,225 @@ def check_missing_migrations(database_url: str) -> None:
             log.error(f"  - {item}")
         raise SystemExit("Migration needs to be generated!")
     print("No migrations need to be generated.")
+
+
+def _get_base_branch_for_comparison() -> str:
+    """
+    Determine the base branch to compare against for detecting new migrations.
+
+    In GitHub Actions PRs, uses GITHUB_BASE_REF (the PR target branch).
+    Otherwise, falls back to origin/main for local development.
+
+    Returns:
+        The base branch reference (e.g., 'origin/main', 'origin/develop')
+    """
+    base_ref = os.environ.get("GITHUB_BASE_REF")
+    if base_ref:
+        base_branch = f"origin/{base_ref}"
+        log.info(f"Comparing against PR target branch: {base_branch}")
+    else:
+        base_branch = "origin/main"
+        log.info(f"No GITHUB_BASE_REF found, comparing against: {base_branch}")
+    return base_branch
+
+
+def _get_new_migration_files(repo_root: str, base_branch: str) -> list[str]:
+    """
+    Get list of new migration files added in the current branch.
+
+    Files are sorted by the timestamp in their filename to ensure chronological order.
+    Migration files follow the pattern: xx_YYYY_MM_DD_HHMM_revision_description.py
+
+    Args:
+        repo_root: Absolute path to the git repository root
+        base_branch: The base branch to compare against
+
+    Returns:
+        List of relative file paths to new migration files, sorted chronologically
+
+    Raises:
+        SystemExit: If the migrations directory does not exist
+    """
+    # Validate that the migrations directory exists
+    migrations_dir = (
+        Path(repo_root)
+        / "src"
+        / "fides"
+        / "api"
+        / "alembic"
+        / "migrations"
+        / "versions"
+    )
+    if not migrations_dir.exists():
+        log.error(f"Migrations directory not found: {migrations_dir}")
+        raise SystemExit(
+            f"Migration directory not found at expected path: {migrations_dir}\n"
+            "If the migrations directory has been moved, update the path in "
+            "fides.api.db.database._get_new_migration_files()"
+        )
+
+    migration_pattern = "src/fides/api/alembic/migrations/versions/*.py"
+    diff_output = subprocess.check_output(
+        [
+            "git",
+            "diff",
+            "--name-only",
+            "--diff-filter=A",  # Only added files
+            f"{base_branch}...HEAD",
+            "--",
+            migration_pattern,
+        ],
+        stderr=subprocess.STDOUT,
+        text=True,
+        cwd=repo_root,
+    ).strip()
+
+    if not diff_output:
+        return []
+
+    migration_files = [f for f in diff_output.split("\n") if f]
+
+    # Sort by filename to ensure chronological order
+    # Files are named: xx_YYYY_MM_DD_HHMM_revision_description.py
+    # Sorting alphabetically will sort them chronologically due to the date format
+    migration_files.sort()
+
+    return migration_files
+
+
+def _parse_migration_revisions(migration_path: Path) -> tuple[str, Optional[str]]:
+    """
+    Extract revision and down_revision IDs from a migration file.
+
+    Args:
+        migration_path: Path to the migration file
+
+    Returns:
+        Tuple of (revision, down_revision). down_revision may be None.
+
+    Raises:
+        SystemExit: If revision ID cannot be found in the file
+    """
+    revision = None
+    down_revision = None
+
+    with open(migration_path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip().startswith("revision ="):
+                revision = line.split("=")[1].strip().strip("\"'")
+            elif line.strip().startswith("down_revision ="):
+                down_rev_value = line.split("=")[1].strip().strip("\"'")
+                if down_rev_value.lower() != "none":
+                    down_revision = down_rev_value
+
+    if not revision:
+        log.error(f"Could not find revision ID in {migration_path}")
+        raise SystemExit(f"Invalid migration file: {migration_path}")
+
+    return revision, down_revision
+
+
+def _test_migrations_upgrade_downgrade_sequence(
+    alembic_config: Config,
+    migrations: list[tuple[str, str, Optional[str]]],
+) -> None:
+    """
+    Test multiple migrations can be upgraded and downgraded successfully in sequence.
+
+    Tests all new migrations together by:
+    1. Downgrading incrementally through all migrations (newest to oldest)
+    2. Upgrading incrementally back through all migrations (oldest to newest)
+
+    After Phase 2, the database is at the newest migration revision, which should
+    match HEAD for the new migrations being tested.
+
+    This approach tests migrations in a realistic sequence and can catch
+    interaction issues between migrations.
+
+    Args:
+        alembic_config: Alembic configuration
+        migrations: List of (migration_name, revision, down_revision) tuples
+
+    Raises:
+        SystemExit: If any migration upgrade/downgrade test fails
+    """
+    if not migrations:
+        return
+
+    log.info(f"Testing {len(migrations)} migration(s) in sequence")
+
+    try:
+        # Phase 1: Downgrade through all migrations (newest to oldest)
+        log.info("Phase 1: Downgrading through all new migrations...")
+        for migration_name, revision, down_revision in reversed(migrations):
+            if down_revision:
+                log.info(f"  Downgrading {migration_name} to {down_revision}...")
+                downgrade_db(alembic_config, down_revision)
+            else:
+                log.info(f"  Skipping {migration_name} (no down_revision)")
+
+        # Phase 2: Upgrade back through all migrations (oldest to newest)
+        log.info("Phase 2: Upgrading back through all new migrations...")
+        for migration_name, revision, down_revision in migrations:
+            log.info(f"  Upgrading {migration_name} to {revision}...")
+            upgrade_db(alembic_config, revision)
+
+        log.info(f"✓ All {len(migrations)} migration(s) passed upgrade/downgrade test")
+
+    except Exception as error:
+        log.error("Migration upgrade/downgrade sequence test failed!")
+        log.error(str(error))
+        raise SystemExit("Migration upgrade/downgrade test failed")
+
+
+def check_new_migrations_upgrade_downgrade(database_url: str) -> None:
+    """
+    Detects new migrations added in the current branch and tests that they
+    can be successfully upgraded and downgraded.
+
+    Uses git to compare against the target/base branch to find new migration files.
+    In CI, uses GITHUB_BASE_REF if available, otherwise falls back to origin/main.
+    """
+    try:
+        # Get the repo root directory
+        repo_root = subprocess.check_output(
+            ["git", "rev-parse", "--show-toplevel"],
+            stderr=subprocess.STDOUT,
+            text=True,
+        ).strip()
+
+        # Determine base branch and get new migration files
+        base_branch = _get_base_branch_for_comparison()
+        migration_files = _get_new_migration_files(repo_root, base_branch)
+
+        if not migration_files:
+            log.info("No new migration files detected in current branch")
+            print("No new migrations to test.")
+            return
+
+        log.info(f"Detected {len(migration_files)} new migration(s) to test")
+
+        # Log the migrations in order
+        for i, file_path in enumerate(migration_files, 1):
+            log.info(f"  {i}. {Path(file_path).name}")
+
+        alembic_config = get_alembic_config(database_url)
+
+        # Parse all migrations first
+        migrations = []
+        for file_path in migration_files:
+            abs_path = Path(repo_root) / file_path
+            revision, down_revision = _parse_migration_revisions(abs_path)
+            migrations.append((abs_path.name, revision, down_revision))
+
+        # Test all migrations in sequence
+        _test_migrations_upgrade_downgrade_sequence(alembic_config, migrations)
+
+        print(
+            f"All {len(migration_files)} new migration(s) passed upgrade/downgrade tests."
+        )
+
+    except subprocess.CalledProcessError as error:
+        # Git command failed - might not be in a git repo or no origin/main
+        log.warning(f"Could not detect new migrations via git: {error}")
+        print("Skipping migration upgrade/downgrade check (git detection failed).")

fides/api/models/detection_discovery/__init__.py

@@ -17,6 +17,7 @@ from .monitor_task import (
     create_monitor_task_with_execution_log,
     update_monitor_task_with_execution_log,
 )
+from .staged_resource_error import StagedResourceError
 
 __all__ = [
     "ClassificationBenchmark",
@@ -27,6 +28,7 @@ __all__ = [
     "SharedMonitorConfig",
     "StagedResource",
     "StagedResourceAncestor",
+    "StagedResourceError",
     "fetch_staged_resources_by_type_query",
     "MonitorTask",
     "MonitorTaskExecutionLog",

fides/api/models/detection_discovery/core.py

@@ -29,6 +29,9 @@ from sqlalchemy.orm.query import Query
 
 from fides.api.db.base_class import Base, FidesBase
 from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.detection_discovery.staged_resource_error import (
+    StagedResourceError,
+)
 from fides.api.models.sql_models import System  # type: ignore[attr-defined]
 
 
@@ -573,6 +576,13 @@ class StagedResource(Base):
     # diff-related fields
     diff_status = Column(String, nullable=True, index=True)
 
+    errors: RelationshipProperty[List[StagedResourceError]] = relationship(
+        "StagedResourceError",
+        foreign_keys=[StagedResourceError.staged_resource_urn],
+        primaryjoin="StagedResource.urn == StagedResourceError.staged_resource_urn",
+        cascade="all, delete-orphan",
+    )
+
     ancestor_links: RelationshipProperty[List[StagedResourceAncestor]] = relationship(
         "StagedResourceAncestor",
         back_populates="descendant_staged_resource",

fides/api/models/detection_discovery/staged_resource_error.py

@@ -0,0 +1,25 @@
+"""Module for staged resource error tracking."""
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+
+from fides.api.db.base_class import Base, FidesBase
+
+
+class StagedResourceError(Base):
+    """
+    DB model for tracking errors associated with staged resources.
+    Linked to StagedResource via FK with cascade delete.
+    """
+
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+    staged_resource_urn = Column(
+        String,
+        ForeignKey("stagedresource.urn", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    message = Column(String, nullable=False)
+    timestamp = Column(DateTime(timezone=True), nullable=False)
+    error_type = Column(String, nullable=False)
+    diff_status = Column(String, nullable=False)  # Phase where error occurred
+    task_id = Column(String, nullable=True, index=True)

fides/api/models/privacy_preference.py

@@ -44,6 +44,7 @@ class RequestOrigin(Enum):
     banner_and_modal = "banner_and_modal"
     api = "api"
     tcf_overlay = "tcf_overlay"
+    headless = "headless"
 
 
 class ConsentMethod(Enum):

fides/api/models/privacy_request/duplicate_group.py

@@ -0,0 +1,84 @@
+import hashlib
+import json
+import uuid
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import Boolean, Column, Text
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base  # type: ignore[attr-defined]
+from fides.config.duplicate_detection_settings import DuplicateDetectionSettings
+
+if TYPE_CHECKING:
+    from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+
+
+def generate_deterministic_uuid(rule_version: str, dedup_key: str) -> uuid.UUID:
+    """Overrides the base class method to generate a deterministic group id.
+
+    The actual rule version is a hash of the duplicate detection settings config,
+    using simple examples here for illustration:
+    - under rule "email", duplicate@example.com → group A,
+    - under rule "email|phone", duplicate@example.com|1234567890 → group B.
+    - ... and so on.
+
+    No collisions, no overlap.
+    """
+    # Combine the rule version and the key into a stable hash
+    hash_input = f"{rule_version}:{dedup_key}".encode("utf-8")
+    return uuid.UUID(hashlib.md5(hash_input).hexdigest())
+
+
+def generate_rule_version(config: DuplicateDetectionSettings) -> str:
+    """Generate a stable short hash for the dedup rule config."""
+    normalized = json.dumps(config.model_dump(), sort_keys=True)
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8]
+
+
+class DuplicateGroup(Base):
+    """
+    A table for storing duplicate request group information.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "duplicate_group"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    rule_version = Column(Text, nullable=False)
+    # TODO: The is_active column is not used yet since we are defaulting to just email for
+    # duplicate detection. When we allow users to configure the duplicate detection settings,
+    # we will need to add a way to activate/deactivate duplicate groups.
+    # This should be done by getting the previous rule version and deactivating all groups with
+    # that rule version. New groups will be created with the new rule version.
+    is_active = Column(Boolean, default=True, nullable=False)
+    dedup_key = Column(Text, nullable=False)
+    privacy_requests = relationship(
+        "PrivacyRequest",
+        back_populates="duplicate_group",
+        lazy="dynamic",
+        primaryjoin="and_(DuplicateGroup.id == PrivacyRequest.duplicate_request_group_id)",
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "DuplicateGroup":
+        """Create a new duplicate group.
+        Override the base class method to generate a deterministic group id.
+        If the group already exists, update the is_active column to True and return the group.
+        """
+        group_id = generate_deterministic_uuid(data["rule_version"], data["dedup_key"])
+        # If the group already exists, update the is_active column to True and return the group
+        duplicate_group = (
+            db.query(DuplicateGroup).filter(DuplicateGroup.id == group_id).one_or_none()
+        )
+        if duplicate_group:
+            duplicate_group.update(db, data={"is_active": True})
+            return duplicate_group
+
+        # If the group does not exist, create a new one
+        data["id"] = group_id
+        return super().create(db, data=data, check_name=check_name)