ethyca-fides 2.73.2b0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/api/v1/endpoints/user_endpoints.py

@@ -609,6 +609,7 @@ def get_users(
     params: Params = Depends(),
     username: Optional[str] = None,
     include_external: bool = True,
+    exclude_approvers: bool = False,
     client: ClientDetail = Security(verify_user_read_scopes),
     authorization: str = Security(oauth2_scheme),
 ) -> AbstractPage[FidesUser]:
@@ -628,10 +629,15 @@ def get_users(
             query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
 
         # Filter out external respondents if include_external is False
-        if not include_external:
-            query = query.join(FidesUserPermissions).filter(
-                ~FidesUserPermissions.roles.op("@>")([EXTERNAL_RESPONDENT])
-            )
+        # Filter out approvers if exclude_approvers is True
+        if not include_external or exclude_approvers:
+            query = query.join(FidesUserPermissions)
+            if not include_external:
+                query = query.filter(
+                    ~FidesUserPermissions.roles.op("@>")([EXTERNAL_RESPONDENT])
+                )
+            if exclude_approvers:
+                query = query.filter(~FidesUserPermissions.roles.op("@>")([APPROVER]))
     else:
         # User has USER_READ_OWN scope, only show their own data
         query = query.filter(FidesUser.id == client.user_id)

fides/api/db/base.py

@@ -80,6 +80,7 @@ from fides.api.models.property import (
     Property,
 )
 from fides.api.models.registration import UserRegistration
+from fides.api.models.saas_template_dataset import SaasTemplateDataset
 from fides.api.models.storage import StorageConfig
 from fides.api.models.system_compass_sync import SystemCompassSync
 from fides.api.models.system_group import SystemGroup, SystemGroupMember

fides/api/db/database.py

@@ -2,7 +2,10 @@
 Contains all of the logic related to the database including connections, setup, teardown, etc.
 """
 
+import os
+import subprocess
 from os import path
+from pathlib import Path
 from typing import Literal, Optional, Tuple
 
 from alembic import command, script
@@ -190,4 +193,228 @@ def check_missing_migrations(database_url: str) -> None:
         for item in result:
             log.error(f"  - {item}")
         raise SystemExit("Migration needs to be generated!")
-    print("No migrations need to be generated.")
+    log.info("No migrations need to be generated.")
+
+
+def _get_base_branch_for_comparison() -> str:
+    """
+    Determine the base branch to compare against for detecting new migrations.
+
+    In GitHub Actions PRs, uses GITHUB_BASE_REF (the PR target branch).
+    Otherwise, falls back to origin/main for local development.
+
+    Returns:
+        The base branch reference (e.g., 'origin/main', 'origin/develop')
+    """
+    base_ref = os.environ.get("GITHUB_BASE_REF")
+    if base_ref:
+        base_branch = f"origin/{base_ref}"
+        log.info(f"Comparing against PR target branch: {base_branch}")
+    else:
+        base_branch = "origin/main"
+        log.info(f"No GITHUB_BASE_REF found, comparing against: {base_branch}")
+    return base_branch
+
+
+def _get_new_migration_files(repo_root: str, base_branch: str) -> list[str]:
+    """
+    Get list of new migration files added in the current branch.
+
+    Files are sorted by the timestamp in their filename to ensure chronological order.
+    Migration files follow the pattern: xx_YYYY_MM_DD_HHMM_revision_description.py
+
+    Args:
+        repo_root: Absolute path to the git repository root
+        base_branch: The base branch to compare against
+
+    Returns:
+        List of relative file paths to new migration files, sorted chronologically
+
+    Raises:
+        SystemExit: If the migrations directory does not exist
+    """
+    # Validate that the migrations directory exists
+    migrations_dir = (
+        Path(repo_root)
+        / "src"
+        / "fides"
+        / "api"
+        / "alembic"
+        / "migrations"
+        / "versions"
+    )
+    if not migrations_dir.exists():
+        log.error(f"Migrations directory not found: {migrations_dir}")
+        raise SystemExit(
+            f"Migration directory not found at expected path: {migrations_dir}\n"
+            "If the migrations directory has been moved, update the path in "
+            "fides.api.db.database._get_new_migration_files()"
+        )
+
+    migration_pattern = "src/fides/api/alembic/migrations/versions/*.py"
+    diff_output = subprocess.check_output(
+        [
+            "git",
+            "diff",
+            "--name-only",
+            "--diff-filter=A",  # Only added files
+            f"{base_branch}...HEAD",
+            "--",
+            migration_pattern,
+        ],
+        stderr=subprocess.STDOUT,
+        text=True,
+        cwd=repo_root,
+    ).strip()
+
+    if not diff_output:
+        return []
+
+    migration_files = [f for f in diff_output.split("\n") if f]
+
+    # Sort by filename to ensure chronological order
+    # Files are named: xx_YYYY_MM_DD_HHMM_revision_description.py
+    # Sorting alphabetically will sort them chronologically due to the date format
+    migration_files.sort()
+
+    return migration_files
+
+
+def _parse_migration_revisions(migration_path: Path) -> tuple[str, Optional[str]]:
+    """
+    Extract revision and down_revision IDs from a migration file.
+
+    Args:
+        migration_path: Path to the migration file
+
+    Returns:
+        Tuple of (revision, down_revision). down_revision may be None.
+
+    Raises:
+        SystemExit: If revision ID cannot be found in the file
+    """
+    revision = None
+    down_revision = None
+
+    with open(migration_path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip().startswith("revision ="):
+                revision = line.split("=")[1].strip().strip("\"'")
+            elif line.strip().startswith("down_revision ="):
+                down_rev_value = line.split("=")[1].strip().strip("\"'")
+                if down_rev_value.lower() != "none":
+                    down_revision = down_rev_value
+
+    if not revision:
+        log.error(f"Could not find revision ID in {migration_path}")
+        raise SystemExit(f"Invalid migration file: {migration_path}")
+
+    return revision, down_revision
+
+
+def _test_migrations_upgrade_downgrade_sequence(
+    alembic_config: Config,
+    migrations: list[tuple[str, str, Optional[str]]],
+) -> None:
+    """
+    Test multiple migrations can be upgraded and downgraded successfully in sequence.
+
+    Tests all new migrations together by:
+    1. Downgrading incrementally through all migrations (newest to oldest)
+    2. Upgrading incrementally back through all migrations (oldest to newest)
+
+    After Phase 2, the database is at the newest migration revision, which should
+    match HEAD for the new migrations being tested.
+
+    This approach tests migrations in a realistic sequence and can catch
+    interaction issues between migrations.
+
+    Args:
+        alembic_config: Alembic configuration
+        migrations: List of (migration_name, revision, down_revision) tuples
+
+    Raises:
+        SystemExit: If any migration upgrade/downgrade test fails
+    """
+    if not migrations:
+        return
+
+    log.info(f"Testing {len(migrations)} migration(s) in sequence")
+
+    try:
+        # Phase 1: Downgrade through all migrations (newest to oldest)
+        log.info("Phase 1: Downgrading through all new migrations...")
+        for migration_name, revision, down_revision in reversed(migrations):
+            if down_revision:
+                log.info(f"  Downgrading {migration_name} to {down_revision}...")
+                downgrade_db(alembic_config, down_revision)
+            else:
+                log.info(f"  Skipping {migration_name} (no down_revision)")
+
+        # Phase 2: Upgrade back through all migrations (oldest to newest)
+        log.info("Phase 2: Upgrading back through all new migrations...")
+        for migration_name, revision, down_revision in migrations:
+            log.info(f"  Upgrading {migration_name} to {revision}...")
+            upgrade_db(alembic_config, revision)
+
+        log.info(f"✓ All {len(migrations)} migration(s) passed upgrade/downgrade test")
+
+    except Exception as error:
+        log.error("Migration upgrade/downgrade sequence test failed!")
+        log.error(str(error))
+        raise SystemExit("Migration upgrade/downgrade test failed")
+
+
+def check_new_migrations_upgrade_downgrade(database_url: str) -> None:
+    """
+    Detects new migrations added in the current branch and tests that they
+    can be successfully upgraded and downgraded.
+
+    Uses git to compare against the target/base branch to find new migration files.
+    In CI, uses GITHUB_BASE_REF if available, otherwise falls back to origin/main.
+    """
+    try:
+        # Get the repo root directory
+        repo_root = subprocess.check_output(
+            ["git", "rev-parse", "--show-toplevel"],
+            stderr=subprocess.STDOUT,
+            text=True,
+        ).strip()
+
+        # Determine base branch and get new migration files
+        base_branch = _get_base_branch_for_comparison()
+        migration_files = _get_new_migration_files(repo_root, base_branch)
+
+        if not migration_files:
+            log.info("No new migration files detected in current branch")
+            log.info("No new migrations to test.")
+            return
+
+        log.info(f"Detected {len(migration_files)} new migration(s) to test")
+
+        # Log the migrations in order
+        for i, file_path in enumerate(migration_files, 1):
+            log.info(f"  {i}. {Path(file_path).name}")
+
+        alembic_config = get_alembic_config(database_url)
+
+        # Parse all migrations first
+        migrations = []
+        for file_path in migration_files:
+            abs_path = Path(repo_root) / file_path
+            revision, down_revision = _parse_migration_revisions(abs_path)
+            migrations.append((abs_path.name, revision, down_revision))
+
+        # Test all migrations in sequence
+        _test_migrations_upgrade_downgrade_sequence(alembic_config, migrations)
+
+        log.info(
+            f"All {len(migration_files)} new migration(s) passed upgrade/downgrade tests."
+        )
+
+    except subprocess.CalledProcessError as error:
+        # Git command failed - might not be in a git repo or no origin/main
+        log.warning(f"Could not detect new migrations via git: {error}")
+        log.warning(
+            "Skipping migration upgrade/downgrade check (git detection failed)."
+        )

fides/api/db/seed.py

@@ -388,7 +388,7 @@ async def load_samples(async_session: AsyncSession) -> None:
                     )  # not supported by this API!
                     instantiate_connection_from_template(
                         db=db_session,
-                        saas_connector_type=connection.saas_connector_type,
+                        connector_template_type=connection.saas_connector_type,
                         template_values=SaasConnectionTemplateValues.model_validate(
                             saas_template_data
                         ),

fides/api/email_templates/templates/external_user_welcome.html

@@ -8,16 +8,20 @@
 <main>
     <p>Hello {{ display_name }},</p>
 
-    <p>An account has been created for you by the privacy team at {{ org_name }} so that you can assist them in responding to data subject requests, as required by the GDPR and many US state laws, for users that have data in your systems.</p>
+    {% if custom_intro %}
+        <div>{{ custom_intro | safe }}</div>
+    {% else %}
+        <p>An account has been created for you by the privacy team at {{ org_name }} so that you can assist them in responding to data subject requests, as required by the GDPR and many US state laws, for users that have data in your systems.</p>
 
-    <p>A portal is available to view and take action on these requests at: <a href="{{ portal_link }}">{{ portal_link }}</a>. Through this portal you can upload or provide user data for users who have requested a copy of their information and attest that users have been removed from your system in the case of an erasure request.</p>
+        <p>A portal is available to view and take action on these requests at: <a href="{{ portal_link }}">{{ portal_link }}</a>. Through this portal you can upload or provide user data for users who have requested a copy of their information and attest that users have been removed from your system in the case of an erasure request.</p>
 
-    <p>The provided link is valid for 30 days and you will be asked to verify your email address each time you access the portal.</p>
+        <p>The provided link is valid for 30 days and you will be asked to verify your email address each time you access the portal.</p>
 
-    <p>If you have any questions, please reach out first to your contacts at: {{ org_name }}</p>
+        <p>If you have any questions, please reach out first to your contacts at: {{ org_name }}</p>
+    {% endif %}
 
     <p>Regards,<br/>
     {{ org_name }} Privacy Team</p>
 </main>
 </body>
-</html> 
+</html>

fides/api/migrations/post_upgrade_index_creation.py

@@ -33,6 +33,9 @@ This means that after the constraint is created, the index will have the same na
 """
 
 POST_UPGRADE_INDEX_CREATION = "post_upgrade_index_creation"
+# This is the threshold for the number of rows in a table to create the index during application startup
+# The value follows the pattern established in migrations for the tables in the TABLE_OBJECT_MAP
+INDEX_ROW_COUNT_THRESHOLD = 1000000
 
 TABLE_OBJECT_MAP: Dict[str, List[Dict[str, str]]] = {
     "currentprivacypreferencev2": [
@@ -93,6 +96,13 @@ TABLE_OBJECT_MAP: Dict[str, List[Dict[str, str]]] = {
             "type": "index",
         },
     ],
+    "providedidentity": [
+        {
+            "name": "ix_providedidentity_privacy_request_id",
+            "statement": "CREATE INDEX CONCURRENTLY ix_providedidentity_privacy_request_id ON providedidentity (privacy_request_id)",
+            "type": "index",
+        },
+    ],
     "servednoticehistory": [
         {
             "name": "ix_servednoticehistory_hashed_external_id",

fides/api/models/connectionconfig.py

@@ -71,7 +71,8 @@ class ConnectionType(enum.Enum):
     timescale = "timescale"
     website = "website"
     # Test connection types, used for testing purposes only
-    test_website = "test_website"  # used for ConfigurableTesteWebsiteMonitor
+    test_datastore = "test_datastore"  # used for ConfigurableTestMonitor
+    test_website = "test_website"  # used for ConfigurableTestWebsiteMonitor
 
     @property
     def human_readable(self) -> str:
@@ -109,6 +110,7 @@ class ConnectionType(enum.Enum):
             ConnectionType.sovrn.value: "Sovrn",
             ConnectionType.timescale.value: "TimescaleDB",
             ConnectionType.website.value: "Website",
+            ConnectionType.test_datastore.value: "Test Datastore Connector -- not for production use",
             ConnectionType.test_website.value: "Test Website Connector -- not for production use",
         }
         try:
@@ -155,6 +157,7 @@ class ConnectionType(enum.Enum):
             ConnectionType.sovrn.value: SystemType.email,
             ConnectionType.timescale.value: SystemType.database,
             ConnectionType.website.value: SystemType.website,
+            ConnectionType.test_datastore.value: SystemType.database,
             ConnectionType.test_website.value: SystemType.website,
         }
 

fides/api/models/detection_discovery/__init__.py

@@ -17,6 +17,7 @@ from .monitor_task import (
     create_monitor_task_with_execution_log,
     update_monitor_task_with_execution_log,
 )
+from .staged_resource_error import StagedResourceError
 
 __all__ = [
     "ClassificationBenchmark",
@@ -27,6 +28,7 @@ __all__ = [
     "SharedMonitorConfig",
     "StagedResource",
     "StagedResourceAncestor",
+    "StagedResourceError",
     "fetch_staged_resources_by_type_query",
     "MonitorTask",
     "MonitorTaskExecutionLog",

fides/api/models/detection_discovery/core.py

@@ -29,6 +29,9 @@ from sqlalchemy.orm.query import Query
 
 from fides.api.db.base_class import Base, FidesBase
 from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.detection_discovery.staged_resource_error import (
+    StagedResourceError,
+)
 from fides.api.models.sql_models import System  # type: ignore[attr-defined]
 
 
@@ -573,6 +576,13 @@ class StagedResource(Base):
     # diff-related fields
     diff_status = Column(String, nullable=True, index=True)
 
+    errors: RelationshipProperty[List[StagedResourceError]] = relationship(
+        "StagedResourceError",
+        foreign_keys=[StagedResourceError.staged_resource_urn],
+        primaryjoin="StagedResource.urn == StagedResourceError.staged_resource_urn",
+        cascade="all, delete-orphan",
+    )
+
     ancestor_links: RelationshipProperty[List[StagedResourceAncestor]] = relationship(
         "StagedResourceAncestor",
         back_populates="descendant_staged_resource",

fides/api/models/detection_discovery/staged_resource_error.py

@@ -0,0 +1,25 @@
+"""Module for staged resource error tracking."""
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+
+from fides.api.db.base_class import Base, FidesBase
+
+
+class StagedResourceError(Base):
+    """
+    DB model for tracking errors associated with staged resources.
+    Linked to StagedResource via FK with cascade delete.
+    """
+
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+    staged_resource_urn = Column(
+        String,
+        ForeignKey("stagedresource.urn", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    message = Column(String, nullable=False)
+    timestamp = Column(DateTime(timezone=True), nullable=False)
+    error_type = Column(String, nullable=False)
+    diff_status = Column(String, nullable=False)  # Phase where error occurred
+    task_id = Column(String, nullable=True, index=True)

fides/api/models/experience_notices.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from uuid import uuid4
 
-from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy import Column, ForeignKey, Integer, String
 
 from fides.api.db.base_class import Base
 
@@ -42,3 +42,8 @@ class ExperienceNotices(Base):
         nullable=False,
         primary_key=True,
     )
+    display_order = Column(
+        Integer,
+        nullable=True,
+        doc="Optional ordering for notices within an experience. Lower values appear first.",
+    )

fides/api/models/identity_definition.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
-from sqlalchemy import Column, String, Text
+from sqlalchemy import Boolean, Column, String, Text
 from sqlalchemy.ext.declarative import declared_attr
 
 from fides.api.db.base_class import Base, FidesBase
@@ -63,3 +63,4 @@ class IdentityDefinition(Base):
         nullable=False,
     )
     created_by = Column(String(255), nullable=True)
+    is_default = Column(Boolean, nullable=False, server_default="false")

fides/api/models/location_regulation_selections.py

@@ -408,6 +408,28 @@ privacy_notice_regions_by_id: Dict[str, Union[Location, LocationGroup]] = (
 )  # should only be accessed for read-only access
 
 
+def get_location_by_id(location: str) -> Optional[Location]:
+    """
+    Get a Location object by ID, with automatic format normalization.
+
+    Handles various location formats (e.g., "US-CA", "us-ca", "us_ca") by
+    normalizing to the internal format (lowercase with underscores) used in locations.yml.
+
+    Args:
+        location: Location string in any format
+
+    Returns:
+        Location object if found, None otherwise
+
+    Examples:
+        >>> get_location_by_id("US-CA")  # ISO 3166 format
+        >>> get_location_by_id("us_ca")  # Internal format
+        >>> get_location_by_id("us-ca")  # Mixed format
+    """
+    normalized = location.lower().replace("-", "_")
+    return locations_by_id.get(normalized)
+
+
 # dynamically create an enum based on definitions loaded from YAML
 # This is a combination of "locations" and "location groups" for use on Privacy Experiences
 PrivacyNoticeRegion = Enum(  # type: ignore[misc]

fides/api/models/manual_webhook.py

@@ -5,7 +5,7 @@ from pydantic import ConfigDict, create_model
 from sqlalchemy import Column, ForeignKey, String, or_, text
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.mutable import MutableList
-from sqlalchemy.orm import Session, relationship
+from sqlalchemy.orm import Session, relationship, selectinload
 
 from fides.api.db.base_class import Base
 from fides.api.models.attachment import Attachment, AttachmentReference
@@ -137,11 +137,22 @@ class AccessManualWebhook(Base):
     ) -> list["AccessManualWebhook"]:
         """Get all enabled access manual webhooks with fields"""
 
-        query = db.query(cls).filter(
-            AccessManualWebhook.connection_config_id == ConnectionConfig.id,
-            ConnectionConfig.disabled.is_(False),
-            AccessManualWebhook.fields != text("'null'"),
-            AccessManualWebhook.fields != "[]",
+        query = (
+            db.query(cls)
+            .join(
+                ConnectionConfig,
+                AccessManualWebhook.connection_config_id == ConnectionConfig.id,
+            )
+            .options(
+                selectinload(AccessManualWebhook.connection_config).selectinload(
+                    ConnectionConfig.system
+                ),  # type: ignore[attr-defined]
+            )
+            .filter(
+                ConnectionConfig.disabled.is_(False),
+                AccessManualWebhook.fields != text("'null'"),
+                AccessManualWebhook.fields != "[]",
+            )
         )
 
         # Add action_type filter only if action_type is provided

fides/api/models/messaging_template.py

@@ -65,6 +65,13 @@ DEFAULT_MESSAGING_TEMPLATES: Dict[str, Any] = {
             "body": "This is your weekly summary of open data subject requests tasks from __ORGANIZATION_NAME__ that require your attention. Please review the tasks in our Privacy Center to ensure timely completion.",
         },
     },
+    MessagingActionType.EXTERNAL_USER_WELCOME.value: {
+        "label": "External user welcome",
+        "content": {
+            "subject": "Welcome to our Privacy Center",
+            "body": "An account has been created for you by the privacy team at __ORG_NAME__ so that you can assist them in responding to data subject requests, as required by the GDPR and many US state laws, for users that have data in your systems. A portal is available to view and take action on these requests at: __PORTAL_LINK__. Through this portal you can upload or provide user data for users who have requested a copy of their information and attest that users have been removed from your system in the case of an erasure request. The provided link is valid for 30 days and you will be asked to verify your email address each time you access the portal. If you have any questions, please reach out first to your contacts at: __ORG_NAME__",
+        },
+    },
 }
 
 

fides/api/models/policy.py

@@ -142,6 +142,9 @@ class Policy(Base):
         except IndexError:
             return None
 
+    def get_all_action_types(self) -> Set[ActionType]:
+        return set(rule.action_type for rule in self.rules)  # type: ignore[attr-defined]
+
     def generate_masking_secrets(self) -> Optional[List[MaskingSecretCache]]:
         """
         Returns a list of masking secrets for the masking strategies in the policy.

fides/api/models/privacy_experience.py

@@ -17,6 +17,7 @@ from fides.api.models import (
     dry_update_data,
     update_if_modified,
 )
+from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.location_regulation_selections import PrivacyNoticeRegion
 from fides.api.models.privacy_notice import PrivacyNotice
 from fides.api.models.property import Property
@@ -48,6 +49,7 @@ class Layer1ButtonOption(Enum):
     ACKNOWLEDGE = "acknowledge"
     OPT_IN_OPT_OUT = "opt_in_opt_out"
     OPT_IN_ONLY = "opt_in_only"
+    GPC_CONDITIONAL = "gpc_conditional"
 
 
 class RejectAllMechanism(Enum):
@@ -271,6 +273,7 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
         secondary="experiencenotices",
         backref="experience_configs",
         lazy="selectin",
+        order_by="ExperienceNotices.display_order.asc().nullslast()",
     )
 
     translations: RelationshipProperty[List[ExperienceTranslation]] = relationship(
@@ -748,15 +751,19 @@ def link_notices_to_experience_config(
 ) -> List[PrivacyNotice]:
     """
     Link supplied Notices to ExperienceConfig and unlink any notices not supplied.
+
+    Preserves the order of notices by setting display_order on the junction table records.
     """
     new_notices: Query = db.query(PrivacyNotice).filter(
         PrivacyNotice.id.in_(notice_ids)
     )
 
+    # Add new notices
     for notice in new_notices:
         if notice not in experience_config.privacy_notices:
             experience_config.privacy_notices.append(notice)
 
+    # Remove notices that are no longer in the list
     to_remove: Set[PrivacyNotice] = set(
         notice for notice in experience_config.privacy_notices
     ) - set(notice for notice in new_notices)
@@ -765,6 +772,19 @@ def link_notices_to_experience_config(
         experience_config.privacy_notices.remove(privacy_notice)
 
     experience_config.save(db)
+
+    # Update display_order on the junction table records based on the order in notice_ids
+    for index, notice_id in enumerate(notice_ids):
+        db.query(ExperienceNotices).filter(
+            ExperienceNotices.notice_id == notice_id,
+            ExperienceNotices.experience_config_id == experience_config.id,
+        ).update({"display_order": index})
+
+    db.commit()
+
+    # Expire the relationship to force reload with correct ordering
+    db.expire(experience_config, ["privacy_notices"])
+
     return experience_config.privacy_notices