ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/db/database.py

@@ -2,14 +2,19 @@
 Contains all of the logic related to the database including connections, setup, teardown, etc.
 """
 
+import os
+import subprocess
 from os import path
+from pathlib import Path
 from typing import Literal, Optional, Tuple
 
 from alembic import command, script
 from alembic.config import Config
 from alembic.runtime import migration
 from loguru import logger as log
+from sqlalchemy import text
 from sqlalchemy.orm import Session
+from sqlalchemy.schema import SchemaItem
 from sqlalchemy_utils.functions import create_database, database_exists
 from sqlalchemy_utils.types.encrypted.encrypted_type import InvalidCiphertextError
 
@@ -20,6 +25,25 @@ from fides.api.util.errors import get_full_exception_name
 
 DatabaseHealth = Literal["healthy", "unhealthy", "needs migration"]
 
+# Tables to exclude from migration auto-generation (e.g., tables without SQLAlchemy models)
+EXCLUDED_TABLES = {
+    "privacy_preferences_current",
+    "privacy_preferences_historic",
+}
+
+
+def include_object(
+    object: SchemaItem,  # pylint: disable=redefined-builtin
+    name: str,
+    type_: str,
+    reflected: bool,
+    compare_to: SchemaItem,
+) -> bool:
+    """Filter out excluded tables from migration comparison and auto-generation."""
+    if type_ == "table" and name in EXCLUDED_TABLES:
+        return False
+    return True
+
 
 def get_alembic_config(database_url: str) -> Config:
     """
@@ -91,6 +115,10 @@ def reset_db(database_url: str) -> None:
         log.info("Dropping tables...")
         Base.metadata.drop_all(connection)
 
+        log.info("Dropping excluded tables without models...")
+        for table_name in EXCLUDED_TABLES:
+            connection.execute(text(f"DROP TABLE IF EXISTS {table_name} CASCADE"))
+
         log.info("Dropping Alembic table...")
         migration_context = migration.MigrationContext.configure(connection)
         version = migration_context._version  # pylint: disable=protected-access
@@ -151,13 +179,242 @@ def check_missing_migrations(database_url: str) -> None:
     Tries to autogenerate migrations, returns True if a migration
     was generated.
     """
-
     engine = get_db_engine(database_uri=database_url)
     connection = engine.connect()
 
-    migration_context = migration.MigrationContext.configure(connection)
+    migration_context = migration.MigrationContext.configure(
+        connection, opts={"include_object": include_object}
+    )
     result = command.autogen.compare_metadata(migration_context, Base.metadata)  # type: ignore[attr-defined]
 
     if result:
-        raise SystemExit("Migrations needs to be generated!")
-    print("No migrations need to be generated.")
+        log.error("Migration needs to be generated!")
+        log.error("Detected the following schema differences:")
+        for item in result:
+            log.error(f"  - {item}")
+        raise SystemExit("Migration needs to be generated!")
+    log.info("No migrations need to be generated.")
+
+
+def _get_base_branch_for_comparison() -> str:
+    """
+    Determine the base branch to compare against for detecting new migrations.
+
+    In GitHub Actions PRs, uses GITHUB_BASE_REF (the PR target branch).
+    Otherwise, falls back to origin/main for local development.
+
+    Returns:
+        The base branch reference (e.g., 'origin/main', 'origin/develop')
+    """
+    base_ref = os.environ.get("GITHUB_BASE_REF")
+    if base_ref:
+        base_branch = f"origin/{base_ref}"
+        log.info(f"Comparing against PR target branch: {base_branch}")
+    else:
+        base_branch = "origin/main"
+        log.info(f"No GITHUB_BASE_REF found, comparing against: {base_branch}")
+    return base_branch
+
+
+def _get_new_migration_files(repo_root: str, base_branch: str) -> list[str]:
+    """
+    Get list of new migration files added in the current branch.
+
+    Files are sorted by the timestamp in their filename to ensure chronological order.
+    Migration files follow the pattern: xx_YYYY_MM_DD_HHMM_revision_description.py
+
+    Args:
+        repo_root: Absolute path to the git repository root
+        base_branch: The base branch to compare against
+
+    Returns:
+        List of relative file paths to new migration files, sorted chronologically
+
+    Raises:
+        SystemExit: If the migrations directory does not exist
+    """
+    # Validate that the migrations directory exists
+    migrations_dir = (
+        Path(repo_root)
+        / "src"
+        / "fides"
+        / "api"
+        / "alembic"
+        / "migrations"
+        / "versions"
+    )
+    if not migrations_dir.exists():
+        log.error(f"Migrations directory not found: {migrations_dir}")
+        raise SystemExit(
+            f"Migration directory not found at expected path: {migrations_dir}\n"
+            "If the migrations directory has been moved, update the path in "
+            "fides.api.db.database._get_new_migration_files()"
+        )
+
+    migration_pattern = "src/fides/api/alembic/migrations/versions/*.py"
+    diff_output = subprocess.check_output(
+        [
+            "git",
+            "diff",
+            "--name-only",
+            "--diff-filter=A",  # Only added files
+            f"{base_branch}...HEAD",
+            "--",
+            migration_pattern,
+        ],
+        stderr=subprocess.STDOUT,
+        text=True,
+        cwd=repo_root,
+    ).strip()
+
+    if not diff_output:
+        return []
+
+    migration_files = [f for f in diff_output.split("\n") if f]
+
+    # Sort by filename to ensure chronological order
+    # Files are named: xx_YYYY_MM_DD_HHMM_revision_description.py
+    # Sorting alphabetically will sort them chronologically due to the date format
+    migration_files.sort()
+
+    return migration_files
+
+
+def _parse_migration_revisions(migration_path: Path) -> tuple[str, Optional[str]]:
+    """
+    Extract revision and down_revision IDs from a migration file.
+
+    Args:
+        migration_path: Path to the migration file
+
+    Returns:
+        Tuple of (revision, down_revision). down_revision may be None.
+
+    Raises:
+        SystemExit: If revision ID cannot be found in the file
+    """
+    revision = None
+    down_revision = None
+
+    with open(migration_path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip().startswith("revision ="):
+                revision = line.split("=")[1].strip().strip("\"'")
+            elif line.strip().startswith("down_revision ="):
+                down_rev_value = line.split("=")[1].strip().strip("\"'")
+                if down_rev_value.lower() != "none":
+                    down_revision = down_rev_value
+
+    if not revision:
+        log.error(f"Could not find revision ID in {migration_path}")
+        raise SystemExit(f"Invalid migration file: {migration_path}")
+
+    return revision, down_revision
+
+
+def _test_migrations_upgrade_downgrade_sequence(
+    alembic_config: Config,
+    migrations: list[tuple[str, str, Optional[str]]],
+) -> None:
+    """
+    Test multiple migrations can be upgraded and downgraded successfully in sequence.
+
+    Tests all new migrations together by:
+    1. Downgrading incrementally through all migrations (newest to oldest)
+    2. Upgrading incrementally back through all migrations (oldest to newest)
+
+    After Phase 2, the database is at the newest migration revision, which should
+    match HEAD for the new migrations being tested.
+
+    This approach tests migrations in a realistic sequence and can catch
+    interaction issues between migrations.
+
+    Args:
+        alembic_config: Alembic configuration
+        migrations: List of (migration_name, revision, down_revision) tuples
+
+    Raises:
+        SystemExit: If any migration upgrade/downgrade test fails
+    """
+    if not migrations:
+        return
+
+    log.info(f"Testing {len(migrations)} migration(s) in sequence")
+
+    try:
+        # Phase 1: Downgrade through all migrations (newest to oldest)
+        log.info("Phase 1: Downgrading through all new migrations...")
+        for migration_name, revision, down_revision in reversed(migrations):
+            if down_revision:
+                log.info(f"  Downgrading {migration_name} to {down_revision}...")
+                downgrade_db(alembic_config, down_revision)
+            else:
+                log.info(f"  Skipping {migration_name} (no down_revision)")
+
+        # Phase 2: Upgrade back through all migrations (oldest to newest)
+        log.info("Phase 2: Upgrading back through all new migrations...")
+        for migration_name, revision, down_revision in migrations:
+            log.info(f"  Upgrading {migration_name} to {revision}...")
+            upgrade_db(alembic_config, revision)
+
+        log.info(f"✓ All {len(migrations)} migration(s) passed upgrade/downgrade test")
+
+    except Exception as error:
+        log.error("Migration upgrade/downgrade sequence test failed!")
+        log.error(str(error))
+        raise SystemExit("Migration upgrade/downgrade test failed")
+
+
+def check_new_migrations_upgrade_downgrade(database_url: str) -> None:
+    """
+    Detects new migrations added in the current branch and tests that they
+    can be successfully upgraded and downgraded.
+
+    Uses git to compare against the target/base branch to find new migration files.
+    In CI, uses GITHUB_BASE_REF if available, otherwise falls back to origin/main.
+    """
+    try:
+        # Get the repo root directory
+        repo_root = subprocess.check_output(
+            ["git", "rev-parse", "--show-toplevel"],
+            stderr=subprocess.STDOUT,
+            text=True,
+        ).strip()
+
+        # Determine base branch and get new migration files
+        base_branch = _get_base_branch_for_comparison()
+        migration_files = _get_new_migration_files(repo_root, base_branch)
+
+        if not migration_files:
+            log.info("No new migration files detected in current branch")
+            log.info("No new migrations to test.")
+            return
+
+        log.info(f"Detected {len(migration_files)} new migration(s) to test")
+
+        # Log the migrations in order
+        for i, file_path in enumerate(migration_files, 1):
+            log.info(f"  {i}. {Path(file_path).name}")
+
+        alembic_config = get_alembic_config(database_url)
+
+        # Parse all migrations first
+        migrations = []
+        for file_path in migration_files:
+            abs_path = Path(repo_root) / file_path
+            revision, down_revision = _parse_migration_revisions(abs_path)
+            migrations.append((abs_path.name, revision, down_revision))
+
+        # Test all migrations in sequence
+        _test_migrations_upgrade_downgrade_sequence(alembic_config, migrations)
+
+        log.info(
+            f"All {len(migration_files)} new migration(s) passed upgrade/downgrade tests."
+        )
+
+    except subprocess.CalledProcessError as error:
+        # Git command failed - might not be in a git repo or no origin/main
+        log.warning(f"Could not detect new migrations via git: {error}")
+        log.warning(
+            "Skipping migration upgrade/downgrade check (git detection failed)."
+        )

fides/api/db/safe_crud.py

@@ -0,0 +1,377 @@
+"""
+This module contains "safe" versions of CRUD operations that do NOT
+manually manage database transactions. Transaction management is left
+to the calling code (endpoints, services) to handle at the appropriate
+boundary.
+
+Use functions from this module instead of fides.api.db.crud
+"""
+
+from collections import defaultdict
+from typing import Any, Dict, List, Tuple, Type, TypeVar
+
+from fastapi import HTTPException
+from loguru import logger as log
+from sqlalchemy import and_, column
+from sqlalchemy import delete as _delete
+from sqlalchemy import or_
+from sqlalchemy import update as _update
+from sqlalchemy.dialects.postgresql import Insert as _insert
+from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.future import select
+from sqlalchemy.sql import Select
+from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
+
+from fides.api.models.sql_models import (  # type: ignore[attr-defined]
+    CustomField,
+    CustomFieldDefinition,
+    FidesBase,
+    ResourceTypes,
+)
+from fides.api.util import errors
+
+# Helps return type be linked to the type of the parameter
+T = TypeVar("T", bound="FidesBase")
+
+
+# CRUD Functions
+async def create_resource(
+    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
+) -> T:
+    """
+    Create a resource in the database.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    with log.contextualize(
+        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
+    ):
+
+        existing_resource = await get_resource(
+            sql_model, resource_dict["fides_key"], async_session, raise_not_found=False
+        )
+
+        if existing_resource is not None:
+            already_exists_error = errors.AlreadyExistsError(
+                sql_model.__name__, resource_dict["fides_key"]
+            )
+            log.bind(error=already_exists_error.detail["error"]).info(  # type: ignore[index]
+                "Failed to insert resource"
+            )
+            raise already_exists_error
+
+        try:
+            log.debug("Creating resource")
+            query = _insert(sql_model.__table__).values(resource_dict)
+            await async_session.execute(query)
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to create resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+
+async def get_custom_fields_filtered(
+    async_session: AsyncSession,
+    resource_types_to_ids: Dict[ResourceTypes, List[str]] = defaultdict(list),
+) -> FidesBase:
+    """
+    Utility function to construct a filtered query for custom field values based on provided mapping of
+    resource types to resource IDs.
+
+    Only custom fields with an "active" CustomFieldDefinition are returned.
+
+    This is for use in bulk querying of custom fields, to avoid multiple round trips to the db.
+    """
+    with log.contextualize(model=CustomField):
+        try:
+            log.debug("Fetching resource")
+            query = select(
+                CustomField.resource_id,
+                CustomField.value,
+                CustomFieldDefinition.resource_type,
+                CustomFieldDefinition.name,
+                CustomFieldDefinition.field_type,
+            ).join(
+                CustomFieldDefinition,
+                CustomFieldDefinition.id == CustomField.custom_field_definition_id,
+            )
+
+            criteria = [
+                and_(
+                    CustomFieldDefinition.resource_type == resource_type.value,
+                    CustomField.resource_id.in_(resource_ids),
+                    # pylint: disable=singleton-comparison
+                    CustomFieldDefinition.active == True,
+                )
+                for resource_type, resource_ids in resource_types_to_ids.items()
+            ]
+            query = query.where(or_(False, *criteria))
+            result = await async_session.execute(query)
+            return result.mappings().all()
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch custom fields with error: '{e}'")
+            raise sa_error
+
+
+async def get_resource(
+    sql_model: Type[T],
+    fides_key: str,
+    async_session: AsyncSession,
+    raise_not_found: bool = True,
+) -> T:
+    """
+    Get a resource from the database by its FidesKey.
+
+    Returns a SQLAlchemy model of that resource.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        try:
+            log.debug("Fetching resource")
+            query = select(sql_model).where(sql_model.fides_key == fides_key)
+            result = await async_session.execute(query)
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch resource with error: '{e}'")
+            raise sa_error
+
+        sql_resource = result.scalars().first()
+        if sql_resource is None and raise_not_found:
+            not_found_error = errors.NotFoundError(sql_model.__name__, fides_key)
+            log.bind(error=not_found_error.detail["error"]).info("Resource not found")  # type: ignore[index]
+            raise not_found_error
+
+        return sql_resource
+
+
+async def get_resource_with_custom_fields(
+    sql_model: Type[T], fides_key: str, async_session: AsyncSession
+) -> Dict[str, Any]:
+    """Get a resource from the database by its FidesKey including it's custom fields.
+
+    Returns a dictionary of that resource.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    resource: T = await get_resource(sql_model, fides_key, async_session)
+    resource_dict = resource.__dict__
+    resource_dict.pop("_sa_instance_state", None)
+
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        try:
+            log.debug("Fetching custom fields for resource")
+            query = (
+                select(CustomFieldDefinition.name, CustomField.value)
+                .join(
+                    CustomField,
+                    CustomField.custom_field_definition_id == CustomFieldDefinition.id,
+                )
+                .where(
+                    (CustomField.resource_id == resource.fides_key)
+                    & (  # pylint: disable=singleton-comparison
+                        CustomFieldDefinition.active == True
+                    )
+                )
+            )
+            result = await async_session.execute(query)
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch custom fields with error: '{e}'")
+            raise sa_error
+
+        custom_fields = result.mappings().all()
+
+    if not custom_fields:
+        return resource_dict
+
+    for field in custom_fields:
+        if field["name"] in resource_dict:
+            resource_dict[field["name"]] = (
+                f"{resource_dict[field['name']]}, {', '.join(field['value'])}"
+            )
+        else:
+            resource_dict[field["name"]] = ", ".join(field["value"])
+
+    return resource_dict
+
+
+async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List[T]:
+    """
+    Get a list of all of the resources of this type from the database.
+
+    Returns a list of SQLAlchemy models of that resource type.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    query = select(sql_model)
+    return await list_resource_query(async_session, query, sql_model)
+
+
+async def list_resource_query(
+    async_session: AsyncSession, query: Select, sql_model: Type[T]
+) -> List[T]:
+    """
+    Utility function to wrap a select query in generic "list_resource" execution handling.
+    Wrapping includes execution against the DB session, logging and error handling.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(sql_model=sql_model.__name__):
+        try:
+            log.debug("Fetching resources")
+            result = await async_session.execute(query)
+            sql_resources = result.scalars().all()
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to fetch resources with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return sql_resources
+
+
+async def update_resource(
+    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
+) -> Dict:
+    """
+    Update a resource in the database by its fides_key.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(
+        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
+    ):
+        await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+        try:
+            log.debug("Updating resource")
+            await async_session.execute(
+                _update(sql_model.__table__)
+                .where(sql_model.fides_key == resource_dict["fides_key"])
+                .values(resource_dict)
+            )
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to update resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+
+async def upsert_resources(
+    sql_model: Type[T], resource_dicts: List[Dict], async_session: AsyncSession
+) -> Tuple[int, int]:
+    """
+    Insert new resources into the database. If a resource already exists,
+    update it by it's fides_key.
+
+    Returns a Tuple containing the counts of inserted and updated rows.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(
+        sql_model=sql_model.__name__,
+        fides_keys=[resource["fides_key"] for resource in resource_dicts],
+    ):
+        try:
+            log.debug("Upserting resources")
+            insert_stmt = (
+                _insert(sql_model.__table__)
+                .values(resource_dicts)
+                .returning(
+                    (column("xmax") == 0),  # Row was inserted
+                    (column("xmax") != 0),  # Row was updated
+                )
+            )
+
+            excluded = dict(insert_stmt.excluded.items())  # type: ignore[attr-defined]
+            excluded.pop("id", None)  # If updating, don't update the "id"
+
+            result = await async_session.execute(
+                insert_stmt.on_conflict_do_update(
+                    index_elements=["fides_key"],
+                    set_=excluded,
+                )
+            )
+
+            inserts, updates = 0, 0
+            for xmax in result:
+                inserts += 1 if xmax[0] else 0
+                updates += 1 if xmax[1] else 0
+
+            return (inserts, updates)
+
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to upsert resources with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+
+async def delete_resource(
+    sql_model: Type[T], fides_key: str, async_session: AsyncSession
+) -> T:
+    """
+    Delete a resource by its fides_key.
+
+    If the resource has child keys referring to it, also delete those.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        sql_resource = await get_resource(sql_model, fides_key, async_session)
+
+        try:
+            # Automatically delete related resources if they are CTL objects
+            if hasattr(sql_model, "parent_key"):
+                log.debug("Deleting resource and its children")
+                query = (
+                    _delete(sql_model.__table__)
+                    .where(
+                        or_(
+                            sql_model.fides_key == fides_key,
+                            sql_model.fides_key.like(f"{fides_key}.%"),
+                        )
+                    )
+                    .execution_options(synchronize_session=False)
+                )
+            else:
+                log.debug("Deleting resource")
+                query = _delete(sql_model.__table__).where(
+                    sql_model.fides_key == fides_key
+                )
+            await async_session.execute(query)
+        except IntegrityError as err:
+            raw_error_text: str = err.orig.args[0]
+
+            if "violates foreign key constraint" in raw_error_text:
+                error_message = "Failed to delete resource! Foreign key constraint found, try deleting related resources first."
+            else:
+                error_message = "Failed to delete resource!"
+
+            log.bind(error="SQL Query integrity error!").error(raw_error_text)
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=error_message,
+            )
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to delete resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return sql_resource