ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/api/api/v1/endpoints/system.py

@@ -30,7 +30,7 @@ from fides.api.db.system import (
     upsert_system,
     validate_privacy_declarations,
 )
-from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
+from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.sql_models import System  # type:ignore[attr-defined]
 from fides.api.oauth.roles import APPROVER
@@ -57,14 +57,12 @@ from fides.api.schemas.system import (
     BasicSystemResponse,
     SystemResponse,
 )
-from fides.api.service.deps import get_system_service
+from fides.api.service.deps import get_connection_service, get_system_service
 from fides.api.util.api_router import APIRouter
 from fides.api.util.connection_util import (
-    connection_status,
     delete_connection_config,
-    get_connection_config_or_error,
     patch_connection_configs,
-    validate_secrets,
+    update_connection_secrets,
 )
 from fides.common.api.scope_registry import (
     CONNECTION_CREATE_OR_UPDATE,
@@ -81,6 +79,7 @@ from fides.common.api.v1.urn_registry import (
     SYSTEM_CONNECTIONS,
     V1_URL_PREFIX,
 )
+from fides.service.connection.connection_service import ConnectionService
 from fides.service.system.system_service import SystemService
 
 SYSTEM_ROUTER = APIRouter(tags=["System"], prefix=f"{V1_URL_PREFIX}/system")
@@ -160,6 +159,7 @@ def patch_connection_secrets(
     db: Session = Depends(deps.get_db),
     unvalidated_secrets: connection_secrets_schemas,
     verify: Optional[bool] = True,
+    connection_service: ConnectionService = Depends(get_connection_service),
 ) -> TestStatusMessage:
     """
     Patch secrets that will be used to connect to a specified connection_type.
@@ -169,45 +169,13 @@ def patch_connection_secrets(
     """
 
     system = get_system(db, fides_key)
-    connection_config: ConnectionConfig = get_connection_config_or_error(
-        db, system.connection_configs.key
+    return update_connection_secrets(
+        connection_service,
+        system.connection_configs.key,
+        unvalidated_secrets,
+        verify,
+        merge_with_existing=True,
     )
-    # Inserts unchanged sensitive values. The FE does not send masked values sensitive secrets.
-    if connection_config.secrets is not None:
-        for key, value in connection_config.secrets.items():
-            if key not in unvalidated_secrets:
-                # unvalidated_secrets is actually a dictionary here.  connection_secrets_schemas
-                # are just provided for documentation but the data was not parsed up front.
-                # That happens below in validate_secrets.
-                unvalidated_secrets[key] = value  # type: ignore
-    else:
-        connection_config.secrets = {}
-
-    validated_secrets = validate_secrets(
-        db, unvalidated_secrets, connection_config
-    ).model_dump(mode="json")
-
-    for key, value in validated_secrets.items():
-        connection_config.secrets[key] = value  # type: ignore
-
-    # Deauthorize an OAuth connection when the secrets are updated. This is necessary because
-    # the existing access tokens may not be valid anymore. This only applies to SaaS connection
-    # configurations that use the "oauth2_authorization_code" authentication strategy.
-    if (
-        connection_config.authorized
-        and connection_config.connection_type == ConnectionType.saas
-    ):
-        del connection_config.secrets["access_token"]
-
-    # Save validated secrets, regardless of whether they've been verified.
-    logger.info("Updating connection config secrets for '{}'", connection_config.key)
-    connection_config.save(db=db)
-
-    msg = f"Secrets updated for ConnectionConfig with key: {connection_config.key}."
-    if verify:
-        return connection_status(connection_config, msg, db)
-
-    return TestStatusMessage(msg=msg, test_status=None)
 
 
 @SYSTEM_CONNECTIONS_ROUTER.delete(

fides/api/api/v1/endpoints/user_endpoints.py

@@ -36,7 +36,7 @@ from fides.api.models.fides_user import FidesUser
 from fides.api.models.fides_user_invite import FidesUserInvite
 from fides.api.models.fides_user_permissions import FidesUserPermissions
 from fides.api.models.sql_models import System  # type: ignore[attr-defined]
-from fides.api.oauth.roles import APPROVER, VIEWER
+from fides.api.oauth.roles import APPROVER, EXTERNAL_RESPONDENT, VIEWER
 from fides.api.oauth.utils import (
     create_temporary_user_for_login_flow,
     extract_payload,
@@ -608,6 +608,7 @@ def get_users(
     db: Session = Depends(get_db),
     params: Params = Depends(),
     username: Optional[str] = None,
+    include_external: bool = True,
     client: ClientDetail = Security(verify_user_read_scopes),
     authorization: str = Security(oauth2_scheme),
 ) -> AbstractPage[FidesUser]:
@@ -625,6 +626,12 @@ def get_users(
         # User has USER_READ scope, can see all users
         if username:
             query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
+
+        # Filter out external respondents if include_external is False
+        if not include_external:
+            query = query.join(FidesUserPermissions).filter(
+                ~FidesUserPermissions.roles.op("@>")([EXTERNAL_RESPONDENT])
+            )
     else:
         # User has USER_READ_OWN scope, only show their own data
         query = query.filter(FidesUser.id == client.user_id)

fides/api/app_setup.py

@@ -39,9 +39,6 @@ from fides.api.oauth.system_manager_oauth_util import (
     verify_oauth_client_for_system_from_request_body_cli,
 )
 from fides.api.oauth.utils import get_root_client, verify_oauth_client_prod
-from fides.api.service.connectors.saas.connector_registry_service import (
-    update_saas_configs,
-)
 
 # pylint: disable=wildcard-import, unused-wildcard-import
 from fides.api.service.saas_request.override_implementations import *
@@ -55,6 +52,7 @@ from fides.api.util.rate_limit import (
     fides_limiter,
     is_rate_limit_enabled,
 )
+from fides.api.util.saas_config_updater import update_saas_configs
 from fides.config import CONFIG
 from fides.config.config_proxy import ConfigProxy
 

fides/api/common_exceptions.py

@@ -179,6 +179,10 @@ class AwaitingAsyncTask(BaseException):
     """Request Task is Awaiting Processing - Awaiting Async Task"""
 
 
+class AwaitingAsyncProcessing(BaseException):
+    """Request Task is actively being processed by external system - Fides is polling"""
+
+
 class UpstreamTasksNotReady(BaseException):
     """Privacy Request Task awaiting upstream tasks"""
 
@@ -402,3 +406,7 @@ class MonitorConfigNotFoundException(BaseException):
 
 class MissingNamespaceSchemaException(BaseException):
     """Attempting to use namespace fides_meta without specifying the schema to validate it."""
+
+
+class ConnectionNotFoundException(BaseException):
+    """ConnectionConfig could not be found"""

fides/api/db/base.py

@@ -33,6 +33,7 @@ from fides.api.models.fides_user_permissions import FidesUserPermissions
 from fides.api.models.fides_user_respondent_email_verification import (
     FidesUserRespondentEmailVerification,
 )
+from fides.api.models.identity_definition import IdentityDefinition
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
 from fides.api.models.manual_task import (
@@ -95,3 +96,4 @@ from fides.api.models.tcf_publisher_restrictions import (
     TCFPublisherRestriction,
 )
 from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride
+from fides.api.models.v3.privacy_preferences import PrivacyPreferences

fides/api/db/database.py

@@ -2,14 +2,19 @@
 Contains all of the logic related to the database including connections, setup, teardown, etc.
 """
 
+import os
+import subprocess
 from os import path
+from pathlib import Path
 from typing import Literal, Optional, Tuple
 
 from alembic import command, script
 from alembic.config import Config
 from alembic.runtime import migration
 from loguru import logger as log
+from sqlalchemy import text
 from sqlalchemy.orm import Session
+from sqlalchemy.schema import SchemaItem
 from sqlalchemy_utils.functions import create_database, database_exists
 from sqlalchemy_utils.types.encrypted.encrypted_type import InvalidCiphertextError
 
@@ -20,6 +25,25 @@ from fides.api.util.errors import get_full_exception_name
 
 DatabaseHealth = Literal["healthy", "unhealthy", "needs migration"]
 
+# Tables to exclude from migration auto-generation (e.g., tables without SQLAlchemy models)
+EXCLUDED_TABLES = {
+    "privacy_preferences_current",
+    "privacy_preferences_historic",
+}
+
+
+def include_object(
+    object: SchemaItem,  # pylint: disable=redefined-builtin
+    name: str,
+    type_: str,
+    reflected: bool,
+    compare_to: SchemaItem,
+) -> bool:
+    """Filter out excluded tables from migration comparison and auto-generation."""
+    if type_ == "table" and name in EXCLUDED_TABLES:
+        return False
+    return True
+
 
 def get_alembic_config(database_url: str) -> Config:
     """
@@ -91,6 +115,10 @@ def reset_db(database_url: str) -> None:
         log.info("Dropping tables...")
         Base.metadata.drop_all(connection)
 
+        log.info("Dropping excluded tables without models...")
+        for table_name in EXCLUDED_TABLES:
+            connection.execute(text(f"DROP TABLE IF EXISTS {table_name} CASCADE"))
+
         log.info("Dropping Alembic table...")
         migration_context = migration.MigrationContext.configure(connection)
         version = migration_context._version  # pylint: disable=protected-access
@@ -151,13 +179,240 @@ def check_missing_migrations(database_url: str) -> None:
     Tries to autogenerate migrations, returns True if a migration
     was generated.
     """
-
     engine = get_db_engine(database_uri=database_url)
     connection = engine.connect()
 
-    migration_context = migration.MigrationContext.configure(connection)
+    migration_context = migration.MigrationContext.configure(
+        connection, opts={"include_object": include_object}
+    )
     result = command.autogen.compare_metadata(migration_context, Base.metadata)  # type: ignore[attr-defined]
 
     if result:
+        log.error("Migration needs to be generated!")
+        log.error("Detected the following schema differences:")
+        for item in result:
+            log.error(f"  - {item}")
         raise SystemExit("Migration needs to be generated!")
     print("No migrations need to be generated.")
+
+
+def _get_base_branch_for_comparison() -> str:
+    """
+    Determine the base branch to compare against for detecting new migrations.
+
+    In GitHub Actions PRs, uses GITHUB_BASE_REF (the PR target branch).
+    Otherwise, falls back to origin/main for local development.
+
+    Returns:
+        The base branch reference (e.g., 'origin/main', 'origin/develop')
+    """
+    base_ref = os.environ.get("GITHUB_BASE_REF")
+    if base_ref:
+        base_branch = f"origin/{base_ref}"
+        log.info(f"Comparing against PR target branch: {base_branch}")
+    else:
+        base_branch = "origin/main"
+        log.info(f"No GITHUB_BASE_REF found, comparing against: {base_branch}")
+    return base_branch
+
+
+def _get_new_migration_files(repo_root: str, base_branch: str) -> list[str]:
+    """
+    Get list of new migration files added in the current branch.
+
+    Files are sorted by the timestamp in their filename to ensure chronological order.
+    Migration files follow the pattern: xx_YYYY_MM_DD_HHMM_revision_description.py
+
+    Args:
+        repo_root: Absolute path to the git repository root
+        base_branch: The base branch to compare against
+
+    Returns:
+        List of relative file paths to new migration files, sorted chronologically
+
+    Raises:
+        SystemExit: If the migrations directory does not exist
+    """
+    # Validate that the migrations directory exists
+    migrations_dir = (
+        Path(repo_root)
+        / "src"
+        / "fides"
+        / "api"
+        / "alembic"
+        / "migrations"
+        / "versions"
+    )
+    if not migrations_dir.exists():
+        log.error(f"Migrations directory not found: {migrations_dir}")
+        raise SystemExit(
+            f"Migration directory not found at expected path: {migrations_dir}\n"
+            "If the migrations directory has been moved, update the path in "
+            "fides.api.db.database._get_new_migration_files()"
+        )
+
+    migration_pattern = "src/fides/api/alembic/migrations/versions/*.py"
+    diff_output = subprocess.check_output(
+        [
+            "git",
+            "diff",
+            "--name-only",
+            "--diff-filter=A",  # Only added files
+            f"{base_branch}...HEAD",
+            "--",
+            migration_pattern,
+        ],
+        stderr=subprocess.STDOUT,
+        text=True,
+        cwd=repo_root,
+    ).strip()
+
+    if not diff_output:
+        return []
+
+    migration_files = [f for f in diff_output.split("\n") if f]
+
+    # Sort by filename to ensure chronological order
+    # Files are named: xx_YYYY_MM_DD_HHMM_revision_description.py
+    # Sorting alphabetically will sort them chronologically due to the date format
+    migration_files.sort()
+
+    return migration_files
+
+
+def _parse_migration_revisions(migration_path: Path) -> tuple[str, Optional[str]]:
+    """
+    Extract revision and down_revision IDs from a migration file.
+
+    Args:
+        migration_path: Path to the migration file
+
+    Returns:
+        Tuple of (revision, down_revision). down_revision may be None.
+
+    Raises:
+        SystemExit: If revision ID cannot be found in the file
+    """
+    revision = None
+    down_revision = None
+
+    with open(migration_path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip().startswith("revision ="):
+                revision = line.split("=")[1].strip().strip("\"'")
+            elif line.strip().startswith("down_revision ="):
+                down_rev_value = line.split("=")[1].strip().strip("\"'")
+                if down_rev_value.lower() != "none":
+                    down_revision = down_rev_value
+
+    if not revision:
+        log.error(f"Could not find revision ID in {migration_path}")
+        raise SystemExit(f"Invalid migration file: {migration_path}")
+
+    return revision, down_revision
+
+
+def _test_migrations_upgrade_downgrade_sequence(
+    alembic_config: Config,
+    migrations: list[tuple[str, str, Optional[str]]],
+) -> None:
+    """
+    Test multiple migrations can be upgraded and downgraded successfully in sequence.
+
+    Tests all new migrations together by:
+    1. Downgrading incrementally through all migrations (newest to oldest)
+    2. Upgrading incrementally back through all migrations (oldest to newest)
+
+    After Phase 2, the database is at the newest migration revision, which should
+    match HEAD for the new migrations being tested.
+
+    This approach tests migrations in a realistic sequence and can catch
+    interaction issues between migrations.
+
+    Args:
+        alembic_config: Alembic configuration
+        migrations: List of (migration_name, revision, down_revision) tuples
+
+    Raises:
+        SystemExit: If any migration upgrade/downgrade test fails
+    """
+    if not migrations:
+        return
+
+    log.info(f"Testing {len(migrations)} migration(s) in sequence")
+
+    try:
+        # Phase 1: Downgrade through all migrations (newest to oldest)
+        log.info("Phase 1: Downgrading through all new migrations...")
+        for migration_name, revision, down_revision in reversed(migrations):
+            if down_revision:
+                log.info(f"  Downgrading {migration_name} to {down_revision}...")
+                downgrade_db(alembic_config, down_revision)
+            else:
+                log.info(f"  Skipping {migration_name} (no down_revision)")
+
+        # Phase 2: Upgrade back through all migrations (oldest to newest)
+        log.info("Phase 2: Upgrading back through all new migrations...")
+        for migration_name, revision, down_revision in migrations:
+            log.info(f"  Upgrading {migration_name} to {revision}...")
+            upgrade_db(alembic_config, revision)
+
+        log.info(f"✓ All {len(migrations)} migration(s) passed upgrade/downgrade test")
+
+    except Exception as error:
+        log.error("Migration upgrade/downgrade sequence test failed!")
+        log.error(str(error))
+        raise SystemExit("Migration upgrade/downgrade test failed")
+
+
+def check_new_migrations_upgrade_downgrade(database_url: str) -> None:
+    """
+    Detects new migrations added in the current branch and tests that they
+    can be successfully upgraded and downgraded.
+
+    Uses git to compare against the target/base branch to find new migration files.
+    In CI, uses GITHUB_BASE_REF if available, otherwise falls back to origin/main.
+    """
+    try:
+        # Get the repo root directory
+        repo_root = subprocess.check_output(
+            ["git", "rev-parse", "--show-toplevel"],
+            stderr=subprocess.STDOUT,
+            text=True,
+        ).strip()
+
+        # Determine base branch and get new migration files
+        base_branch = _get_base_branch_for_comparison()
+        migration_files = _get_new_migration_files(repo_root, base_branch)
+
+        if not migration_files:
+            log.info("No new migration files detected in current branch")
+            print("No new migrations to test.")
+            return
+
+        log.info(f"Detected {len(migration_files)} new migration(s) to test")
+
+        # Log the migrations in order
+        for i, file_path in enumerate(migration_files, 1):
+            log.info(f"  {i}. {Path(file_path).name}")
+
+        alembic_config = get_alembic_config(database_url)
+
+        # Parse all migrations first
+        migrations = []
+        for file_path in migration_files:
+            abs_path = Path(repo_root) / file_path
+            revision, down_revision = _parse_migration_revisions(abs_path)
+            migrations.append((abs_path.name, revision, down_revision))
+
+        # Test all migrations in sequence
+        _test_migrations_upgrade_downgrade_sequence(alembic_config, migrations)
+
+        print(
+            f"All {len(migration_files)} new migration(s) passed upgrade/downgrade tests."
+        )
+
+    except subprocess.CalledProcessError as error:
+        # Git command failed - might not be in a git repo or no origin/main
+        log.warning(f"Could not detect new migrations via git: {error}")
+        print("Skipping migration upgrade/downgrade check (git detection failed).")

fides/api/email_templates/get_email_template.py

@@ -9,6 +9,7 @@ from fides.api.email_templates.template_names import (
     CONSENT_REQUEST_VERIFICATION_TEMPLATE,
     EMAIL_ERASURE_REQUEST_FULFILLMENT,
     EXTERNAL_USER_WELCOME,
+    MANUAL_TASK_DIGEST,
     PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE,
     PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE,
     PRIVACY_REQUEST_ERROR_NOTIFICATION_TEMPLATE,
@@ -59,6 +60,8 @@ def get_email_template(  # pylint: disable=too-many-return-statements, too-many-
         return template_env.get_template(USER_INVITE)
     if action_type == MessagingActionType.EXTERNAL_USER_WELCOME:
         return template_env.get_template(EXTERNAL_USER_WELCOME)
+    if action_type == MessagingActionType.MANUAL_TASK_DIGEST:
+        return template_env.get_template(MANUAL_TASK_DIGEST)
 
     logger.error("No corresponding template linked to the {}", action_type)
     raise EmailTemplateUnhandledActionType(

fides/api/email_templates/template_names.py

@@ -11,3 +11,4 @@ PRIVACY_REQUEST_REVIEW_APPROVE_TEMPLATE = "privacy_request_review_approve.html"
 TEST_MESSAGE_TEMPLATE = "test_message.html"
 USER_INVITE = "user_invite.html"
 EXTERNAL_USER_WELCOME = "external_user_welcome.html"
+MANUAL_TASK_DIGEST = "manual_task_digest.html"