ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/api/util/connection_util.py

@@ -6,30 +6,17 @@ from fideslang.validation import FidesKey
 from loguru import logger
 from pydantic import Field, ValidationError
 from sqlalchemy.orm import Session
-from starlette.status import (
-    HTTP_400_BAD_REQUEST,
-    HTTP_404_NOT_FOUND,
-    HTTP_422_UNPROCESSABLE_ENTITY,
-)
+from starlette.status import HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY
 
 from fides.api.api import deps
 from fides.api.common_exceptions import (
-    ClientUnsuccessfulException,
-    ConnectionException,
+    ConnectionNotFoundException,
     KeyOrNameAlreadyExists,
+    SaaSConfigNotFoundException,
 )
 from fides.api.common_exceptions import ValidationError as FidesValidationError
-from fides.api.models.connectionconfig import (
-    ConnectionConfig,
-    ConnectionTestStatus,
-    ConnectionType,
-)
+from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.models.datasetconfig import DatasetConfig
-from fides.api.models.manual_task import (
-    ManualTask,
-    ManualTaskParentEntityType,
-    ManualTaskType,
-)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.models.sql_models import Dataset as CtlDataset  # type: ignore
@@ -38,7 +25,6 @@ from fides.api.schemas.api import BulkUpdateFailed
 from fides.api.schemas.connection_configuration import (
     ConnectionConfigSecretsSchema,
     connection_secrets_schemas,
-    get_connection_secrets_schema,
 )
 from fides.api.schemas.connection_configuration.connection_config import (
     BulkPutConnectionConfiguration,
@@ -48,23 +34,13 @@ from fides.api.schemas.connection_configuration.connection_config import (
 from fides.api.schemas.connection_configuration.connection_secrets import (
     TestStatusMessage,
 )
-from fides.api.schemas.connection_configuration.connection_secrets_dynamic_erasure_email import (
-    validate_dynamic_erasure_email_dataset_references,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_saas import (
-    validate_saas_secrets_external_references,
-)
-from fides.api.schemas.connection_configuration.saas_config_template_values import (
-    SaasConnectionTemplateValues,
-)
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
-from fides.api.service.connectors import get_connector
-from fides.api.service.connectors.saas.connector_registry_service import (
-    ConnectorRegistry,
-    create_connection_config_from_template_no_save,
+from fides.common.api.v1.urn_registry import SAAS_CONFIG
+from fides.service.connection.connection_service import (
+    ConnectionService,
+    ConnectorTemplateNotFound,
 )
-from fides.api.util.logger import Pii
-from fides.common.api.v1.urn_registry import CONNECTION_TYPES, SAAS_CONFIG
+from fides.service.event_audit_service import EventAuditService
 from fides.service.privacy_request.privacy_request_service import queue_privacy_request
 
 # pylint: disable=too-many-nested-blocks,too-many-branches,too-many-statements
@@ -95,7 +71,7 @@ def requeue_requires_input_requests(db: Session) -> None:
             )
 
 
-def _validate_secrets_error_message() -> str:
+def validate_secrets_error_message() -> str:
     return f"A SaaS config to validate the secrets is unavailable for this connection config, please add one via {SAAS_CONFIG}"
 
 
@@ -106,56 +82,30 @@ def validate_secrets(
 ) -> ConnectionConfigSecretsSchema:
     """Validate incoming connection configuration secrets."""
 
-    connection_type = connection_config.connection_type
-    saas_config = connection_config.get_saas_config()
-    if connection_type == ConnectionType.saas and saas_config is None:
-        raise HTTPException(
-            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=_validate_secrets_error_message(),
-        )
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
 
     try:
-        schema = get_connection_secrets_schema(connection_type.value, saas_config)  # type: ignore
-        logger.info(
-            "Validating secrets on connection config with key '{}'",
-            connection_config.key,
+        return connection_service.validate_secrets(request_body, connection_config)
+    except SaaSConfigNotFoundException:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=validate_secrets_error_message(),
         )
-        connection_secrets = schema.model_validate(request_body)
+    except FidesValidationError as e:
+        # Check if the exception has the original pydantic errors attached
+        raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message)
     except ValidationError as e:
-        # Intentionally excluding the pydantic url, input, and ctx fields from the error response to hide
-        # potentially sensitive information
         errors = e.errors(include_url=False, include_input=False)
         for err in errors:
             # Additionally, manually remove the context from the error message -
             # this may contain sensitive information
             err.pop("ctx", None)
-
         raise HTTPException(
             status_code=HTTP_422_UNPROCESSABLE_ENTITY,
             detail=jsonable_encoder(errors),
         )
 
-    # SaaS secrets with external references must go through extra validation
-    if connection_type == ConnectionType.saas:
-        try:
-            validate_saas_secrets_external_references(db, schema, connection_secrets)  # type: ignore
-
-        except FidesValidationError as e:
-            raise HTTPException(
-                status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message
-            )
-
-    # For dynamic erasure emails we must validate the recipient email address
-    if connection_type == ConnectionType.dynamic_erasure_email:
-        try:
-            validate_dynamic_erasure_email_dataset_references(db, connection_secrets)
-        except FidesValidationError as e:
-            raise HTTPException(
-                status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message
-            )
-
-    return connection_secrets
-
 
 def patch_connection_configs(
     db: Session,
@@ -166,134 +116,20 @@ def patch_connection_configs(
     failed: List[BulkUpdateFailed] = []
     logger.info("Starting bulk upsert for {} connection configuration(s)", len(configs))
 
-    for config in configs:
-        # Retrieve the existing connection config from the database
-        existing_connection_config = None
-        if config.key:
-            existing_connection_config = ConnectionConfig.get_by(
-                db, field="key", value=config.key
-            )
-
-        if config.connection_type == "saas":
-            if config.secrets:
-                # This is here rather than with the get_connection_config_or_error because
-                # it will also throw an HTTPException if validation fails and we don't want
-                # to catch it in this case.
-                if existing_connection_config:
-                    config.secrets = validate_secrets(
-                        db, config.secrets, existing_connection_config
-                    )
-                else:
-                    if not config.saas_connector_type:
-                        raise HTTPException(
-                            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
-                            detail="saas_connector_type is missing",
-                        )
-
-                    connector_template = ConnectorRegistry.get_connector_template(
-                        config.saas_connector_type
-                    )
-                    if not connector_template:
-                        raise HTTPException(
-                            status_code=HTTP_404_NOT_FOUND,
-                            detail=f"SaaS connector type '{config.saas_connector_type}' is not yet available in Fides. For a list of available SaaS connectors, refer to {CONNECTION_TYPES}.",
-                        )
-                    try:
-                        template_values = SaasConnectionTemplateValues(
-                            name=config.name,
-                            key=config.key,
-                            description=config.description,
-                            secrets=config.secrets,
-                            instance_key=config.key,
-                        )
-                        if system:
-                            connection_config = (
-                                create_connection_config_from_template_no_save(
-                                    db,
-                                    connector_template,
-                                    template_values,
-                                    system_id=system.id,
-                                )
-                            )
-                        else:
-                            connection_config = (
-                                create_connection_config_from_template_no_save(
-                                    db,
-                                    connector_template,
-                                    template_values,
-                                )
-                            )
-                    except KeyOrNameAlreadyExists as exc:
-                        raise HTTPException(
-                            status_code=HTTP_400_BAD_REQUEST,
-                            detail=exc.args[0],
-                        )
-                    except ValidationError as e:
-                        # The "input" potentially contains sensitive info and the Pydantic-specific "url" is not helpful
-                        errors = e.errors(include_url=False, include_input=False)
-                        for err in errors:
-                            # Additionally, manually remove the context from the error message -
-                            # this may contain sensitive information
-                            err.pop("ctx", None)
-
-                        raise HTTPException(
-                            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
-                            detail=jsonable_encoder(errors),
-                        )
-
-                    connection_config.secrets = validate_secrets(
-                        db,
-                        template_values.secrets,
-                        connection_config,
-                    ).model_dump(mode="json")
-                    connection_config.save(db=db)
-                    created_or_updated.append(
-                        ConnectionConfigurationResponse.model_validate(
-                            connection_config
-                        )
-                    )
-                    continue
+    # Initialize the connection service
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
 
+    for config in configs:
         orig_data = config.model_dump(serialize_as_any=True, mode="json").copy()
-        config_dict = config.model_dump(serialize_as_any=True, exclude_unset=True)
-        config_dict.pop("saas_connector_type", None)
-
-        if existing_connection_config:
-            config_dict = {
-                key: value
-                for key, value in {
-                    **existing_connection_config.__dict__,
-                    **config.model_dump(serialize_as_any=True, exclude_unset=True),
-                }.items()
-                if isinstance(value, bool) or value
-            }
-
-        if system:
-            config_dict["system_id"] = system.id
 
         try:
-            connection_config = ConnectionConfig.create_or_update(
-                db, data=config_dict, check_name=False
+            # Use the service to handle individual connection config processing
+            connection_response = connection_service.create_or_update_connection_config(
+                config, system
             )
+            created_or_updated.append(connection_response)
 
-            # Automatically create a ManualTask if this is a connection config of type manual_task
-            # and it doesn't already have one
-            if (
-                connection_config.connection_type == ConnectionType.manual_task
-                and not connection_config.manual_task
-            ):
-                ManualTask.create(
-                    db=db,
-                    data={
-                        "task_type": ManualTaskType.privacy_request,
-                        "parent_entity_id": connection_config.id,
-                        "parent_entity_type": ManualTaskParentEntityType.connection_config,
-                    },
-                )
-
-            created_or_updated.append(
-                ConnectionConfigurationResponse.model_validate(connection_config)
-            )
         except KeyOrNameAlreadyExists as exc:
             logger.warning(
                 "Create/update failed for connection config with key '{}': {}",
@@ -309,6 +145,29 @@ def patch_connection_configs(
                     data=orig_data,
                 )
             )
+        except ConnectorTemplateNotFound as exc:
+            raise HTTPException(
+                status_code=HTTP_404_NOT_FOUND,
+                detail=str(exc),
+            )
+        except ValidationError as e:
+            # The "input" potentially contains sensitive info and the Pydantic-specific "url" is not helpful
+            errors = e.errors(include_url=False, include_input=False)
+            for err in errors:
+                # Additionally, manually remove the context from the error message -
+                # this may contain sensitive information
+                err.pop("ctx", None)
+
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=jsonable_encoder(errors),
+            )
+        except ValueError as exc:
+            # Handle missing saas_connector_type and other validation errors
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=str(exc),
+            )
         except Exception as e:
             logger.warning(
                 "Create/update failed for connection config with key '{}'.", config.key
@@ -327,6 +186,12 @@ def patch_connection_configs(
     # Check if possibly disabling a manual webhook here causes us to need to queue affected privacy requests
     requeue_requires_input_requests(db)
 
+    logger.info(
+        "Completed bulk upsert for {} connection configuration(s): {} succeeded, {} failed",
+        len(configs),
+        len(created_or_updated),
+        len(failed),
+    )
     return BulkPutConnectionConfiguration(
         succeeded=created_or_updated,
         failed=failed,
@@ -352,6 +217,12 @@ def delete_connection_config(db: Session, connection_key: FidesKey) -> None:
     connection_config = get_connection_config_or_error(db, connection_key)
     connection_type = connection_config.connection_type
     logger.info("Deleting connection config with key '{}'.", connection_key)
+
+    # Initialize services for audit logging
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
+
+    # Handle SaaS-specific cleanup
     if connection_config.saas_config:
         saas_dataset_fides_key = connection_config.saas_config.get("fides_key")
 
@@ -369,7 +240,8 @@ def delete_connection_config(db: Session, connection_key: FidesKey) -> None:
             )
             saas_dataset.delete(db)  # type: ignore[union-attr]
 
-    connection_config.delete(db)
+    # Use connection service to delete with audit logging
+    connection_service.delete_connection_config(connection_key)
 
     # Access Manual Webhooks are cascade deleted if their ConnectionConfig is deleted,
     # so we queue any privacy requests that are no longer blocked by webhooks
@@ -382,30 +254,42 @@ def connection_status(
 ) -> TestStatusMessage:
     """Connect, verify with a trivial query or API request, and report the status."""
 
-    connector = get_connector(connection_config)
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
+    return connection_service.connection_status(connection_config, msg)
 
-    try:
-        status: Optional[ConnectionTestStatus] = connector.test_connection()
 
-    except (ConnectionException, ClientUnsuccessfulException) as exc:
-        logger.warning(
-            "Connection test failed on {}: {}",
-            connection_config.key,
-            Pii(str(exc)),
+def update_connection_secrets(
+    connection_service: ConnectionService,
+    connection_key: FidesKey,
+    unvalidated_secrets: connection_secrets_schemas,
+    verify: Optional[bool],
+    merge_with_existing: Optional[bool] = False,
+) -> TestStatusMessage:
+    try:
+        return connection_service.update_secrets(
+            connection_key, unvalidated_secrets, verify, merge_with_existing
+        )
+    except ConnectionNotFoundException:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"No connection config found with key {connection_key}",
         )
-        connection_config.update_test_status(
-            test_status=ConnectionTestStatus.failed, db=db
+    except SaaSConfigNotFoundException:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=validate_secrets_error_message(),
         )
-        return TestStatusMessage(
-            msg=msg,
-            test_status=ConnectionTestStatus.failed,
-            failure_reason=str(exc),
+    except ValidationError as e:
+        errors = e.errors(include_url=False, include_input=False)
+        for err in errors:
+            # Additionally, manually remove the context from the error message -
+            # this may contain sensitive information
+            err.pop("ctx", None)
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=jsonable_encoder(errors),
         )
-
-    logger.info("Connection test {} on {}", status.value, connection_config.key)  # type: ignore
-    connection_config.update_test_status(test_status=status, db=db)  # type: ignore
-
-    return TestStatusMessage(
-        msg=msg,
-        test_status=status,
-    )
+    except FidesValidationError as e:
+        # Check if the exception has the original pydantic errors attached
+        raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message)

fides/api/util/event_audit_util.py

@@ -0,0 +1,230 @@
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, Optional, Tuple
+
+from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.event_audit import EventAuditType
+from fides.api.schemas.connection_configuration.connection_oauth_config import (
+    OAuthConfigSchema,
+)
+from fides.api.util.connection_type import get_connection_type_secret_schema
+from fides.api.util.masking_util import mask_sensitive_fields
+
+
+def normalize_value(value: Any) -> Any:
+    """
+    Normalize values for consistent comparison and serialization.
+
+    - Converts datetime objects to ISO strings
+    - Converts enums to their string values
+    - Converts lists of enums to lists of string values
+    - Treats None and empty strings as equivalent (both become None)
+    """
+    # Treat None and empty strings as equivalent
+    if value == "" or value is None:
+        return None
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, Enum):
+        return value.value
+    if (
+        isinstance(value, list) and value and hasattr(value[0], "value")
+    ):  # List of enums
+        return [item.value for item in value]
+    return value
+
+
+def _get_saas_connector_type(connection_config: ConnectionConfig) -> Optional[str]:
+    """
+    Helper function to extract SaaS connector type from connection config.
+
+    Args:
+        connection_config: The connection configuration
+
+    Returns:
+        SaaS connector type if available, None otherwise
+    """
+    connection_type = connection_config.connection_type.value  # type: ignore[attr-defined]
+    if connection_type == "saas" and connection_config.saas_config:
+        try:
+            saas_config = connection_config.get_saas_config()
+            if saas_config:
+                return saas_config.type
+        except Exception:
+            # If SaaS config is invalid, try to get type directly from raw config
+            if (
+                isinstance(connection_config.saas_config, dict)
+                and "type" in connection_config.saas_config
+            ):
+                return connection_config.saas_config["type"]
+    return None
+
+
+def _create_connection_event_details(
+    connection_config: ConnectionConfig,
+    operation_type: str,
+    changed_fields: Optional[set] = None,
+) -> Dict[str, Any]:
+    """
+    Create standardized event details for connection audit events.
+
+    Args:
+        connection_config: The connection configuration
+        operation_type: Type of operation (create, update, delete)
+        changed_fields: Set of field names that changed (None means include all fields)
+
+    Returns:
+        Standardized event details dictionary
+    """
+    event_details: dict[str, Any] = {
+        "operation_type": operation_type,
+    }
+
+    # Define fields to exclude from audit details
+    excluded_fields = {
+        "id",
+        "created_at",
+        "updated_at",
+        "secrets",
+    }
+
+    # Dynamically add connection config fields (only changed fields if specified)
+    configuration_changes: dict[str, Any] = {}
+    for column in connection_config.__table__.columns:
+        field_name = column.name
+        if field_name not in excluded_fields:
+            # If changed_fields is specified, only include fields that changed
+            if changed_fields is not None and field_name not in changed_fields:
+                continue
+
+            value = getattr(connection_config, field_name)
+
+            configuration_changes[field_name] = normalize_value(value)
+
+    event_details["configuration_changes"] = configuration_changes
+    return event_details
+
+
+def generate_connection_audit_event_details(
+    event_type: EventAuditType,
+    connection_config: ConnectionConfig,
+    description: Optional[str] = None,
+    changed_fields: Optional[set] = None,
+) -> Tuple[Dict[str, Any], str]:
+    """
+    Create an audit event for connection operations.
+
+    Args:
+        event_type: Type of audit event
+        connection_config: The connection configuration
+        description: Human-readable description
+        changed_fields: Set of field names that changed (None means include all fields)
+
+    Returns:
+        Created EventAudit instance
+    """
+    # Determine operation type from event type
+    operation_type = event_type.value.split(".")[
+        -1
+    ]  # e.g., "created", "updated", "deleted"
+
+    # Create standardized event details
+    if operation_type == "deleted":
+        event_details = {
+            "operation_type": operation_type,
+        }
+    else:
+        event_details = _create_connection_event_details(
+            connection_config,
+            operation_type,
+            changed_fields,
+        )
+
+    # Generate description if not provided
+    connection_type = connection_config.connection_type.value  # type: ignore[attr-defined]
+    connector_type = _get_saas_connector_type(connection_config)
+
+    description = f"Connection {operation_type}: {connector_type if connector_type else connection_type} connection '{connection_config.key}'"
+
+    return event_details, description
+
+
+def generate_connection_secrets_event_details(
+    event_type: EventAuditType,
+    connection_config: ConnectionConfig,
+    secrets_modified: Dict[str, Any],
+) -> Tuple[Dict[str, Any], str]:
+    """
+    Create an audit event for connection secrets operations.
+
+    Args:
+        event_type: Type of audit event (should be a connection.secrets.* event)
+        connection_config: The connection configuration
+        secrets_modified: Dict of secret field names and values that were modified
+        changed_fields: Set of field names that changed (None means include all fields)
+        description: Human-readable description
+
+    Returns:
+        Created EventAudit instance
+    """
+    # Determine operation type from event type
+    operation_type = event_type.value.split(".")[
+        -1
+    ]  # e.g., "created", "updated", "deleted"
+
+    # Mask the secrets using the existing masking utility
+    saas_config = connection_config.get_saas_config()
+    connection_type = connection_config.connection_type.value  # type: ignore[attr-defined]
+    connection_type = (
+        saas_config.type
+        if connection_type == "saas" and saas_config
+        else connection_type
+    )
+
+    # Get the secret schema for this connection type
+    try:
+        secret_schema = None
+
+        # For HTTPS connections with OAuth config, check if we should use OAuth schema
+        if connection_type == "https" and connection_config.oauth_config:
+            # Get OAuth fields dynamically from the schema
+            oauth_schema = OAuthConfigSchema.model_json_schema()
+            oauth_fields = set(oauth_schema.get("properties", {}).keys())
+            secrets_keys = set(secrets_modified.keys())
+
+            # If we're updating OAuth fields, use OAuth schema
+            if secrets_keys.intersection(oauth_fields):
+                secret_schema = oauth_schema
+
+        # If we haven't set a schema yet, use the connection type's secret schema
+        if secret_schema is None:
+            secret_schema = get_connection_type_secret_schema(
+                connection_type=connection_type
+            )
+
+        # Use existing masking function
+        masked_secrets = mask_sensitive_fields(secrets_modified, secret_schema)
+    except NoSuchConnectionTypeSecretSchemaError:
+        # if there is no schema, mask every secret
+        masked_secrets = {key: "**********" for key in secrets_modified}
+
+    # Create event details with masked secret values
+    event_details: dict[str, Any] = {
+        "secrets": masked_secrets,
+    }
+
+    # Add SaaS connector type if applicable
+    connector_type = _get_saas_connector_type(connection_config)
+    if connector_type:
+        event_details["saas_connector_type"] = connector_type
+
+    # Generate description
+    secret_names = ", ".join(secrets_modified) if secrets_modified else "secrets"
+
+    if connector_type:
+        description = f"Connection secrets {operation_type}: {connector_type} connection '{connection_config.key}' - {secret_names}"
+    else:
+        description = f"Connection secrets {operation_type}: {connection_type} connection '{connection_config.key}' - {secret_names}"
+
+    return event_details, description

fides/api/util/logger_context_utils.py

@@ -165,7 +165,9 @@ def request_details(
 
     if response is not None:
         if CONFIG.dev_mode and response.content:
-            details[LoggerContextKeys.response.value] = response.content.decode("utf-8")
+            details[LoggerContextKeys.response.value] = response.content.decode(
+                "utf-8", errors="replace"
+            )
 
         details[LoggerContextKeys.status_code.value] = response.status_code