ethyca-fides 2.71.2b0__py2.py3-none-any.whl → 2.72.0rc0__py2.py3-none-any.whl

fides/api/service/messaging/message_dispatch_service.py

@@ -28,6 +28,7 @@ from fides.api.schemas.messaging.messaging import (
     ErrorNotificationBodyParams,
     ExternalUserWelcomeBodyParams,
     FidesopsMessage,
+    ManualTaskDigestBodyParams,
     MessagingActionType,
     MessagingMethod,
     MessagingServiceDetails,
@@ -177,6 +178,7 @@ def dispatch_message(
             UserInviteBodyParams,
             ErrorNotificationBodyParams,
             ExternalUserWelcomeBodyParams,
+            ManualTaskDigestBodyParams,
         ]
     ] = None,
     subject_override: Optional[str] = None,
@@ -352,7 +354,7 @@ def _render(template_str: str, variables: Optional[Dict] = None) -> str:
     return template_str
 
 
-def _build_email(  # pylint: disable=too-many-return-statements, too-many-branches
+def _build_email(  # pylint: disable=too-many-return-statements, too-many-branches, too-many-statements
     config_proxy: ConfigProxy,
     action_type: MessagingActionType,
     body_params: Any,
@@ -494,6 +496,41 @@ def _build_email(  # pylint: disable=too-many-return-statements, too-many-branch
             template_variables=variables,
         )
 
+    if action_type == MessagingActionType.MANUAL_TASK_DIGEST:
+        variables = {
+            "vendor_contact_name": body_params.vendor_contact_name,
+            "organization_name": body_params.organization_name,
+            "portal_url": body_params.portal_url,
+            "imminent_task_count": body_params.imminent_task_count,
+            "upcoming_task_count": body_params.upcoming_task_count,
+            "total_task_count": body_params.total_task_count,
+            "company_logo_url": body_params.company_logo_url,
+        }
+
+        # Start with default subject
+        subject = f"Weekly DSR Summary from {body_params.organization_name}"
+
+        # If messaging template exists, extract customizable content
+        if messaging_template:
+            # Use custom subject if provided
+            if "subject" in messaging_template.content:
+                subject = _render(messaging_template.content["subject"], variables)
+
+            # Use custom body content to replace the intro text in HTML template
+            custom_body = messaging_template.content.get("body", "")
+            if custom_body.strip():
+                # Replace the default intro text with the custom body content
+                rendered_custom_body = _render(custom_body, variables)
+                variables["intro_text"] = rendered_custom_body
+
+        # Always use the HTML template
+        base_template = get_email_template(action_type)
+        return EmailForActionType(
+            subject=subject,
+            body=base_template.render(variables),
+            template_variables=variables,
+        )
+
     logger.error("Message action type {} is not implemented", action_type)
     raise MessageDispatchException(
         f"Message action type {action_type} is not implemented"

fides/api/tasks/__init__.py

@@ -86,7 +86,6 @@ class DatabaseTask(Task):  # pylint: disable=W0223
         # to prevent session overlap when requests are executing concurrently
         # when in task_always_eager mode (i.e. without proper workers)
         new_session = self._sessionmaker()
-        logger.debug(f"DatabaseTaskSession ID: {id(new_session)}. Self ID: {id(self)}")
         return new_session
 
 

fides/api/util/connection_util.py

@@ -6,30 +6,17 @@ from fideslang.validation import FidesKey
 from loguru import logger
 from pydantic import Field, ValidationError
 from sqlalchemy.orm import Session
-from starlette.status import (
-    HTTP_400_BAD_REQUEST,
-    HTTP_404_NOT_FOUND,
-    HTTP_422_UNPROCESSABLE_ENTITY,
-)
+from starlette.status import HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY
 
 from fides.api.api import deps
 from fides.api.common_exceptions import (
-    ClientUnsuccessfulException,
-    ConnectionException,
+    ConnectionNotFoundException,
     KeyOrNameAlreadyExists,
+    SaaSConfigNotFoundException,
 )
 from fides.api.common_exceptions import ValidationError as FidesValidationError
-from fides.api.models.connectionconfig import (
-    ConnectionConfig,
-    ConnectionTestStatus,
-    ConnectionType,
-)
+from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.models.datasetconfig import DatasetConfig
-from fides.api.models.manual_task import (
-    ManualTask,
-    ManualTaskParentEntityType,
-    ManualTaskType,
-)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.models.sql_models import Dataset as CtlDataset  # type: ignore
@@ -38,7 +25,6 @@ from fides.api.schemas.api import BulkUpdateFailed
 from fides.api.schemas.connection_configuration import (
     ConnectionConfigSecretsSchema,
     connection_secrets_schemas,
-    get_connection_secrets_schema,
 )
 from fides.api.schemas.connection_configuration.connection_config import (
     BulkPutConnectionConfiguration,
@@ -48,23 +34,13 @@ from fides.api.schemas.connection_configuration.connection_config import (
 from fides.api.schemas.connection_configuration.connection_secrets import (
     TestStatusMessage,
 )
-from fides.api.schemas.connection_configuration.connection_secrets_dynamic_erasure_email import (
-    validate_dynamic_erasure_email_dataset_references,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_saas import (
-    validate_saas_secrets_external_references,
-)
-from fides.api.schemas.connection_configuration.saas_config_template_values import (
-    SaasConnectionTemplateValues,
-)
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
-from fides.api.service.connectors import get_connector
-from fides.api.service.connectors.saas.connector_registry_service import (
-    ConnectorRegistry,
-    create_connection_config_from_template_no_save,
+from fides.common.api.v1.urn_registry import SAAS_CONFIG
+from fides.service.connection.connection_service import (
+    ConnectionService,
+    ConnectorTemplateNotFound,
 )
-from fides.api.util.logger import Pii
-from fides.common.api.v1.urn_registry import CONNECTION_TYPES, SAAS_CONFIG
+from fides.service.event_audit_service import EventAuditService
 from fides.service.privacy_request.privacy_request_service import queue_privacy_request
 
 # pylint: disable=too-many-nested-blocks,too-many-branches,too-many-statements
@@ -95,7 +71,7 @@ def requeue_requires_input_requests(db: Session) -> None:
             )
 
 
-def _validate_secrets_error_message() -> str:
+def validate_secrets_error_message() -> str:
     return f"A SaaS config to validate the secrets is unavailable for this connection config, please add one via {SAAS_CONFIG}"
 
 
@@ -106,56 +82,30 @@ def validate_secrets(
 ) -> ConnectionConfigSecretsSchema:
     """Validate incoming connection configuration secrets."""
 
-    connection_type = connection_config.connection_type
-    saas_config = connection_config.get_saas_config()
-    if connection_type == ConnectionType.saas and saas_config is None:
-        raise HTTPException(
-            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=_validate_secrets_error_message(),
-        )
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
 
     try:
-        schema = get_connection_secrets_schema(connection_type.value, saas_config)  # type: ignore
-        logger.info(
-            "Validating secrets on connection config with key '{}'",
-            connection_config.key,
+        return connection_service.validate_secrets(request_body, connection_config)
+    except SaaSConfigNotFoundException:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=validate_secrets_error_message(),
         )
-        connection_secrets = schema.model_validate(request_body)
+    except FidesValidationError as e:
+        # Check if the exception has the original pydantic errors attached
+        raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message)
     except ValidationError as e:
-        # Intentionally excluding the pydantic url, input, and ctx fields from the error response to hide
-        # potentially sensitive information
         errors = e.errors(include_url=False, include_input=False)
         for err in errors:
             # Additionally, manually remove the context from the error message -
             # this may contain sensitive information
             err.pop("ctx", None)
-
         raise HTTPException(
             status_code=HTTP_422_UNPROCESSABLE_ENTITY,
             detail=jsonable_encoder(errors),
         )
 
-    # SaaS secrets with external references must go through extra validation
-    if connection_type == ConnectionType.saas:
-        try:
-            validate_saas_secrets_external_references(db, schema, connection_secrets)  # type: ignore
-
-        except FidesValidationError as e:
-            raise HTTPException(
-                status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message
-            )
-
-    # For dynamic erasure emails we must validate the recipient email address
-    if connection_type == ConnectionType.dynamic_erasure_email:
-        try:
-            validate_dynamic_erasure_email_dataset_references(db, connection_secrets)
-        except FidesValidationError as e:
-            raise HTTPException(
-                status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message
-            )
-
-    return connection_secrets
-
 
 def patch_connection_configs(
     db: Session,
@@ -166,134 +116,20 @@ def patch_connection_configs(
     failed: List[BulkUpdateFailed] = []
     logger.info("Starting bulk upsert for {} connection configuration(s)", len(configs))
 
-    for config in configs:
-        # Retrieve the existing connection config from the database
-        existing_connection_config = None
-        if config.key:
-            existing_connection_config = ConnectionConfig.get_by(
-                db, field="key", value=config.key
-            )
-
-        if config.connection_type == "saas":
-            if config.secrets:
-                # This is here rather than with the get_connection_config_or_error because
-                # it will also throw an HTTPException if validation fails and we don't want
-                # to catch it in this case.
-                if existing_connection_config:
-                    config.secrets = validate_secrets(
-                        db, config.secrets, existing_connection_config
-                    )
-                else:
-                    if not config.saas_connector_type:
-                        raise HTTPException(
-                            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
-                            detail="saas_connector_type is missing",
-                        )
-
-                    connector_template = ConnectorRegistry.get_connector_template(
-                        config.saas_connector_type
-                    )
-                    if not connector_template:
-                        raise HTTPException(
-                            status_code=HTTP_404_NOT_FOUND,
-                            detail=f"SaaS connector type '{config.saas_connector_type}' is not yet available in Fides. For a list of available SaaS connectors, refer to {CONNECTION_TYPES}.",
-                        )
-                    try:
-                        template_values = SaasConnectionTemplateValues(
-                            name=config.name,
-                            key=config.key,
-                            description=config.description,
-                            secrets=config.secrets,
-                            instance_key=config.key,
-                        )
-                        if system:
-                            connection_config = (
-                                create_connection_config_from_template_no_save(
-                                    db,
-                                    connector_template,
-                                    template_values,
-                                    system_id=system.id,
-                                )
-                            )
-                        else:
-                            connection_config = (
-                                create_connection_config_from_template_no_save(
-                                    db,
-                                    connector_template,
-                                    template_values,
-                                )
-                            )
-                    except KeyOrNameAlreadyExists as exc:
-                        raise HTTPException(
-                            status_code=HTTP_400_BAD_REQUEST,
-                            detail=exc.args[0],
-                        )
-                    except ValidationError as e:
-                        # The "input" potentially contains sensitive info and the Pydantic-specific "url" is not helpful
-                        errors = e.errors(include_url=False, include_input=False)
-                        for err in errors:
-                            # Additionally, manually remove the context from the error message -
-                            # this may contain sensitive information
-                            err.pop("ctx", None)
-
-                        raise HTTPException(
-                            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
-                            detail=jsonable_encoder(errors),
-                        )
-
-                    connection_config.secrets = validate_secrets(
-                        db,
-                        template_values.secrets,
-                        connection_config,
-                    ).model_dump(mode="json")
-                    connection_config.save(db=db)
-                    created_or_updated.append(
-                        ConnectionConfigurationResponse.model_validate(
-                            connection_config
-                        )
-                    )
-                    continue
+    # Initialize the connection service
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
 
+    for config in configs:
         orig_data = config.model_dump(serialize_as_any=True, mode="json").copy()
-        config_dict = config.model_dump(serialize_as_any=True, exclude_unset=True)
-        config_dict.pop("saas_connector_type", None)
-
-        if existing_connection_config:
-            config_dict = {
-                key: value
-                for key, value in {
-                    **existing_connection_config.__dict__,
-                    **config.model_dump(serialize_as_any=True, exclude_unset=True),
-                }.items()
-                if isinstance(value, bool) or value
-            }
-
-        if system:
-            config_dict["system_id"] = system.id
 
         try:
-            connection_config = ConnectionConfig.create_or_update(
-                db, data=config_dict, check_name=False
+            # Use the service to handle individual connection config processing
+            connection_response = connection_service.create_or_update_connection_config(
+                config, system
             )
+            created_or_updated.append(connection_response)
 
-            # Automatically create a ManualTask if this is a connection config of type manual_task
-            # and it doesn't already have one
-            if (
-                connection_config.connection_type == ConnectionType.manual_task
-                and not connection_config.manual_task
-            ):
-                ManualTask.create(
-                    db=db,
-                    data={
-                        "task_type": ManualTaskType.privacy_request,
-                        "parent_entity_id": connection_config.id,
-                        "parent_entity_type": ManualTaskParentEntityType.connection_config,
-                    },
-                )
-
-            created_or_updated.append(
-                ConnectionConfigurationResponse.model_validate(connection_config)
-            )
         except KeyOrNameAlreadyExists as exc:
             logger.warning(
                 "Create/update failed for connection config with key '{}': {}",
@@ -309,6 +145,29 @@ def patch_connection_configs(
                     data=orig_data,
                 )
             )
+        except ConnectorTemplateNotFound as exc:
+            raise HTTPException(
+                status_code=HTTP_404_NOT_FOUND,
+                detail=str(exc),
+            )
+        except ValidationError as e:
+            # The "input" potentially contains sensitive info and the Pydantic-specific "url" is not helpful
+            errors = e.errors(include_url=False, include_input=False)
+            for err in errors:
+                # Additionally, manually remove the context from the error message -
+                # this may contain sensitive information
+                err.pop("ctx", None)
+
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=jsonable_encoder(errors),
+            )
+        except ValueError as exc:
+            # Handle missing saas_connector_type and other validation errors
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=str(exc),
+            )
         except Exception as e:
             logger.warning(
                 "Create/update failed for connection config with key '{}'.", config.key
@@ -327,6 +186,12 @@ def patch_connection_configs(
     # Check if possibly disabling a manual webhook here causes us to need to queue affected privacy requests
     requeue_requires_input_requests(db)
 
+    logger.info(
+        "Completed bulk upsert for {} connection configuration(s): {} succeeded, {} failed",
+        len(configs),
+        len(created_or_updated),
+        len(failed),
+    )
     return BulkPutConnectionConfiguration(
         succeeded=created_or_updated,
         failed=failed,
@@ -352,6 +217,12 @@ def delete_connection_config(db: Session, connection_key: FidesKey) -> None:
     connection_config = get_connection_config_or_error(db, connection_key)
     connection_type = connection_config.connection_type
     logger.info("Deleting connection config with key '{}'.", connection_key)
+
+    # Initialize services for audit logging
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
+
+    # Handle SaaS-specific cleanup
     if connection_config.saas_config:
         saas_dataset_fides_key = connection_config.saas_config.get("fides_key")
 
@@ -369,7 +240,8 @@ def delete_connection_config(db: Session, connection_key: FidesKey) -> None:
             )
             saas_dataset.delete(db)  # type: ignore[union-attr]
 
-    connection_config.delete(db)
+    # Use connection service to delete with audit logging
+    connection_service.delete_connection_config(connection_key)
 
     # Access Manual Webhooks are cascade deleted if their ConnectionConfig is deleted,
     # so we queue any privacy requests that are no longer blocked by webhooks
@@ -382,30 +254,42 @@ def connection_status(
 ) -> TestStatusMessage:
     """Connect, verify with a trivial query or API request, and report the status."""
 
-    connector = get_connector(connection_config)
+    event_audit_service = EventAuditService(db)
+    connection_service = ConnectionService(db, event_audit_service)
+    return connection_service.connection_status(connection_config, msg)
 
-    try:
-        status: Optional[ConnectionTestStatus] = connector.test_connection()
 
-    except (ConnectionException, ClientUnsuccessfulException) as exc:
-        logger.warning(
-            "Connection test failed on {}: {}",
-            connection_config.key,
-            Pii(str(exc)),
+def update_connection_secrets(
+    connection_service: ConnectionService,
+    connection_key: FidesKey,
+    unvalidated_secrets: connection_secrets_schemas,
+    verify: Optional[bool],
+    merge_with_existing: Optional[bool] = False,
+) -> TestStatusMessage:
+    try:
+        return connection_service.update_secrets(
+            connection_key, unvalidated_secrets, verify, merge_with_existing
+        )
+    except ConnectionNotFoundException:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"No connection config found with key {connection_key}",
         )
-        connection_config.update_test_status(
-            test_status=ConnectionTestStatus.failed, db=db
+    except SaaSConfigNotFoundException:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=validate_secrets_error_message(),
         )
-        return TestStatusMessage(
-            msg=msg,
-            test_status=ConnectionTestStatus.failed,
-            failure_reason=str(exc),
+    except ValidationError as e:
+        errors = e.errors(include_url=False, include_input=False)
+        for err in errors:
+            # Additionally, manually remove the context from the error message -
+            # this may contain sensitive information
+            err.pop("ctx", None)
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=jsonable_encoder(errors),
         )
-
-    logger.info("Connection test {} on {}", status.value, connection_config.key)  # type: ignore
-    connection_config.update_test_status(test_status=status, db=db)  # type: ignore
-
-    return TestStatusMessage(
-        msg=msg,
-        test_status=status,
-    )
+    except FidesValidationError as e:
+        # Check if the exception has the original pydantic errors attached
+        raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.message)