ethyca-fides 2.71.1rc2__py2.py3-none-any.whl → 2.72.0__py2.py3-none-any.whl

fides/api/service/connectors/saas/connector_registry_service.py

@@ -1,24 +1,18 @@
 # pylint: disable=protected-access
 import os
 from abc import ABC, abstractmethod
-from typing import Dict, Iterable, List, Optional, Type
+from typing import Dict, List, Optional, Type
 from zipfile import ZipFile
 
 from fideslang.models import Dataset
 from loguru import logger
-from packaging.version import Version
 from packaging.version import parse as parse_version
 from sqlalchemy.orm import Session
 
 from fides.api.api.deps import get_api_session
 from fides.api.common_exceptions import ValidationError
 from fides.api.cryptography.cryptographic_util import str_to_b64_str
-from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.custom_connector_template import CustomConnectorTemplate
-from fides.api.models.datasetconfig import DatasetConfig
-from fides.api.schemas.connection_configuration.saas_config_template_values import (
-    SaasConnectionTemplateValues,
-)
 from fides.api.schemas.saas.connector_template import ConnectorTemplate
 from fides.api.schemas.saas.saas_config import SaaSConfig
 from fides.api.service.authentication.authentication_strategy_oauth2_authorization_code import (
@@ -31,8 +25,6 @@ from fides.api.util.saas_util import (
     load_config_from_string,
     load_dataset_from_string,
     load_yaml_as_string,
-    replace_config_placeholders,
-    replace_dataset_placeholders,
     replace_version,
 )
 from fides.api.util.unsafe_file_util import verify_svg, verify_zip
@@ -317,132 +309,3 @@ class ConnectorRegistry:
         Returns an object containing the various SaaS connector artifacts
         """
         return cls._get_combined_templates().get(connector_type)
-
-
-def create_connection_config_from_template_no_save(
-    db: Session,
-    template: ConnectorTemplate,
-    template_values: SaasConnectionTemplateValues,
-    system_id: Optional[str] = None,
-) -> ConnectionConfig:
-    """Creates a SaaS connection config from a template without saving it."""
-    # Load SaaS config from template and replace every instance of "<instance_fides_key>" with the fides_key
-    # the user has chosen
-    config_from_template: Dict = replace_config_placeholders(
-        template.config, "<instance_fides_key>", template_values.instance_key
-    )
-
-    data = template_values.generate_config_data_from_template(
-        config_from_template=config_from_template
-    )
-
-    if system_id:
-        data["system_id"] = system_id
-
-    # Create SaaS ConnectionConfig
-    connection_config = ConnectionConfig.create_without_saving(db, data=data)
-
-    return connection_config
-
-
-def upsert_dataset_config_from_template(
-    db: Session,
-    connection_config: ConnectionConfig,
-    template: ConnectorTemplate,
-    template_values: SaasConnectionTemplateValues,
-) -> DatasetConfig:
-    """
-    Creates a `DatasetConfig` from a template
-    and associates it with a ConnectionConfig.
-    If the `DatasetConfig` already exists in the db,
-    then the existing record is updated.
-    """
-    # Load the dataset config from template and replace every instance of "<instance_fides_key>" with the fides_key
-    # the user has chosen
-    dataset_from_template: Dict = replace_dataset_placeholders(
-        template.dataset, "<instance_fides_key>", template_values.instance_key
-    )
-    data = {
-        "connection_config_id": connection_config.id,
-        "fides_key": template_values.instance_key,
-        "dataset": dataset_from_template,  # Currently used for upserting a CTL Dataset
-    }
-    dataset_config = DatasetConfig.create_or_update(db, data=data)
-    return dataset_config
-
-
-def update_saas_configs(db: Session) -> None:
-    """
-    Updates SaaS config instances currently in the DB if to the
-    corresponding template in the registry are found.
-
-    Effectively an "update script" for SaaS config instances,
-    to be run on server bootstrap.
-    """
-    for connector_type in ConnectorRegistry.connector_types():
-        logger.debug(
-            "Determining if any updates are needed for connectors of type {} based on templates...",
-            connector_type,
-        )
-        template: ConnectorTemplate = ConnectorRegistry.get_connector_template(  # type: ignore
-            connector_type
-        )
-        saas_config = SaaSConfig(**load_config_from_string(template.config))
-        template_version: Version = parse_version(saas_config.version)
-
-        connection_configs: Iterable[ConnectionConfig] = ConnectionConfig.filter(
-            db=db,
-            conditions=(ConnectionConfig.saas_config["type"].astext == connector_type),
-        ).all()
-        for connection_config in connection_configs:
-            saas_config_instance = SaaSConfig.model_validate(
-                connection_config.saas_config
-            )
-            if parse_version(saas_config_instance.version) < template_version:
-                logger.info(
-                    "Updating SaaS config instance '{}' of type '{}' as its version, {}, was found to be lower than the template version {}",
-                    saas_config_instance.fides_key,
-                    connector_type,
-                    saas_config_instance.version,
-                    template_version,
-                )
-                try:
-                    update_saas_instance(
-                        db,
-                        connection_config,
-                        template,
-                        saas_config_instance,
-                    )
-                except Exception:
-                    logger.exception(
-                        "Encountered error attempting to update SaaS config instance {}",
-                        saas_config_instance.fides_key,
-                    )
-
-
-def update_saas_instance(
-    db: Session,
-    connection_config: ConnectionConfig,
-    template: ConnectorTemplate,
-    saas_config_instance: SaaSConfig,
-) -> None:
-    """
-    Replace in the DB the existing SaaS instance configuration data
-    (SaaSConfig, DatasetConfig) associated with the given ConnectionConfig
-    with new instance configuration data based on the given ConnectorTemplate
-    """
-    template_vals = SaasConnectionTemplateValues(
-        name=connection_config.name,
-        key=connection_config.key,
-        description=connection_config.description,
-        secrets=connection_config.secrets,
-        instance_key=saas_config_instance.fides_key,
-    )
-
-    config_from_template: Dict = replace_config_placeholders(
-        template.config, "<instance_fides_key>", template_vals.instance_key
-    )
-
-    connection_config.update_saas_config(db, SaaSConfig(**config_from_template))
-
-    upsert_dataset_config_from_template(db, connection_config, template, template_vals)

fides/api/service/connectors/saas_connector.py

@@ -67,6 +67,7 @@ from fides.api.util.logger_context_utils import (
 from fides.api.util.saas_util import (
     ALL_OBJECT_FIELDS,
     CUSTOM_PRIVACY_REQUEST_FIELDS,
+    PRIVACY_REQUEST_OBJECT,
     assign_placeholders,
     map_param_values,
 )
@@ -276,6 +277,8 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         if custom_privacy_request_fields:
             input_data[CUSTOM_PRIVACY_REQUEST_FIELDS] = [custom_privacy_request_fields]
 
+        input_data[PRIVACY_REQUEST_OBJECT] = [privacy_request.to_safe_dict()]
+
         rows: List[Row] = []
         for read_request in read_requests:
             self.set_saas_request_state(read_request)
@@ -300,7 +303,12 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             # if a path is provided, it means we want to generate HTTP requests from the config
             if read_request.path:
                 prepared_requests: List[Tuple[SaaSRequestParams, Dict[str, Any]]] = (
-                    query_config.generate_requests(input_data, policy, read_request)
+                    query_config.generate_requests(
+                        input_data,
+                        policy,
+                        read_request,
+                        privacy_request=privacy_request,
+                    )
                 )
 
                 # Iterates through initial list of prepared requests and through subsequent

fides/api/service/deps.py

@@ -6,6 +6,7 @@ from fides.api.api.deps import get_config, get_config_proxy, get_db
 from fides.api.db.ctl_session import get_async_db
 from fides.config import FidesConfig
 from fides.config.config_proxy import ConfigProxy
+from fides.service.connection.connection_service import ConnectionService
 from fides.service.dataset.dataset_config_service import DatasetConfigService
 from fides.service.dataset.dataset_service import DatasetService
 from fides.service.event_audit_service import EventAuditService
@@ -59,5 +60,12 @@ def get_taxonomy_service(
     return TaxonomyService(db, event_audit_service)
 
 
+def get_connection_service(
+    db: Session = Depends(get_db),
+    event_audit_service: EventAuditService = Depends(get_event_audit_service),
+) -> ConnectionService:
+    return ConnectionService(db, event_audit_service)
+
+
 def get_system_service(db: AsyncSession = Depends(get_async_db)) -> SystemService:
     return SystemService(db)

fides/api/service/messaging/message_dispatch_service.py

@@ -28,6 +28,7 @@ from fides.api.schemas.messaging.messaging import (
     ErrorNotificationBodyParams,
     ExternalUserWelcomeBodyParams,
     FidesopsMessage,
+    ManualTaskDigestBodyParams,
     MessagingActionType,
     MessagingMethod,
     MessagingServiceDetails,
@@ -177,6 +178,7 @@ def dispatch_message(
             UserInviteBodyParams,
             ErrorNotificationBodyParams,
             ExternalUserWelcomeBodyParams,
+            ManualTaskDigestBodyParams,
         ]
     ] = None,
     subject_override: Optional[str] = None,
@@ -352,7 +354,7 @@ def _render(template_str: str, variables: Optional[Dict] = None) -> str:
     return template_str
 
 
-def _build_email(  # pylint: disable=too-many-return-statements, too-many-branches
+def _build_email(  # pylint: disable=too-many-return-statements, too-many-branches, too-many-statements
     config_proxy: ConfigProxy,
     action_type: MessagingActionType,
     body_params: Any,
@@ -494,6 +496,41 @@ def _build_email(  # pylint: disable=too-many-return-statements, too-many-branch
             template_variables=variables,
         )
 
+    if action_type == MessagingActionType.MANUAL_TASK_DIGEST:
+        variables = {
+            "vendor_contact_name": body_params.vendor_contact_name,
+            "organization_name": body_params.organization_name,
+            "portal_url": body_params.portal_url,
+            "imminent_task_count": body_params.imminent_task_count,
+            "upcoming_task_count": body_params.upcoming_task_count,
+            "total_task_count": body_params.total_task_count,
+            "company_logo_url": body_params.company_logo_url,
+        }
+
+        # Start with default subject
+        subject = f"Weekly DSR Summary from {body_params.organization_name}"
+
+        # If messaging template exists, extract customizable content
+        if messaging_template:
+            # Use custom subject if provided
+            if "subject" in messaging_template.content:
+                subject = _render(messaging_template.content["subject"], variables)
+
+            # Use custom body content to replace the intro text in HTML template
+            custom_body = messaging_template.content.get("body", "")
+            if custom_body.strip():
+                # Replace the default intro text with the custom body content
+                rendered_custom_body = _render(custom_body, variables)
+                variables["intro_text"] = rendered_custom_body
+
+        # Always use the HTML template
+        base_template = get_email_template(action_type)
+        return EmailForActionType(
+            subject=subject,
+            body=base_template.render(variables),
+            template_variables=variables,
+        )
+
     logger.error("Message action type {} is not implemented", action_type)
     raise MessageDispatchException(
         f"Message action type {action_type} is not implemented"

fides/api/service/privacy_request/request_runner_service.py

@@ -4,7 +4,11 @@ from copy import deepcopy
 from datetime import datetime, timedelta
 from typing import Any, Optional
 
+import sqlalchemy.exc
 from loguru import logger
+
+# pylint: disable=no-name-in-module
+from psycopg2.errors import InternalError_  # type: ignore[import-untyped]
 from pydantic import ValidationError as PydanticValidationError
 from sqlalchemy.orm import Query, Session
 
@@ -270,6 +274,11 @@ def upload_access_results(
     return download_urls
 
 
+@log_context(
+    capture_args={
+        "privacy_request_id": LoggerContextKeys.privacy_request_id,
+    }
+)
 def save_access_results(
     session: Session,
     privacy_request: PrivacyRequest,
@@ -278,12 +287,34 @@ def save_access_results(
 ) -> None:
     """Save the results we uploaded to the user for later retrieval"""
     # Save the results we uploaded to the user for later retrieval
-    privacy_request.save_filtered_access_results(session, rule_filtered_results)
     # Saving access request URL's on the privacy request in case DSR 3.0
     # exits processing before the email is sent
     privacy_request.access_result_urls = {"access_result_urls": download_urls}
     privacy_request.save(session)
 
+    # Try to save the backup results, but don't fail the DSR if this fails
+    try:
+        privacy_request.save_filtered_access_results(session, rule_filtered_results)
+        logger.info("Successfully saved backup filtered access results to database")
+    except (
+        InternalError_,  # invalid memory alloc request size 1073741824
+        sqlalchemy.exc.StatementError,  # SQL statement errors
+        # Python memory errors
+        MemoryError,  # system out of memory
+        OverflowError,  # numeric overflow during serialization
+    ) as exc:
+        logger.warning(
+            "Failed to save backup of DSR results to database after successful S3 upload. "
+            "DSR will continue processing. Error: {}",
+            str(exc),
+        )
+    except Exception as exc:
+        logger.error(
+            "Failed to save backup of DSR results to database after successful S3 upload. "
+            "DSR will continue processing. Unexpected Error: {}",
+            str(exc),
+        )
+
 
 @log_context(
     capture_args={

fides/api/tasks/__init__.py

@@ -86,7 +86,6 @@ class DatabaseTask(Task):  # pylint: disable=W0223
         # to prevent session overlap when requests are executing concurrently
         # when in task_always_eager mode (i.e. without proper workers)
         new_session = self._sessionmaker()
-        logger.debug(f"DatabaseTaskSession ID: {id(new_session)}. Self ID: {id(self)}")
         return new_session