ethyca-fides 2.73.2b0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/util/saas_config_updater.py

@@ -6,12 +6,13 @@ from packaging.version import parse as parse_version
 from sqlalchemy.orm import Session
 
 from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.saas_template_dataset import SaasTemplateDataset
 from fides.api.schemas.saas.connector_template import ConnectorTemplate
 from fides.api.schemas.saas.saas_config import SaaSConfig
 from fides.api.service.connectors.saas.connector_registry_service import (
     ConnectorRegistry,
 )
-from fides.api.util.saas_util import load_config_from_string
+from fides.api.util.saas_util import load_config_from_string, load_dataset_from_string
 from fides.service.connection.connection_service import ConnectionService
 from fides.service.event_audit_service import EventAuditService
 
@@ -34,9 +35,18 @@ def update_saas_configs(db: Session) -> None:
         template: ConnectorTemplate = ConnectorRegistry.get_connector_template(  # type: ignore
             connector_type
         )
+
+        # Store the original template dataset (with placeholders) instead of the modified version
+        template_dataset_json = load_dataset_from_string(template.dataset)
+
+        SaasTemplateDataset.get_or_create(
+            db=db,
+            connector_type=connector_type,
+            dataset_json=template_dataset_json,
+        )
+
         saas_config = SaaSConfig(**load_config_from_string(template.config))
         template_version: Version = parse_version(saas_config.version)
-
         connection_configs: Iterable[ConnectionConfig] = ConnectionConfig.filter(
             db=db,
             conditions=(ConnectionConfig.saas_config["type"].astext == connector_type),

fides/api/worker/__init__.py

@@ -14,6 +14,8 @@ from fides.api.tasks import (
     DISCOVERY_MONITORS_PROMOTION_QUEUE_NAME,
     DSR_QUEUE_NAME,
     MESSAGING_QUEUE_NAME,
+    PRIVACY_PREFERENCES_EXPORT_JOB_QUEUE_NAME,
+    PRIVACY_PREFERENCES_INGESTION_JOB_QUEUE_NAME,
     PRIVACY_PREFERENCES_QUEUE_NAME,
     celery_app,
 )
@@ -39,6 +41,8 @@ def start_worker(
         default_queue_name,
         MESSAGING_QUEUE_NAME,
         PRIVACY_PREFERENCES_QUEUE_NAME,
+        PRIVACY_PREFERENCES_EXPORT_JOB_QUEUE_NAME,
+        PRIVACY_PREFERENCES_INGESTION_JOB_QUEUE_NAME,
         DSR_QUEUE_NAME,
         CONSENT_WEBHOOK_QUEUE_NAME,
         DISCOVERY_MONITORS_DETECTION_QUEUE_NAME,

fides/common/api/scope_registry.py

@@ -76,6 +76,7 @@ VALIDATE = "validate"
 VIEW_DATA = "view_data"
 WEBHOOK = "webhook"
 WORKER_STATS = "worker-stats"
+HEAP_DUMP = "heap_dump"
 
 ASSIGN_OWNERS = "assign_owners"
 
@@ -146,6 +147,8 @@ DATASET_TEST = f"{DATASET}:{TEST}"
 
 ENCRYPTION_EXEC = f"{ENCRYPTION}:{EXEC}"
 
+HEAP_DUMP_EXEC = f"{HEAP_DUMP}:{EXEC}"
+
 EVALUATION_CREATE = f"{EVALUATION}:{CREATE}"
 EVALUATION_READ = f"{EVALUATION}:{READ}"
 EVALUATION_UPDATE = f"{EVALUATION}:{UPDATE}"
@@ -209,6 +212,7 @@ SAAS_CONFIG_READ = f"{SAAS_CONFIG}:{READ}"
 
 SAAS_CONNECTION_INSTANTIATE = f"{CONNECTION}:{INSTANTIATE}"
 CONNECTOR_TEMPLATE_REGISTER = f"{CONNECTOR_TEMPLATE}:{REGISTER}"
+CONNECTOR_TEMPLATE_READ = f"{CONNECTOR_TEMPLATE}:{READ}"
 
 SCOPE_READ = f"{SCOPE}:{READ}"
 
@@ -263,6 +267,7 @@ SCOPE_DOCS = {
     CONNECTION_AUTHORIZE: "OAuth2 Authorization",
     CONNECTION_TYPE_READ: "View types of connections",
     CONNECTOR_TEMPLATE_REGISTER: "Register a connector template",
+    CONNECTOR_TEMPLATE_READ: "View connector template configurations",
     CONSENT_READ: "Read consent preferences",
     CONSENT_SETTINGS_READ: "Read org-wide consent settings",
     CONSENT_SETTINGS_UPDATE: "Update org-wide consent settings",
@@ -295,6 +300,7 @@ SCOPE_DOCS = {
     DATASET_READ: "View datasets",
     DATASET_TEST: "Run a standalone privacy request test for a dataset",
     ENCRYPTION_EXEC: "Encrypt data",
+    HEAP_DUMP_EXEC: "Execute a heap dump for memory diagnostics",
     MESSAGING_TEMPLATE_UPDATE: "Update messaging templates",
     EVALUATION_CREATE: "Create evaluation",
     EVALUATION_READ: "Read evaluations",

fides/common/api/v1/urn_registry.py

@@ -87,6 +87,7 @@ PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated"
 PRIVACY_REQUEST_BULK_FINALIZE = "/privacy-request/bulk/finalize"
 PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry"
 PRIVACY_REQUEST_BULK_SOFT_DELETE = "/privacy-request/bulk/soft-delete"
+PRIVACY_REQUEST_CANCEL = "/privacy-request/administrate/cancel"
 PRIVACY_REQUEST_DENY = "/privacy-request/administrate/deny"
 PRIVACY_REQUEST_SOFT_DELETE = "/privacy-request/{privacy_request_id}/soft-delete"
 REQUEST_STATUS_LOGS = "/privacy-request/{privacy_request_id}/log"
@@ -190,13 +191,21 @@ YAML_DATASETS = YAML + CONNECTION_DATASETS
 # SaaS Config URLs
 SAAS_CONFIG_VALIDATE = CONNECTION_BY_KEY + "/validate_saas_config"
 SAAS_CONFIG = CONNECTION_BY_KEY + "/saas_config"
-SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{saas_connector_type}"
+SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{connector_template_type}"
+
+# Connector Template URLs
+CONNECTOR_TEMPLATES = "/connector-templates"
+CONNECTOR_TEMPLATES_REGISTER = "/connector-templates/register"
+CONNECTOR_TEMPLATES_CONFIG = "/connector-templates/{connector_template_type}/config"
+CONNECTOR_TEMPLATES_DATASET = "/connector-templates/{connector_template_type}/dataset"
+
+# Deprecated: Old connector template register URL
 REGISTER_CONNECTOR_TEMPLATE = "/connector_template/register"
 
 # System Connections
 SYSTEM_CONNECTIONS = "/system/{fides_key}/connection"
 INSTANTIATE_SYSTEM_CONNECTION = (
-    "/system/{fides_key}/connection/instantiate/{saas_connector_type}"
+    "/system/{fides_key}/connection/instantiate/{connector_template_type}"
 )
 
 # User URLs

fides/config/__init__.py

@@ -19,7 +19,7 @@ from pydantic_settings import (
 from fides.common.utils import echo_red
 
 from .admin_ui_settings import AdminUISettings
-from .celery_settings import CelerySettings
+from .celery_settings import CelerySettings, merge_celery_environment
 from .cli_settings import CLISettings
 from .consent_settings import ConsentSettings
 from .credentials_settings import merge_credentials_environment
@@ -157,6 +157,12 @@ def build_config(config_dict: Dict[str, Any]) -> FidesConfig:
     config_dict = handle_deprecated_fields(config_dict)
     config_dict = handle_deprecated_env_variables(config_dict)
 
+    # Logic for merging additional Celery environment variables before validation.
+    # This allows arbitrary Celery configuration via FIDES__CELERY__* env vars,
+    # not just the explicitly defined fields in CelerySettings.
+    # Environment variables override TOML file values.
+    config_dict["celery"] = merge_celery_environment(config_dict.get("celery", {}))
+
     settings_map: Dict[str, Any] = {
         "admin_ui": AdminUISettings,
         "celery": CelerySettings,
@@ -208,7 +214,7 @@ def get_config(config_path_override: str = "", verbose: bool = False) -> FidesCo
         settings = toml.load(config_path)
         config = build_config(config_dict=settings)
         if verbose:
-            print(f"> Loaded config from: {config_path}")
+            log.info(f"Loaded config from: {config_path}")
         return config
     except FileNotFoundError:
         pass

fides/config/celery_settings.py

@@ -1,3 +1,8 @@
+import json
+import os
+from json import JSONDecodeError
+from typing import Any, Dict
+
 from pydantic import Field
 from pydantic_settings import SettingsConfigDict
 
@@ -23,3 +28,40 @@ class CelerySettings(FidesSettings):
         "If False, tasks are sent to the queue.",
     )
     model_config = SettingsConfigDict(env_prefix=ENV_PREFIX)
+
+
+def merge_celery_environment(celery_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Given a dict of config settings, merges configs which are found
+    in environment variables. Environment variable configs are treated
+    as overrides. Environment variables must use the prefix FIDES__CELERY__.
+
+    Example:
+        FIDES__CELERY__WORKER_PREFETCH_MULTIPLIER=4
+
+    Merges to:
+        {"worker_prefetch_multiplier": 4}
+
+    This allows arbitrary Celery configuration to be set via environment variables,
+    not just the explicitly defined fields in CelerySettings.
+    """
+    for key, value in os.environ.items():
+        if key.startswith(ENV_PREFIX):
+            # Strip the prefix and convert to lowercase
+            stripped_key = key[len(ENV_PREFIX) :].lower()
+            if stripped_key:
+                # Normalize boolean strings only to preserve case-sensitive values
+                # like URLs (e.g., Redis://myhost) or passwords
+                normalized_value = value
+                if value.lower() in ("true", "false"):
+                    normalized_value = value.lower()
+
+                # Use JSON parsing to handle type conversion properly
+                # This handles booleans (true/false), integers, floats, etc.
+                try:
+                    celery_dict[stripped_key] = json.loads(normalized_value)
+                except (JSONDecodeError, ValueError):
+                    # If JSON parsing fails, keep it as a string
+                    celery_dict[stripped_key] = value
+
+    return celery_dict

fides/service/messaging/messaging_service.py

@@ -2,12 +2,15 @@ import secrets
 from typing import Optional, Set, Union
 
 from loguru import logger
+from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import (
     IdentityNotFoundException,
+    MessageDispatchException,
     PolicyNotFoundException,
 )
+from fides.api.models.messaging import MessagingConfig
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
     ConsentRequest,
@@ -29,15 +32,22 @@ from fides.api.schemas.policy import ActionType
 from fides.api.schemas.redis_cache import Identity
 from fides.api.service.messaging.message_dispatch_service import (
     EMAIL_JOIN_STRING,
-    dispatch_message,
     dispatch_message_task,
+    get_email_messaging_config_service_type,
     message_send_enabled,
 )
 from fides.api.tasks import MESSAGING_QUEUE_NAME
-from fides.config import FidesConfig, get_config
+from fides.config import FidesConfig
 from fides.config.config_proxy import ConfigProxy
 
 
+class MessageDispatchParams(BaseModel):
+    message_meta: Optional[FidesopsMessage]
+    service_type: Optional[str]
+    to_identity: Optional[Identity]
+    property_id: Optional[str]
+
+
 class MessagingService:
     def __init__(self, db: Session, config: FidesConfig, config_proxy: ConfigProxy):
         self.db = db
@@ -67,15 +77,15 @@ class MessagingService:
 
             dispatch_message_task.apply_async(
                 queue=MESSAGING_QUEUE_NAME,
-                kwargs={
-                    "message_meta": FidesopsMessage(
+                kwargs=MessageDispatchParams(
+                    message_meta=FidesopsMessage(
                         action_type=MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE,
                         body_params=None,
-                    ).model_dump(mode="json"),
-                    "service_type": self.config_proxy.notifications.notification_service_type,
-                    "to_identity": to_identity.model_dump(mode="json"),
-                    "property_id": privacy_request.property_id,
-                },
+                    ),
+                    service_type=self.config_proxy.notifications.notification_service_type,
+                    to_identity=to_identity,
+                    property_id=privacy_request.property_id,
+                ).model_dump(mode="json"),
             )
 
     def send_request_denied(
@@ -103,17 +113,17 @@ class MessagingService:
 
             dispatch_message_task.apply_async(
                 queue=MESSAGING_QUEUE_NAME,
-                kwargs={
-                    "message_meta": FidesopsMessage(
+                kwargs=MessageDispatchParams(
+                    message_meta=FidesopsMessage(
                         action_type=MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY,
                         body_params=RequestReviewDenyBodyParams(
                             rejection_reason=deny_reason
                         ),
-                    ).model_dump(mode="json"),
-                    "service_type": self.config_proxy.notifications.notification_service_type,
-                    "to_identity": to_identity.model_dump(mode="json"),
-                    "property_id": privacy_request.property_id,
-                },
+                    ),
+                    service_type=self.config_proxy.notifications.notification_service_type,
+                    to_identity=to_identity,
+                    property_id=privacy_request.property_id,
+                ).model_dump(mode="json"),
             )
 
     def send_verification_code(
@@ -131,19 +141,37 @@ class MessagingService:
             MessagingActionType.SUBJECT_IDENTITY_VERIFICATION,
             self.config_proxy.execution.subject_identity_verification_required,
         ):
+            # Note: Fallback to email messaging config is only used for ConsentRequests
+            service_type = (
+                self.config_proxy.notifications.notification_service_type
+                or get_email_messaging_config_service_type(db=self.db)
+            )
+            if not service_type:
+                raise MessageDispatchException(
+                    "No notification service type configured."
+                )
+
+            # For PrivacyRequest, validate the config exists before dispatching
+            if isinstance(request, PrivacyRequest):
+                MessagingConfig.get_configuration(db=self.db, service_type=service_type)
+
             verification_code = _generate_id_verification_code()
             request.cache_identity_verification_code(verification_code)
 
-            dispatch_message(
-                self.db,
-                action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION,
-                to_identity=to_identity,
-                service_type=self.config_proxy.notifications.notification_service_type,
-                message_body_params=SubjectIdentityVerificationBodyParams(
-                    verification_code=verification_code,
-                    verification_code_ttl_seconds=self.config.redis.identity_verification_code_ttl_seconds,
-                ),
-                property_id=property_id,
+            dispatch_message_task.apply_async(
+                queue=MESSAGING_QUEUE_NAME,
+                kwargs=MessageDispatchParams(
+                    message_meta=FidesopsMessage(
+                        action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION,
+                        body_params=SubjectIdentityVerificationBodyParams(
+                            verification_code=verification_code,
+                            verification_code_ttl_seconds=self.config.redis.identity_verification_code_ttl_seconds,
+                        ),
+                    ),
+                    service_type=service_type,
+                    to_identity=to_identity,
+                    property_id=property_id,
+                ).model_dump(mode="json"),
             )
 
         return verification_code
@@ -185,17 +213,17 @@ class MessagingService:
 
             dispatch_message_task.apply_async(
                 queue=MESSAGING_QUEUE_NAME,
-                kwargs={
-                    "message_meta": FidesopsMessage(
+                kwargs=MessageDispatchParams(
+                    message_meta=FidesopsMessage(
                         action_type=MessagingActionType.PRIVACY_REQUEST_RECEIPT,
                         body_params=RequestReceiptBodyParams(
                             request_types=request_types_list
                         ),
-                    ).model_dump(mode="json"),
-                    "service_type": self.config.notifications.notification_service_type,
-                    "to_identity": identity.labeled_dict(),
-                    "property_id": privacy_request.property_id,
-                },
+                    ),
+                    service_type=self.config.notifications.notification_service_type,
+                    to_identity=identity,
+                    property_id=privacy_request.property_id,
+                ).model_dump(mode="json"),
             )
 
 
@@ -229,17 +257,17 @@ def check_and_dispatch_error_notifications(db: Session) -> None:
         for email in privacy_request_notifications[0].email.split(EMAIL_JOIN_STRING):
             dispatch_message_task.apply_async(
                 queue=MESSAGING_QUEUE_NAME,
-                kwargs={
-                    "message_meta": FidesopsMessage(
+                kwargs=MessageDispatchParams(
+                    message_meta=FidesopsMessage(
                         action_type=MessagingActionType.PRIVACY_REQUEST_ERROR_NOTIFICATION,
                         body_params=ErrorNotificationBodyParams(
                             unsent_errors=len(unsent_errors)
                         ),
-                    ).model_dump(mode="json"),
-                    "service_type": config_proxy.notifications.notification_service_type,
-                    "to_identity": {"email": email},
-                    "property_id": None,
-                },
+                    ),
+                    service_type=config_proxy.notifications.notification_service_type,
+                    to_identity=Identity(email=email),
+                    property_id=None,
+                ).model_dump(mode="json"),
             )
 
         for error in unsent_errors:
@@ -248,33 +276,6 @@ def check_and_dispatch_error_notifications(db: Session) -> None:
     return None
 
 
-def send_verification_code_to_user(
-    db: Session,
-    request: Union[ConsentRequest, PrivacyRequest],
-    to_identity: Optional[Identity],
-    property_id: Optional[str],
-) -> str:
-    """Generate and cache a verification code, and then message the user"""
-    config = get_config()
-    config_proxy = ConfigProxy(db)
-    verification_code = _generate_id_verification_code()
-    request.cache_identity_verification_code(verification_code)
-    messaging_action_type = MessagingActionType.SUBJECT_IDENTITY_VERIFICATION
-    dispatch_message(
-        db,
-        action_type=messaging_action_type,
-        to_identity=to_identity,
-        service_type=config_proxy.notifications.notification_service_type,
-        message_body_params=SubjectIdentityVerificationBodyParams(
-            verification_code=verification_code,
-            verification_code_ttl_seconds=config.redis.identity_verification_code_ttl_seconds,
-        ),
-        property_id=property_id,
-    )
-
-    return verification_code
-
-
 def send_privacy_request_receipt_message_to_user(
     policy: Optional[Policy],
     to_identity: Optional[Identity],
@@ -305,13 +306,13 @@ def send_privacy_request_receipt_message_to_user(
 
     dispatch_message_task.apply_async(
         queue=MESSAGING_QUEUE_NAME,
-        kwargs={
-            "message_meta": FidesopsMessage(
+        kwargs=MessageDispatchParams(
+            message_meta=FidesopsMessage(
                 action_type=MessagingActionType.PRIVACY_REQUEST_RECEIPT,
                 body_params=RequestReceiptBodyParams(request_types=request_types_list),
-            ).model_dump(mode="json"),
-            "service_type": service_type,
-            "to_identity": to_identity.labeled_dict(),
-            "property_id": property_id,
-        },
+            ),
+            service_type=service_type,
+            to_identity=to_identity,
+            property_id=property_id,
+        ).model_dump(mode="json"),
     )