ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/api/util/masking_util.py

@@ -0,0 +1,31 @@
+from typing import Any, Dict
+
+
+def mask_sensitive_fields(
+    connection_secrets: Dict[str, Any], secret_schema: Dict[str, Any]
+) -> Dict[str, Any]:
+    """
+    Mask sensitive fields in the given secrets based on the provided schema.
+    This function traverses the given secrets dictionary and uses the provided schema to
+    identify fields that have been marked as sensitive. The function replaces the sensitive
+    field values with a mask string ('********').
+    Args:
+        connection_secrets (Dict[str, Any]): The secrets to be masked.
+        secret_schema (Dict[str, Any]): The schema defining which fields are sensitive.
+    Returns:
+        Dict[str, Any]: The secrets dictionary with sensitive fields masked.
+    """
+    if connection_secrets is None:
+        return connection_secrets
+
+    secret_schema_properties = secret_schema["properties"]
+    new_connection_secrets = {}
+
+    for key, value in connection_secrets.items():
+        if key in secret_schema_properties:
+            if secret_schema_properties.get(key, {}).get("sensitive", False):
+                new_connection_secrets[key] = "**********"
+            else:
+                new_connection_secrets[key] = value
+
+    return new_connection_secrets

fides/api/util/memory_watchdog.py

@@ -13,6 +13,7 @@ killed by the system's OOM killer which would result in a WorkerLostError and in
 
 from __future__ import annotations
 
+import gc
 import os
 import signal
 import threading
@@ -21,6 +22,7 @@ from functools import wraps
 from types import FrameType, TracebackType
 from typing import Any, Callable, Literal, Optional, Type, TypeVar, Union
 
+import objgraph  # type: ignore
 import psutil  # type: ignore
 from loguru import logger
 
@@ -56,6 +58,118 @@ class MemoryLimitExceeded(RuntimeError):
         self.memory_percent: Optional[float] = memory_percent
 
 
+def _capture_heap_dump() -> None:
+    """Capture and log actionable heap dump information for memory leak diagnosis.
+
+    This function builds and logs a comprehensive memory profiling report as a single
+    message to help identify memory leaks. It's designed to be called when memory
+    threshold is exceeded, just before task termination.
+
+    The report includes:
+    - Process memory stats (RSS, VMS)
+    - Top 10 object type counts (spot anomalies like 100k of a custom class)
+    - Garbage collector stats (uncollectable objects indicate leaks)
+
+    All exceptions are caught to ensure heap dump failures don't prevent graceful
+    task termination.
+    """
+    try:
+        # Force garbage collection to get accurate counts
+        gc.collect()
+
+        report_lines = [
+            "",  # Leading newline for visual separation
+            "=" * 80,
+            "MEMORY DUMP - THRESHOLD EXCEEDED",
+            "=" * 80,
+            "",
+        ]
+
+        # Section 1: Process Memory Stats
+        try:
+            process = psutil.Process()
+            mem_info = process.memory_info()
+            report_lines.extend(
+                [
+                    "PROCESS MEMORY STATS",
+                    "-" * 80,
+                    f"RSS (Resident Set Size):  {mem_info.rss / 1024 / 1024:.1f} MiB",
+                    f"VMS (Virtual Memory):     {mem_info.vms / 1024 / 1024:.1f} MiB",
+                    "",
+                ]
+            )
+        except Exception as exc:
+            logger.debug("Failed to get process memory stats: {}", exc)
+
+        # Section 2: Object Type Counts (most actionable for finding leaks)
+        type_stats = objgraph.most_common_types(limit=10)
+
+        report_lines.extend(
+            [
+                "OBJECT TYPE COUNTS (Top 10)",
+                "-" * 80,
+            ]
+        )
+
+        if type_stats:
+            max_typename_len = max(len(typename) for typename, _ in type_stats)
+
+            for typename, count in type_stats:
+                # Right-align count for easy scanning
+                report_lines.append(f"{typename:<{max_typename_len}}  {count:>10,}")
+        else:
+            report_lines.append("No object type data available")
+
+        report_lines.append("")
+
+        # Section 3: Garbage Collector Stats
+        gc_count = gc.get_count()
+
+        report_lines.extend(
+            [
+                "GARBAGE COLLECTOR STATS",
+                "-" * 80,
+                f"Collections: gen0={gc_count[0]}, gen1={gc_count[1]}, gen2={gc_count[2]}",
+                f"Tracked objects: {len(gc.get_objects()):,}",
+            ]
+        )
+
+        # Show uncollectable objects (strong indicator of memory leaks)
+        uncollectable = gc.garbage
+        if uncollectable:
+            report_lines.append(
+                f"⚠ UNCOLLECTABLE OBJECTS: {len(uncollectable)} (LEAK DETECTED!)"
+            )
+            # Show types of uncollectable objects
+            uncollectable_types: dict[str, int] = {}
+            for obj in uncollectable[:50]:  # Limit to first 50
+                obj_type = type(obj).__name__
+                uncollectable_types[obj_type] = uncollectable_types.get(obj_type, 0) + 1
+            for obj_type, count in sorted(
+                uncollectable_types.items(), key=lambda x: x[1], reverse=True
+            ):
+                report_lines.append(f"  {obj_type}: {count}")
+        else:
+            report_lines.append("No uncollectable objects detected")
+
+        report_lines.extend(
+            [
+                "",
+                "=" * 80,
+                "END OF MEMORY DUMP",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        # Log the entire report as a single message
+        heap_dump_report = "\n".join(report_lines)
+        logger.error(heap_dump_report)
+
+    except Exception as exc:  # pragma: no cover
+        logger.exception("Failed to capture heap dump: {}", exc)
+
+
 class MemoryWatchdog:
     """Background memory monitor that enables graceful task termination.
 
@@ -138,6 +252,10 @@ class MemoryWatchdog:
         """Convert the signal into an exception on the main thread."""
         current_percent = _system_memory_percent()
         logger.error("Memory limit exceeded: {}%", current_percent)
+
+        # Capture heap dump before terminating
+        _capture_heap_dump()
+
         self.stop()
         raise MemoryLimitExceeded(
             "Memory usage exceeded threshold", memory_percent=current_percent

fides/api/util/saas_config_updater.py

@@ -0,0 +1,66 @@
+from typing import Iterable
+
+from loguru import logger
+from packaging.version import Version
+from packaging.version import parse as parse_version
+from sqlalchemy.orm import Session
+
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.schemas.saas.connector_template import ConnectorTemplate
+from fides.api.schemas.saas.saas_config import SaaSConfig
+from fides.api.service.connectors.saas.connector_registry_service import (
+    ConnectorRegistry,
+)
+from fides.api.util.saas_util import load_config_from_string
+from fides.service.connection.connection_service import ConnectionService
+from fides.service.event_audit_service import EventAuditService
+
+
+def update_saas_configs(db: Session) -> None:
+    """
+    Updates SaaS config instances currently in the DB if to the
+    corresponding template in the registry are found.
+
+    Effectively an "update script" for SaaS config instances,
+    to be run on server bootstrap.
+    """
+    event_audit_service = EventAuditService(db)
+    saas_connection_service = ConnectionService(db, event_audit_service)
+    for connector_type in ConnectorRegistry.connector_types():
+        logger.debug(
+            "Determining if any updates are needed for connectors of type {} based on templates...",
+            connector_type,
+        )
+        template: ConnectorTemplate = ConnectorRegistry.get_connector_template(  # type: ignore
+            connector_type
+        )
+        saas_config = SaaSConfig(**load_config_from_string(template.config))
+        template_version: Version = parse_version(saas_config.version)
+
+        connection_configs: Iterable[ConnectionConfig] = ConnectionConfig.filter(
+            db=db,
+            conditions=(ConnectionConfig.saas_config["type"].astext == connector_type),
+        ).all()
+        for connection_config in connection_configs:
+            saas_config_instance = SaaSConfig.model_validate(
+                connection_config.saas_config
+            )
+            if parse_version(saas_config_instance.version) < template_version:
+                logger.info(
+                    "Updating SaaS config instance '{}' of type '{}' as its version, {}, was found to be lower than the template version {}",
+                    saas_config_instance.fides_key,
+                    connector_type,
+                    saas_config_instance.version,
+                    template_version,
+                )
+                try:
+                    saas_connection_service.update_saas_instance(
+                        connection_config,
+                        template,
+                        saas_config_instance,
+                    )
+                except Exception:
+                    logger.exception(
+                        "Encountered error attempting to update SaaS config instance {}",
+                        saas_config_instance.fides_key,
+                    )

fides/api/util/saas_util.py

@@ -16,7 +16,7 @@ from fides.api.common_exceptions import FidesopsException, ValidationError
 from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
 from fides.api.graph.config import Collection, CollectionAddress, Field, GraphDataset
 from fides.api.models.privacy_request import PrivacyRequest
-from fides.api.schemas.saas.saas_config import SaaSConfig, SaaSRequest
+from fides.api.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest
 from fides.api.schemas.saas.shared_schemas import SaaSRequestParams
 from fides.config import CONFIG
 from fides.config.helpers import load_file
@@ -26,6 +26,7 @@ PRIVACY_REQUEST_ID = "privacy_request_id"
 MASKED_OBJECT_FIELDS = "masked_object_fields"
 ALL_OBJECT_FIELDS = "all_object_fields"
 CUSTOM_PRIVACY_REQUEST_FIELDS = "custom_privacy_request_fields"
+PRIVACY_REQUEST_OBJECT = "privacy_request"
 UUID = "uuid"
 ISO_8601_DATETIME = "iso_8601_datetime"
 FIELD_LIST = "field_list"
@@ -335,6 +336,32 @@ def assign_placeholders(value: Any, param_values: Dict[str, Any]) -> Optional[An
     return value
 
 
+def check_dataset_missing_reference_values(
+    input_data: Dict[str, Any], param_values: Optional[List[ParamValue]]
+) -> List[str]:
+    """
+    Check if all the dataset reference values are present in the input_data map
+    """
+    # get the list of param_value references
+    required_param_value_references = [
+        param_value.name for param_value in param_values or [] if param_value.references
+    ]
+
+    # extract the keys from inside the fides_grouped_inputs and append them the other input_data keys
+    provided_input_keys = (
+        list(input_data.get("fidesops_grouped_inputs")[0].keys())  # type: ignore
+        if input_data.get("fidesops_grouped_inputs")
+        else []
+    ) + list(input_data.keys())
+
+    # find the missing values
+    missing_dataset_reference_values = list(
+        set(required_param_value_references) - set(provided_input_keys)
+    )
+
+    return missing_dataset_reference_values
+
+
 def map_param_values(
     action: str,
     context: str,

fides/common/api/scope_registry.py

@@ -226,10 +226,6 @@ SYSTEM_MANAGER_READ = f"{SYSTEM_MANAGER}:{READ}"
 SYSTEM_MANAGER_UPDATE = f"{SYSTEM_MANAGER}:{UPDATE}"
 SYSTEM_MANAGER_DELETE = f"{SYSTEM_MANAGER}:{DELETE}"
 
-TAXONOMY_CREATE = f"{TAXONOMY}:{CREATE}"
-TAXONOMY_UPDATE = f"{TAXONOMY}:{UPDATE}"
-TAXONOMY_DELETE = f"{TAXONOMY}:{DELETE}"
-
 USER_CREATE = f"{USER}:{CREATE}"
 USER_DELETE = f"{USER}:{DELETE}"
 USER_READ = f"{USER}:{READ}"
@@ -357,9 +353,6 @@ SCOPE_DOCS = {
     SYSTEM_MANAGER_READ: "Read systems users can manage",
     SYSTEM_MANAGER_DELETE: "Delete systems user can manage",
     SYSTEM_MANAGER_UPDATE: "Update systems user can manage",
-    TAXONOMY_CREATE: "Create local taxonomy",
-    TAXONOMY_DELETE: "Delete local taxonomy",
-    TAXONOMY_UPDATE: "Update local taxonomy",
     USER_CREATE: "Create users",
     USER_UPDATE: "Update users",
     USER_DELETE: "Remove users",

fides/common/api/v1/urn_registry.py

@@ -84,8 +84,10 @@ PRIVACY_REQUEST_BATCH_EMAIL_SEND = (
     "/privacy-request/administrate/process-awaiting-email-send"
 )
 PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated"
+PRIVACY_REQUEST_BULK_FINALIZE = "/privacy-request/bulk/finalize"
 PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry"
 PRIVACY_REQUEST_BULK_SOFT_DELETE = "/privacy-request/bulk/soft-delete"
+PRIVACY_REQUEST_CANCEL = "/privacy-request/administrate/cancel"
 PRIVACY_REQUEST_DENY = "/privacy-request/administrate/deny"
 PRIVACY_REQUEST_SOFT_DELETE = "/privacy-request/{privacy_request_id}/soft-delete"
 REQUEST_STATUS_LOGS = "/privacy-request/{privacy_request_id}/log"

fides/config/__init__.py

@@ -19,11 +19,12 @@ from pydantic_settings import (
 from fides.common.utils import echo_red
 
 from .admin_ui_settings import AdminUISettings
-from .celery_settings import CelerySettings
+from .celery_settings import CelerySettings, merge_celery_environment
 from .cli_settings import CLISettings
 from .consent_settings import ConsentSettings
 from .credentials_settings import merge_credentials_environment
 from .database_settings import DatabaseSettings
+from .duplicate_detection_settings import DuplicateDetectionSettings
 from .execution_settings import ExecutionSettings
 from .fides_settings import FidesSettings
 from .helpers import handle_deprecated_env_variables, handle_deprecated_fields
@@ -79,6 +80,7 @@ class FidesConfig(FidesSettings):
         description="This is a special section that is used to store arbitrary key/value pairs to be used as credentials."
     )
     database: DatabaseSettings
+    privacy_request_duplicate_detection: DuplicateDetectionSettings
     execution: ExecutionSettings
     logging: LoggingSettings
     notifications: NotificationSettings
@@ -155,12 +157,19 @@ def build_config(config_dict: Dict[str, Any]) -> FidesConfig:
     config_dict = handle_deprecated_fields(config_dict)
     config_dict = handle_deprecated_env_variables(config_dict)
 
+    # Logic for merging additional Celery environment variables before validation.
+    # This allows arbitrary Celery configuration via FIDES__CELERY__* env vars,
+    # not just the explicitly defined fields in CelerySettings.
+    # Environment variables override TOML file values.
+    config_dict["celery"] = merge_celery_environment(config_dict.get("celery", {}))
+
     settings_map: Dict[str, Any] = {
         "admin_ui": AdminUISettings,
         "celery": CelerySettings,
         "consent": ConsentSettings,
         "cli": CLISettings,
         "database": DatabaseSettings,
+        "privacy_request_duplicate_detection": DuplicateDetectionSettings,
         "execution": ExecutionSettings,
         "logging": LoggingSettings,
         "notifications": NotificationSettings,

fides/config/celery_settings.py

@@ -1,3 +1,8 @@
+import json
+import os
+from json import JSONDecodeError
+from typing import Any, Dict
+
 from pydantic import Field
 from pydantic_settings import SettingsConfigDict
 
@@ -23,3 +28,40 @@ class CelerySettings(FidesSettings):
         "If False, tasks are sent to the queue.",
     )
     model_config = SettingsConfigDict(env_prefix=ENV_PREFIX)
+
+
+def merge_celery_environment(celery_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Given a dict of config settings, merges configs which are found
+    in environment variables. Environment variable configs are treated
+    as overrides. Environment variables must use the prefix FIDES__CELERY__.
+
+    Example:
+        FIDES__CELERY__WORKER_PREFETCH_MULTIPLIER=4
+
+    Merges to:
+        {"worker_prefetch_multiplier": 4}
+
+    This allows arbitrary Celery configuration to be set via environment variables,
+    not just the explicitly defined fields in CelerySettings.
+    """
+    for key, value in os.environ.items():
+        if key.startswith(ENV_PREFIX):
+            # Strip the prefix and convert to lowercase
+            stripped_key = key[len(ENV_PREFIX) :].lower()
+            if stripped_key:
+                # Normalize boolean strings only to preserve case-sensitive values
+                # like URLs (e.g., Redis://myhost) or passwords
+                normalized_value = value
+                if value.lower() in ("true", "false"):
+                    normalized_value = value.lower()
+
+                # Use JSON parsing to handle type conversion properly
+                # This handles booleans (true/false), integers, floats, etc.
+                try:
+                    celery_dict[stripped_key] = json.loads(normalized_value)
+                except (JSONDecodeError, ValueError):
+                    # If JSON parsing fails, keep it as a string
+                    celery_dict[stripped_key] = value
+
+    return celery_dict

fides/config/config_proxy.py

@@ -155,6 +155,15 @@ class ConsentSettingsProxy(ConfigProxyBase):
     override_vendor_purposes: bool
 
 
+class DuplicateDetectionSettingsProxy(ConfigProxyBase):
+    prefix = "privacy_request_duplicate_detection"
+
+    enabled: bool
+    time_window_days: int
+    match_identity_fields: List[str]
+
+
+# pylint: disable=too-many-instance-attributes
 class ConfigProxy:
     """
     ConfigProxy instances allow access to "resolved" config properties
@@ -183,6 +192,7 @@ class ConfigProxy:
         self.storage = StorageSettingsProxy(db)
         self.security = SecuritySettingsProxy(db)
         self.consent = ConsentSettingsProxy(db)
+        self.privacy_request_duplicate_detection = DuplicateDetectionSettingsProxy(db)
         self.privacy_center = PrivacyCenterSettingsProxy(db)
 
     def load_current_cors_domains_into_middleware(self, app: FastAPI) -> None:

fides/config/duplicate_detection_settings.py

@@ -0,0 +1,31 @@
+from typing import List
+
+from pydantic import Field
+from pydantic_settings import SettingsConfigDict
+
+from .fides_settings import FidesSettings
+
+
+class DuplicateDetectionSettings(FidesSettings):
+    """Configuration settings for Duplicate Privacy Request Detection."""
+
+    enabled: bool = Field(
+        default=False,
+        description="Whether duplicate detection is enabled. Disabled by default.",
+    )
+
+    time_window_days: int = Field(
+        default=365,
+        description="Time window in days for duplicate detection. Default is 1 year.",
+        ge=1,
+        le=3650,  # Max 10 years
+    )
+
+    match_identity_fields: List[str] = Field(
+        default=["email"],
+        description="Identity field names to match on for duplicate detection (e.g., 'email', 'phone_number'). Default is email only.",
+    )
+
+    model_config = SettingsConfigDict(
+        env_prefix="FIDES__PRIVACY_REQUEST_DUPLICATE_DETECTION__"
+    )

fides/config/execution_settings.py

@@ -61,9 +61,13 @@ class ExecutionSettings(FidesSettings):
         default=3,
         description="The number of times a privacy request will be requeued when its tasks are interrupted before being marked as error",
     )
-    async_tasks_status_polling_interval_seconds: int = Field(
-        default=3600,
-        description="Seconds between polling for async tasks to requeue",
+    async_polling_interval_hours: int = Field(
+        default=1,
+        description="Hours between status checks for async tasks",
+    )
+    async_polling_request_timeout_days: int = Field(
+        default=30,
+        description="Maximum time in days to wait for an async polling request to complete before timing out",
     )
     erasure_request_finalization_required: bool = Field(
         default=False,

fides/config/utils.py

@@ -70,4 +70,9 @@ CONFIG_KEY_ALLOWLIST = {
     "consent": ["override_vendor_purposes"],
     "admin_ui": ["enabled", "url", "error_notification_mode"],
     "privacy_center": ["url"],
+    "privacy_request_duplicate_detection": [
+        "enabled",
+        "time_window_days",
+        "match_identity_fields",
+    ],
 }

fides/data/language/languages.yml

@@ -9,6 +9,8 @@ languages:
   name: Catalan
 - id: cs
   name: Czech
+- id: cy
+  name: Welsh
 - id: da
   name: Danish
 - id: de