ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/util/lock.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from contextlib import contextmanager
-from typing import Generator
+from typing import Generator, Optional
 
 from loguru import logger
 from redis.lock import Lock
@@ -18,7 +18,12 @@ def get_redis_lock(lock_key: str, timeout: int) -> Lock:
 
 
 @contextmanager
-def redis_lock(lock_key: str, timeout: int) -> Generator[Lock | None, None, None]:
+def redis_lock(
+    lock_key: str,
+    timeout: int,
+    blocking: bool = False,
+    blocking_timeout: Optional[int] = None,
+) -> Generator[Lock | None, None, None]:
     """
     A context manager for acquiring a Redis lock.
     If the lock is acquired, it yields the lock object.
@@ -30,7 +35,12 @@ def redis_lock(lock_key: str, timeout: int) -> Generator[Lock | None, None, None
     """
     lock = get_redis_lock(lock_key, timeout)
 
-    if not lock.acquire(blocking=False):
+    # If we're blocking but no blocking timeout is provided
+    # fall back to the lock timeout as the blocking timeout
+    if blocking and blocking_timeout is None:
+        blocking_timeout = timeout
+
+    if not lock.acquire(blocking=blocking, blocking_timeout=blocking_timeout):
         logger.info(
             f"Another process is already running for lock '{lock_key}'. Skipping this execution."
         )

fides/api/util/logger.py

@@ -7,8 +7,10 @@ Defines the logging format and other helper functions to be used throughout the
 from __future__ import annotations
 
 import logging
+import os
 import sys
 from contextlib import contextmanager
+from types import FrameType
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional
 
 from loguru import logger
@@ -24,6 +26,37 @@ if TYPE_CHECKING:
 MASKED = "MASKED"
 
 
+class InterceptHandler(logging.Handler):
+    """
+    Intercept standard library logging and redirect to Loguru.
+
+    This handler is added to the root logger to capture logs from libraries
+    that use standard library logging (SQLAlchemy, Alembic, Celery, etc.)
+    and route them through Loguru for consistent formatting and serialization.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Emit a log record by routing it through Loguru."""
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = str(record.levelno)
+
+        frame: Optional[FrameType] = logging.currentframe()
+        depth = 2
+        # Get the directory path of the logging module to match all files within it
+        logging_module_path = os.path.dirname(logging.__file__) + os.sep
+        while frame is not None and frame.f_code.co_filename.startswith(
+            logging_module_path
+        ):
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(
+            level, record.getMessage()
+        )
+
+
 class RedisSink:
     """A sink that writes log messages to Redis."""
 
@@ -134,18 +167,30 @@ def create_handler_dicts(
         "catch": True,
     }
 
+    # Helper function to check if a log record has custom extra context
+    # Loguru always includes an 'extra' dict, so we need to check if it has any keys
+    def has_custom_extra(log_record: Dict) -> bool:
+        """Check if log record has custom extra context beyond Loguru's defaults."""
+        extra = log_record.get("extra", {})
+        return len(extra) > 0
+
+    # Helper to filter logs without custom extra
+    def filter_standard(log_record: Dict) -> bool:
+        """Filter for logs without custom extra context."""
+        return not has_custom_extra(log_record)
+
     # Configure handler
     standard_dict = {
         **base_config,
         "sink": sink,
-        "filter": lambda logRecord: not bool(logRecord["extra"]),
+        "filter": filter_standard,
     }
 
     # Create extra dict with additional formatting for logs with extra context
     extra_dict = {
         **standard_dict,
         "format": log_format + " | <dim>{extra}</dim>",
-        "filter": lambda logRecord: bool(logRecord["extra"]),
+        "filter": has_custom_extra,
     }
 
     return [standard_dict, extra_dict]
@@ -171,13 +216,14 @@ def setup(config: FidesConfig) -> None:
     # Configure main sink from config
     destination = config.logging.destination
     main_sink = sys.stdout if not destination else destination
+    is_json_serialization = config.logging.serialization == "json"
     handlers.extend(
         create_handler_dicts(
             level=config.logging.level,
             include_called_from=config.dev_mode,
             sink=main_sink,
-            serialize=config.logging.serialization == "json",
-            colorize=config.logging.colorize,
+            serialize=is_json_serialization,
+            colorize=config.logging.colorize and not is_json_serialization,
         )
     )
 
@@ -196,6 +242,14 @@ def setup(config: FidesConfig) -> None:
 
     logger.configure(handlers=handlers)
 
+    # Add InterceptHandler to root logger to capture standard library logs
+    # This intercepts logs from SQLAlchemy, Alembic, Celery, etc.
+    logging.root.handlers = [InterceptHandler()]
+    logging.root.setLevel(config.logging.level)
+
+    # Capture Python warnings and redirect them to logging so they get formatted by Loguru
+    logging.captureWarnings(True)
+
 
 def obfuscate_message(message: str) -> str:
     """

fides/api/util/logger_context_utils.py

@@ -165,7 +165,9 @@ def request_details(
 
     if response is not None:
         if CONFIG.dev_mode and response.content:
-            details[LoggerContextKeys.response.value] = response.content.decode("utf-8")
+            details[LoggerContextKeys.response.value] = response.content.decode(
+                "utf-8", errors="replace"
+            )
 
         details[LoggerContextKeys.status_code.value] = response.status_code
 

fides/api/util/masking_util.py

@@ -0,0 +1,31 @@
+from typing import Any, Dict
+
+
+def mask_sensitive_fields(
+    connection_secrets: Dict[str, Any], secret_schema: Dict[str, Any]
+) -> Dict[str, Any]:
+    """
+    Mask sensitive fields in the given secrets based on the provided schema.
+    This function traverses the given secrets dictionary and uses the provided schema to
+    identify fields that have been marked as sensitive. The function replaces the sensitive
+    field values with a mask string ('********').
+    Args:
+        connection_secrets (Dict[str, Any]): The secrets to be masked.
+        secret_schema (Dict[str, Any]): The schema defining which fields are sensitive.
+    Returns:
+        Dict[str, Any]: The secrets dictionary with sensitive fields masked.
+    """
+    if connection_secrets is None:
+        return connection_secrets
+
+    secret_schema_properties = secret_schema["properties"]
+    new_connection_secrets = {}
+
+    for key, value in connection_secrets.items():
+        if key in secret_schema_properties:
+            if secret_schema_properties.get(key, {}).get("sensitive", False):
+                new_connection_secrets[key] = "**********"
+            else:
+                new_connection_secrets[key] = value
+
+    return new_connection_secrets

fides/api/util/memory_watchdog.py

@@ -13,6 +13,7 @@ killed by the system's OOM killer which would result in a WorkerLostError and in
 
 from __future__ import annotations
 
+import gc
 import os
 import signal
 import threading
@@ -21,6 +22,7 @@ from functools import wraps
 from types import FrameType, TracebackType
 from typing import Any, Callable, Literal, Optional, Type, TypeVar, Union
 
+import objgraph  # type: ignore
 import psutil  # type: ignore
 from loguru import logger
 
@@ -56,6 +58,118 @@ class MemoryLimitExceeded(RuntimeError):
         self.memory_percent: Optional[float] = memory_percent
 
 
+def _capture_heap_dump() -> None:
+    """Capture and log actionable heap dump information for memory leak diagnosis.
+
+    This function builds and logs a comprehensive memory profiling report as a single
+    message to help identify memory leaks. It's designed to be called when memory
+    threshold is exceeded, just before task termination.
+
+    The report includes:
+    - Process memory stats (RSS, VMS)
+    - Top 10 object type counts (spot anomalies like 100k of a custom class)
+    - Garbage collector stats (uncollectable objects indicate leaks)
+
+    All exceptions are caught to ensure heap dump failures don't prevent graceful
+    task termination.
+    """
+    try:
+        # Force garbage collection to get accurate counts
+        gc.collect()
+
+        report_lines = [
+            "",  # Leading newline for visual separation
+            "=" * 80,
+            "MEMORY DUMP",
+            "=" * 80,
+            "",
+        ]
+
+        # Section 1: Process Memory Stats
+        try:
+            process = psutil.Process()
+            mem_info = process.memory_info()
+            report_lines.extend(
+                [
+                    "PROCESS MEMORY STATS",
+                    "-" * 80,
+                    f"RSS (Resident Set Size):  {mem_info.rss / 1024 / 1024:.1f} MiB",
+                    f"VMS (Virtual Memory):     {mem_info.vms / 1024 / 1024:.1f} MiB",
+                    "",
+                ]
+            )
+        except Exception as exc:
+            logger.debug("Failed to get process memory stats: {}", exc)
+
+        # Section 2: Object Type Counts (most actionable for finding leaks)
+        type_stats = objgraph.most_common_types(limit=10)
+
+        report_lines.extend(
+            [
+                "OBJECT TYPE COUNTS (Top 10)",
+                "-" * 80,
+            ]
+        )
+
+        if type_stats:
+            max_typename_len = max(len(typename) for typename, _ in type_stats)
+
+            for typename, count in type_stats:
+                # Right-align count for easy scanning
+                report_lines.append(f"{typename:<{max_typename_len}}  {count:>10,}")
+        else:
+            report_lines.append("No object type data available")
+
+        report_lines.append("")
+
+        # Section 3: Garbage Collector Stats
+        gc_count = gc.get_count()
+
+        report_lines.extend(
+            [
+                "GARBAGE COLLECTOR STATS",
+                "-" * 80,
+                f"Collections: gen0={gc_count[0]}, gen1={gc_count[1]}, gen2={gc_count[2]}",
+                f"Tracked objects: {len(gc.get_objects()):,}",
+            ]
+        )
+
+        # Show uncollectable objects (strong indicator of memory leaks)
+        uncollectable = gc.garbage
+        if uncollectable:
+            report_lines.append(
+                f"⚠ UNCOLLECTABLE OBJECTS: {len(uncollectable)} (LEAK DETECTED!)"
+            )
+            # Show types of uncollectable objects
+            uncollectable_types: dict[str, int] = {}
+            for obj in uncollectable[:50]:  # Limit to first 50
+                obj_type = type(obj).__name__
+                uncollectable_types[obj_type] = uncollectable_types.get(obj_type, 0) + 1
+            for obj_type, count in sorted(
+                uncollectable_types.items(), key=lambda x: x[1], reverse=True
+            ):
+                report_lines.append(f"  {obj_type}: {count}")
+        else:
+            report_lines.append("No uncollectable objects detected")
+
+        report_lines.extend(
+            [
+                "",
+                "=" * 80,
+                "END OF MEMORY DUMP",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        # Log the entire report as a single message
+        heap_dump_report = "\n".join(report_lines)
+        logger.error(heap_dump_report)
+
+    except Exception as exc:  # pragma: no cover
+        logger.exception("Failed to capture heap dump: {}", exc)
+
+
 class MemoryWatchdog:
     """Background memory monitor that enables graceful task termination.
 
@@ -138,6 +252,10 @@ class MemoryWatchdog:
         """Convert the signal into an exception on the main thread."""
         current_percent = _system_memory_percent()
         logger.error("Memory limit exceeded: {}%", current_percent)
+
+        # Capture heap dump before terminating
+        _capture_heap_dump()
+
         self.stop()
         raise MemoryLimitExceeded(
             "Memory usage exceeded threshold", memory_percent=current_percent

fides/api/util/rate_limit.py

@@ -180,15 +180,33 @@ is_rate_limit_enabled = (
     CONFIG.security.rate_limit_client_ip_header is not None
     and CONFIG.security.rate_limit_client_ip_header != ""
 )
-fides_limiter = Limiter(
-    storage_uri=CONFIG.redis.connection_url_unencoded,
-    application_limits=[
-        CONFIG.security.request_rate_limit
-    ],  # Creates ONE shared bucket for all endpoints
+
+disabled_limiter = Limiter(
+    default_limits=[CONFIG.security.request_rate_limit],
     headers_enabled=True,
     key_prefix=CONFIG.security.rate_limit_prefix,
     key_func=safe_rate_limit_key,
     retry_after="http-date",
-    in_memory_fallback_enabled=False,  # Fall back to no rate limiting if Redis unavailable
-    enabled=is_rate_limit_enabled,
+    enabled=False,
 )
+
+try:
+    if is_rate_limit_enabled:
+        fides_limiter = Limiter(
+            storage_uri=CONFIG.redis.connection_url_unencoded,
+            application_limits=[
+                CONFIG.security.request_rate_limit
+            ],  # Creates ONE shared bucket for all endpoints
+            headers_enabled=True,
+            key_prefix=CONFIG.security.rate_limit_prefix,
+            key_func=safe_rate_limit_key,
+            retry_after="http-date",
+            in_memory_fallback_enabled=False,  # Fall back to no rate limiting if Redis unavailable
+            enabled=is_rate_limit_enabled,
+        )
+    else:
+        fides_limiter = disabled_limiter
+except Exception as e:
+    logger.exception("Error instantiating rate limiter: {}", e)
+    if is_rate_limit_enabled:
+        raise e

fides/api/util/saas_config_updater.py

@@ -0,0 +1,76 @@
+from typing import Iterable
+
+from loguru import logger
+from packaging.version import Version
+from packaging.version import parse as parse_version
+from sqlalchemy.orm import Session
+
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.saas_template_dataset import SaasTemplateDataset
+from fides.api.schemas.saas.connector_template import ConnectorTemplate
+from fides.api.schemas.saas.saas_config import SaaSConfig
+from fides.api.service.connectors.saas.connector_registry_service import (
+    ConnectorRegistry,
+)
+from fides.api.util.saas_util import load_config_from_string, load_dataset_from_string
+from fides.service.connection.connection_service import ConnectionService
+from fides.service.event_audit_service import EventAuditService
+
+
+def update_saas_configs(db: Session) -> None:
+    """
+    Updates SaaS config instances currently in the DB if to the
+    corresponding template in the registry are found.
+
+    Effectively an "update script" for SaaS config instances,
+    to be run on server bootstrap.
+    """
+    event_audit_service = EventAuditService(db)
+    saas_connection_service = ConnectionService(db, event_audit_service)
+    for connector_type in ConnectorRegistry.connector_types():
+        logger.debug(
+            "Determining if any updates are needed for connectors of type {} based on templates...",
+            connector_type,
+        )
+        template: ConnectorTemplate = ConnectorRegistry.get_connector_template(  # type: ignore
+            connector_type
+        )
+
+        # Store the original template dataset (with placeholders) instead of the modified version
+        template_dataset_json = load_dataset_from_string(template.dataset)
+
+        SaasTemplateDataset.get_or_create(
+            db=db,
+            connector_type=connector_type,
+            dataset_json=template_dataset_json,
+        )
+
+        saas_config = SaaSConfig(**load_config_from_string(template.config))
+        template_version: Version = parse_version(saas_config.version)
+        connection_configs: Iterable[ConnectionConfig] = ConnectionConfig.filter(
+            db=db,
+            conditions=(ConnectionConfig.saas_config["type"].astext == connector_type),
+        ).all()
+        for connection_config in connection_configs:
+            saas_config_instance = SaaSConfig.model_validate(
+                connection_config.saas_config
+            )
+            if parse_version(saas_config_instance.version) < template_version:
+                logger.info(
+                    "Updating SaaS config instance '{}' of type '{}' as its version, {}, was found to be lower than the template version {}",
+                    saas_config_instance.fides_key,
+                    connector_type,
+                    saas_config_instance.version,
+                    template_version,
+                )
+                try:
+                    saas_connection_service.update_saas_instance(
+                        connection_config,
+                        template,
+                        saas_config_instance,
+                    )
+                except Exception:
+                    logger.exception(
+                        "Encountered error attempting to update SaaS config instance {}",
+                        saas_config_instance.fides_key,
+                    )

fides/api/util/saas_util.py

@@ -16,7 +16,7 @@ from fides.api.common_exceptions import FidesopsException, ValidationError
 from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
 from fides.api.graph.config import Collection, CollectionAddress, Field, GraphDataset
 from fides.api.models.privacy_request import PrivacyRequest
-from fides.api.schemas.saas.saas_config import SaaSConfig, SaaSRequest
+from fides.api.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest
 from fides.api.schemas.saas.shared_schemas import SaaSRequestParams
 from fides.config import CONFIG
 from fides.config.helpers import load_file
@@ -26,6 +26,7 @@ PRIVACY_REQUEST_ID = "privacy_request_id"
 MASKED_OBJECT_FIELDS = "masked_object_fields"
 ALL_OBJECT_FIELDS = "all_object_fields"
 CUSTOM_PRIVACY_REQUEST_FIELDS = "custom_privacy_request_fields"
+PRIVACY_REQUEST_OBJECT = "privacy_request"
 UUID = "uuid"
 ISO_8601_DATETIME = "iso_8601_datetime"
 FIELD_LIST = "field_list"
@@ -335,6 +336,32 @@ def assign_placeholders(value: Any, param_values: Dict[str, Any]) -> Optional[An
     return value
 
 
+def check_dataset_missing_reference_values(
+    input_data: Dict[str, Any], param_values: Optional[List[ParamValue]]
+) -> List[str]:
+    """
+    Check if all the dataset reference values are present in the input_data map
+    """
+    # get the list of param_value references
+    required_param_value_references = [
+        param_value.name for param_value in param_values or [] if param_value.references
+    ]
+
+    # extract the keys from inside the fides_grouped_inputs and append them the other input_data keys
+    provided_input_keys = (
+        list(input_data.get("fidesops_grouped_inputs")[0].keys())  # type: ignore
+        if input_data.get("fidesops_grouped_inputs")
+        else []
+    ) + list(input_data.keys())
+
+    # find the missing values
+    missing_dataset_reference_values = list(
+        set(required_param_value_references) - set(provided_input_keys)
+    )
+
+    return missing_dataset_reference_values
+
+
 def map_param_values(
     action: str,
     context: str,

fides/api/worker/__init__.py

@@ -8,11 +8,14 @@ from loguru import logger
 from fides.api.db.base import Base  # type: ignore
 from fides.api.service.saas_request.override_implementations import *
 from fides.api.tasks import (
+    CONSENT_WEBHOOK_QUEUE_NAME,
     DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME,
     DISCOVERY_MONITORS_DETECTION_QUEUE_NAME,
     DISCOVERY_MONITORS_PROMOTION_QUEUE_NAME,
     DSR_QUEUE_NAME,
     MESSAGING_QUEUE_NAME,
+    PRIVACY_PREFERENCES_EXPORT_JOB_QUEUE_NAME,
+    PRIVACY_PREFERENCES_INGESTION_JOB_QUEUE_NAME,
     PRIVACY_PREFERENCES_QUEUE_NAME,
     celery_app,
 )
@@ -38,7 +41,10 @@ def start_worker(
         default_queue_name,
         MESSAGING_QUEUE_NAME,
         PRIVACY_PREFERENCES_QUEUE_NAME,
+        PRIVACY_PREFERENCES_EXPORT_JOB_QUEUE_NAME,
+        PRIVACY_PREFERENCES_INGESTION_JOB_QUEUE_NAME,
         DSR_QUEUE_NAME,
+        CONSENT_WEBHOOK_QUEUE_NAME,
         DISCOVERY_MONITORS_DETECTION_QUEUE_NAME,
         DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME,
         DISCOVERY_MONITORS_PROMOTION_QUEUE_NAME,

fides/common/api/scope_registry.py

@@ -76,6 +76,7 @@ VALIDATE = "validate"
 VIEW_DATA = "view_data"
 WEBHOOK = "webhook"
 WORKER_STATS = "worker-stats"
+HEAP_DUMP = "heap_dump"
 
 ASSIGN_OWNERS = "assign_owners"
 
@@ -146,6 +147,8 @@ DATASET_TEST = f"{DATASET}:{TEST}"
 
 ENCRYPTION_EXEC = f"{ENCRYPTION}:{EXEC}"
 
+HEAP_DUMP_EXEC = f"{HEAP_DUMP}:{EXEC}"
+
 EVALUATION_CREATE = f"{EVALUATION}:{CREATE}"
 EVALUATION_READ = f"{EVALUATION}:{READ}"
 EVALUATION_UPDATE = f"{EVALUATION}:{UPDATE}"
@@ -209,6 +212,7 @@ SAAS_CONFIG_READ = f"{SAAS_CONFIG}:{READ}"
 
 SAAS_CONNECTION_INSTANTIATE = f"{CONNECTION}:{INSTANTIATE}"
 CONNECTOR_TEMPLATE_REGISTER = f"{CONNECTOR_TEMPLATE}:{REGISTER}"
+CONNECTOR_TEMPLATE_READ = f"{CONNECTOR_TEMPLATE}:{READ}"
 
 SCOPE_READ = f"{SCOPE}:{READ}"
 
@@ -226,10 +230,6 @@ SYSTEM_MANAGER_READ = f"{SYSTEM_MANAGER}:{READ}"
 SYSTEM_MANAGER_UPDATE = f"{SYSTEM_MANAGER}:{UPDATE}"
 SYSTEM_MANAGER_DELETE = f"{SYSTEM_MANAGER}:{DELETE}"
 
-TAXONOMY_CREATE = f"{TAXONOMY}:{CREATE}"
-TAXONOMY_UPDATE = f"{TAXONOMY}:{UPDATE}"
-TAXONOMY_DELETE = f"{TAXONOMY}:{DELETE}"
-
 USER_CREATE = f"{USER}:{CREATE}"
 USER_DELETE = f"{USER}:{DELETE}"
 USER_READ = f"{USER}:{READ}"
@@ -267,6 +267,7 @@ SCOPE_DOCS = {
     CONNECTION_AUTHORIZE: "OAuth2 Authorization",
     CONNECTION_TYPE_READ: "View types of connections",
     CONNECTOR_TEMPLATE_REGISTER: "Register a connector template",
+    CONNECTOR_TEMPLATE_READ: "View connector template configurations",
     CONSENT_READ: "Read consent preferences",
     CONSENT_SETTINGS_READ: "Read org-wide consent settings",
     CONSENT_SETTINGS_UPDATE: "Update org-wide consent settings",
@@ -299,6 +300,7 @@ SCOPE_DOCS = {
     DATASET_READ: "View datasets",
     DATASET_TEST: "Run a standalone privacy request test for a dataset",
     ENCRYPTION_EXEC: "Encrypt data",
+    HEAP_DUMP_EXEC: "Execute a heap dump for memory diagnostics",
     MESSAGING_TEMPLATE_UPDATE: "Update messaging templates",
     EVALUATION_CREATE: "Create evaluation",
     EVALUATION_READ: "Read evaluations",
@@ -357,9 +359,6 @@ SCOPE_DOCS = {
     SYSTEM_MANAGER_READ: "Read systems users can manage",
     SYSTEM_MANAGER_DELETE: "Delete systems user can manage",
     SYSTEM_MANAGER_UPDATE: "Update systems user can manage",
-    TAXONOMY_CREATE: "Create local taxonomy",
-    TAXONOMY_DELETE: "Delete local taxonomy",
-    TAXONOMY_UPDATE: "Update local taxonomy",
     USER_CREATE: "Create users",
     USER_UPDATE: "Update users",
     USER_DELETE: "Remove users",

fides/common/api/v1/urn_registry.py

@@ -66,7 +66,8 @@ MESSAGING_ACTIVE_DEFAULT = "/messaging/default/active"
 MESSAGING_STATUS = "/messaging/default/status"
 MESSAGING_DEFAULT_SECRETS = "/messaging/default/{service_type}/secret"
 MESSAGING_DEFAULT_BY_TYPE = "/messaging/default/{service_type}"
-MESSAGING_TEST = "/messaging/config/test"
+MESSAGING_TEST_DEPRECATED = "/messaging/config/test"  # DEPRECATED
+MESSAGING_TEST = "/messaging/test/{service_type}"
 MESSAGING_EMAIL_INVITE_STATUS = "/messaging/email-invite/status"
 
 # Policy URLs
@@ -83,8 +84,10 @@ PRIVACY_REQUEST_BATCH_EMAIL_SEND = (
     "/privacy-request/administrate/process-awaiting-email-send"
 )
 PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated"
+PRIVACY_REQUEST_BULK_FINALIZE = "/privacy-request/bulk/finalize"
 PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry"
 PRIVACY_REQUEST_BULK_SOFT_DELETE = "/privacy-request/bulk/soft-delete"
+PRIVACY_REQUEST_CANCEL = "/privacy-request/administrate/cancel"
 PRIVACY_REQUEST_DENY = "/privacy-request/administrate/deny"
 PRIVACY_REQUEST_SOFT_DELETE = "/privacy-request/{privacy_request_id}/soft-delete"
 REQUEST_STATUS_LOGS = "/privacy-request/{privacy_request_id}/log"
@@ -157,6 +160,7 @@ CONNECTION_TYPE_SECRETS = "/connection_type/{connection_type}/secret"
 CONNECTIONS = "/connection"
 CONNECTION_BY_KEY = "/connection/{connection_key}"
 CONNECTION_SECRETS = "/connection/{connection_key}/secret"
+CONNECTION_OAUTH = "/connection/{connection_key}/oauth"
 CONNECTION_TEST = "/connection/{connection_key}/test"
 AUTHORIZE = "/connection/{connection_key}/authorize"
 
@@ -187,13 +191,21 @@ YAML_DATASETS = YAML + CONNECTION_DATASETS
 # SaaS Config URLs
 SAAS_CONFIG_VALIDATE = CONNECTION_BY_KEY + "/validate_saas_config"
 SAAS_CONFIG = CONNECTION_BY_KEY + "/saas_config"
-SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{saas_connector_type}"
+SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{connector_template_type}"
+
+# Connector Template URLs
+CONNECTOR_TEMPLATES = "/connector-templates"
+CONNECTOR_TEMPLATES_REGISTER = "/connector-templates/register"
+CONNECTOR_TEMPLATES_CONFIG = "/connector-templates/{connector_template_type}/config"
+CONNECTOR_TEMPLATES_DATASET = "/connector-templates/{connector_template_type}/dataset"
+
+# Deprecated: Old connector template register URL
 REGISTER_CONNECTOR_TEMPLATE = "/connector_template/register"
 
 # System Connections
 SYSTEM_CONNECTIONS = "/system/{fides_key}/connection"
 INSTANTIATE_SYSTEM_CONNECTION = (
-    "/system/{fides_key}/connection/instantiate/{saas_connector_type}"
+    "/system/{fides_key}/connection/instantiate/{connector_template_type}"
 )
 
 # User URLs