ethyca-fides 2.69.0rc10__py2.py3-none-any.whl → 2.69.1__py2.py3-none-any.whl

fides/api/service/privacy_request/dsr_package/utils.py

@@ -0,0 +1,268 @@
+from typing import Any, List, Optional
+
+from fideslang.models import Dataset, DatasetField
+from loguru import logger
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import ActionType
+
+
+# TODO: keeping this for a bit to help with development and testing
+def get_redaction_entities_map(db: Session) -> set[str]:
+    """
+    Create a set of hierarchical entity keys that should be redacted based on fides_meta.redact: name.
+
+    This utility function reads all enabled dataset configurations from the database
+    and builds a set of hierarchical entity keys (dataset_name, dataset_name.collection_name,
+    dataset_name.collection_name.field_name) that have fides_meta.redact set to "name".
+
+    Supports deeply nested field structures with unlimited nesting depth.
+
+    Args:
+        db: Database session
+
+    Returns:
+        Set of hierarchical entity keys that should be redacted
+    """
+    redaction_entities = set()
+
+    try:
+        dataset_configs = DatasetConfig.all(db=db)
+
+        for dataset_config in dataset_configs:
+            ctl_dataset = dataset_config.ctl_dataset
+            if not ctl_dataset:
+                continue
+
+            dataset = Dataset.model_validate(dataset_config.ctl_dataset)
+            # Intentionally using the fides_key instead of name since it's always provided
+            dataset_name = dataset.fides_key
+
+            # Check dataset level
+            if dataset.fides_meta and dataset.fides_meta.redact == "name":
+                redaction_entities.add(dataset_name)
+
+            # Check collection level
+            for collection_dict in dataset.collections:
+                # Collections are stored as dictionaries in the database
+                collection_name = collection_dict.name
+                if not collection_name:
+                    continue
+
+                collection_path = f"{dataset_name}.{collection_name}"
+                collection_fides_meta = collection_dict.fides_meta
+
+                if collection_fides_meta and collection_fides_meta.redact == "name":
+                    redaction_entities.add(collection_path)
+
+                # Check field level (with recursive nested field support)
+                _traverse_fields_for_redaction(
+                    collection_dict.fields, collection_path, redaction_entities
+                )
+
+    except Exception as exc:
+        # Log error but don't fail, just return empty set
+        logger.warning(f"Error extracting redaction configurations: {exc}")
+
+    return redaction_entities
+
+
+def get_redaction_entities_map_db(db: Session) -> set[str]:
+    """
+    Create a set of hierarchical entity keys that should be redacted based on fides_meta.redact: name.
+
+    This function uses a hybrid approach:
+    1. First identifies datasets that contain ANY redaction metadata at any level
+    2. Then processes only those datasets with redaction metadata
+
+
+    Args:
+        db: Database session
+
+    Returns:
+        Set of hierarchical entity keys that should be redacted
+    """
+    redaction_entities: set[str] = set()
+
+    try:
+        # Step 1: Pre-filter to find datasets with ANY redaction metadata
+        # Simple existence check - no paths needed, just check if redaction exists anywhere
+        pre_filter_query = """
+        SELECT DISTINCT dc.ctl_dataset_id
+        FROM datasetconfig dc
+        JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+        WHERE
+            -- Dataset-level redaction
+            ds.fides_meta->>'redact' = 'name'
+            OR
+            -- Collection-level redaction
+            EXISTS (
+                SELECT 1 FROM jsonb_array_elements(ds.collections::jsonb) AS collection
+                WHERE collection->'fides_meta'->>'redact' = 'name'
+                LIMIT 1
+            )
+            OR
+            -- Field-level redaction using jsonb_path_query
+            EXISTS (
+                SELECT 1
+                FROM jsonb_path_query(ds.collections::jsonb, '$.**.fides_meta') AS fides_meta
+                WHERE fides_meta->>'redact' = 'name'
+                LIMIT 1
+            )
+        """
+
+        candidate_datasets = db.execute(pre_filter_query).fetchall()
+
+        if not candidate_datasets:
+            logger.debug("No datasets found with redaction metadata")
+            return redaction_entities
+
+        logger.debug(
+            f"Pre-filtered to {len(candidate_datasets)} datasets with redaction metadata"
+        )
+
+        # Step 2: Process only the candidate datasets with targeted queries
+        # Convert to a format we can use in SQL ANY clause
+        dataset_ids = [row[0] for row in candidate_datasets]
+
+        # Query for dataset-level redactions (only on candidate datasets)
+        dataset_query = text(
+            """
+        SELECT ds.fides_key as entity_path
+        FROM datasetconfig dc
+        JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+        WHERE ds.id = ANY(:dataset_ids)
+            AND ds.fides_meta->>'redact' = 'name'
+        """
+        )
+
+        dataset_results = db.execute(
+            dataset_query, {"dataset_ids": dataset_ids}
+        ).fetchall()
+        for row in dataset_results:
+            redaction_entities.add(row[0])
+
+        # Query for collection-level redactions (only on candidate datasets)
+        collection_query = text(
+            """
+        SELECT ds.fides_key || '.' || (collection->>'name') as entity_path
+        FROM datasetconfig dc
+        JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+        CROSS JOIN LATERAL jsonb_array_elements(ds.collections::jsonb) AS collection
+        WHERE ds.id = ANY(:dataset_ids)
+            AND collection->'fides_meta'->>'redact' = 'name'
+            AND collection->>'name' IS NOT NULL
+        """
+        )
+
+        collection_results = db.execute(
+            collection_query, {"dataset_ids": dataset_ids}
+        ).fetchall()
+        for row in collection_results:
+            redaction_entities.add(row[0])
+
+        # Query for field-level redactions (including nested fields)
+        # This uses a recursive CTE to handle arbitrary nesting levels
+        field_query = text(
+            """
+        WITH RECURSIVE field_hierarchy AS (
+            -- Base case: top-level fields in collections (only candidate datasets)
+            SELECT
+                ds.fides_key || '.' ||
+                    (collection->>'name') || '.' ||
+                    (field->>'name') as entity_path,
+                field->'fields' as nested_fields,
+                field->'fides_meta'->>'redact' as redact_value
+            FROM datasetconfig dc
+            JOIN ctl_datasets ds ON dc.ctl_dataset_id = ds.id
+            CROSS JOIN LATERAL jsonb_array_elements(ds.collections::jsonb) AS collection
+            CROSS JOIN LATERAL jsonb_array_elements(collection->'fields') AS field
+            WHERE ds.id = ANY(:dataset_ids)
+                AND collection->>'name' IS NOT NULL
+                AND field->>'name' IS NOT NULL
+
+            UNION ALL
+
+            -- Recursive case: nested fields
+            SELECT
+                fh.entity_path || '.' || (nested_field->>'name') as entity_path,
+                nested_field->'fields' as nested_fields,
+                nested_field->'fides_meta'->>'redact' as redact_value
+            FROM field_hierarchy fh
+            CROSS JOIN LATERAL jsonb_array_elements(fh.nested_fields) AS nested_field
+            WHERE jsonb_typeof(fh.nested_fields) = 'array'
+                AND nested_field->>'name' IS NOT NULL
+        )
+        SELECT DISTINCT entity_path
+        FROM field_hierarchy
+        WHERE redact_value = 'name'
+        """
+        )
+
+        field_results = db.execute(field_query, {"dataset_ids": dataset_ids}).fetchall()
+        for row in field_results:
+            redaction_entities.add(row[0])
+
+        logger.debug(f"Found {len(redaction_entities)} entities requiring redaction")
+
+    except Exception as exc:
+        # Log error but don't fail, just return empty set
+        logger.warning(
+            f"Error extracting redaction configurations from database: {exc}"
+        )
+
+    return redaction_entities
+
+
+def map_privacy_request(privacy_request: PrivacyRequest) -> dict[str, Any]:
+    """Creates a map with a subset of values from the privacy request"""
+    request_data: dict[str, Any] = {}
+    request_data["id"] = privacy_request.id
+
+    action_type: Optional[ActionType] = privacy_request.policy.get_action_type()
+    if action_type:
+        request_data["type"] = action_type.value
+
+    request_data["identity"] = {
+        key: value
+        for key, value in privacy_request.get_persisted_identity()
+        .labeled_dict(include_default_labels=True)
+        .items()
+        if value["value"] is not None
+    }
+
+    if privacy_request.requested_at:
+        request_data["requested_at"] = privacy_request.requested_at.strftime(
+            "%m/%d/%Y %H:%M %Z"
+        )
+    return request_data
+
+
+def _traverse_fields_for_redaction(
+    fields: List[DatasetField], current_path: str, redaction_entities: set[str]
+) -> None:
+    """
+    Recursively traverse nested fields to find redaction entities.
+
+    Args:
+        fields: List of field dictionaries to traverse
+        current_path: Current hierarchical path (e.g., "dataset.collection")
+        redaction_entities: Set to add redacted field paths to
+    """
+    for field in fields:
+        field_name = field.name
+        if not field_name:
+            continue
+
+        field_path = f"{current_path}.{field_name}"
+        field_fides_meta = field.fides_meta
+
+        if field_fides_meta and field_fides_meta.redact == "name":
+            redaction_entities.add(field_path)
+
+        # Recursively check nested fields
+        if field.fields:
+            _traverse_fields_for_redaction(field.fields, field_path, redaction_entities)

fides/api/service/storage/streaming/smart_open_streaming_storage.py

@@ -17,7 +17,7 @@ from fides.api.common_exceptions import StorageUploadError
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.storage.storage import ResponseFormat
 from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
-    DsrReportBuilder,
+    DSRReportBuilder,
 )
 from fides.api.service.storage.streaming.dsr_storage import (
     create_dsr_report_files_generator,
@@ -346,7 +346,7 @@ class SmartOpenStreamingStorage:
         )
 
     def _collect_and_validate_attachments_from_dsr_builder(
-        self, data: dict, dsr_builder: "DsrReportBuilder"
+        self, data: dict, dsr_builder: "DSRReportBuilder"
     ) -> list[AttachmentProcessingInfo]:
         """Collect and validate attachments using the DSR report builder's processed attachments.
 
@@ -544,7 +544,7 @@ class SmartOpenStreamingStorage:
         """
         # Generate the DSR report first
         try:
-            dsr_builder = DsrReportBuilder(
+            dsr_builder = DSRReportBuilder(
                 privacy_request=privacy_request,
                 dsr_data=data,
                 enable_streaming=True,

fides/api/tasks/storage.py

@@ -12,7 +12,7 @@ from loguru import logger
 from fides.api.common_exceptions import StorageUploadError
 from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
 from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
-    DsrReportBuilder,
+    DSRReportBuilder,
 )
 from fides.api.service.storage.gcs import get_gcs_blob
 from fides.api.service.storage.s3 import (
@@ -47,7 +47,7 @@ def write_to_in_memory_buffer(
     logger.debug("Writing data to in-memory buffer")
     try:
         if resp_format == ResponseFormat.html.value:
-            return DsrReportBuilder(
+            return DSRReportBuilder(
                 privacy_request=privacy_request,
                 dsr_data=data,
             ).generate()

fides/api/util/endpoint_utils.py

@@ -5,8 +5,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from fastapi import HTTPException
 from fideslang import FidesModelType
-from slowapi import Limiter
-from slowapi.util import get_remote_address  # type: ignore
 from sqlalchemy.ext.asyncio import AsyncSession
 from starlette.status import HTTP_400_BAD_REQUEST
 
@@ -23,7 +21,6 @@ from fides.common.api.scope_registry import (
     ORGANIZATION,
     SYSTEM,
 )
-from fides.config import CONFIG
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     ModelWithDefaultField,
@@ -44,16 +41,6 @@ CLI_SCOPE_PREFIX_MAPPING: Dict[str, str] = {
     "system": SYSTEM,
 }
 
-# Used for rate limiting with Slow API
-# Decorate individual routes to deviate from the default rate limits
-fides_limiter = Limiter(
-    default_limits=[CONFIG.security.request_rate_limit],
-    headers_enabled=True,
-    key_prefix=CONFIG.security.rate_limit_prefix,
-    key_func=get_remote_address,
-    retry_after="http-date",
-)
-
 
 async def forbid_if_editing_is_default(
     sql_model: Base,

fides/api/util/rate_limit.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+from typing import Optional
+
+from fastapi import Request
+from fastapi.responses import JSONResponse
+from loguru import logger
+from slowapi import Limiter
+from slowapi.util import get_remote_address  # type: ignore
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from fides.config import CONFIG
+
+
+class InvalidClientIPError(Exception):
+    def __init__(self, detail: str, header_value: str, header_name: str):
+        self.detail = detail
+        self.header_value = header_value
+        self.header_name = header_name
+        super().__init__(detail)
+
+
+def validate_client_ip(ip: Optional[str]) -> bool:
+    """
+    Returns true if the provided ip is valid and not from a reserved range.
+    Returns false otherwise.
+    """
+    if not ip:
+        return False
+    try:
+        ip_obj = ip_address(ip)
+        if (
+            ip_obj.is_loopback
+            or ip_obj.is_link_local
+            or ip_obj.is_reserved
+            or ip_obj.is_multicast
+            or ip_obj.is_private
+        ):
+            return False
+        return True
+    except ValueError:
+        return False
+
+
+def _extract_hostname_from_ip(ip: str) -> Optional[str]:
+    """
+    Extract hostname/IP address from header value, stripping port if present.
+
+    Simple string-based approach following the reference implementation pattern.
+    Does not validate whether the result is a valid IP address.
+
+    Examples:
+        # IPv4 cases
+        _extract_hostname_from_ip("192.168.1.1") -> "192.168.1.1"
+        _extract_hostname_from_ip("192.168.1.1:8080") -> "192.168.1.1"
+
+        # IPv6 cases
+        _extract_hostname_from_ip("2001:db8::1") -> "2001:db8::1"
+        _extract_hostname_from_ip("[2001:db8::1]:8080") -> "2001:db8::1"
+
+        # Edge cases (alidation will later reject)
+        _extract_hostname_from_ip("192.168.1.1, 192.168.1.2") -> "192.168.1.1, 192.168.1.2"
+        _extract_hostname_from_ip("not-an-ip:8080") -> "not-an-ip"
+
+        # Error
+        _extract_hostname_from_ip("") -> raises ValueError
+
+    Raises:
+        ValueError: If no hostname can be extracted from the input
+    """
+
+    clean_ip = ip.strip()
+
+    if not clean_ip:
+        raise ValueError("Could not parse IP from header value")
+
+    # Handle IPv6 with port: [IPv6]:port
+    if "]:" in clean_ip:
+        return clean_ip.split("]:")[0].replace("[", "").strip()
+
+    # Handle IPv4 with port: IPv4:port
+    if ":" in clean_ip and "::" not in clean_ip:
+        return clean_ip.split(":")[0].strip()
+
+    # Return as-is (IPv6 without port, IPv4 without port, or other values)
+    return clean_ip
+
+
+def _resolve_client_ip_from_header(request: Request, strict: bool) -> str:
+    """Shared resolver for client IP from the configured header.
+
+    - When strict=True: raise InvalidClientIPError on invalid/malformed header values.
+    - When strict=False: never raise; fall back to the connection source IP.
+    """
+    header_name = CONFIG.security.rate_limit_client_ip_header
+    if not header_name:
+        # This line should never be reached when rate limiting is enabled
+        logger.warning(
+            "Rate limit client IP header not configured. Falling back to source IP.",
+            header_name,
+        )
+        return get_remote_address(request)
+
+    ip_address_from_header = request.headers.get(header_name)
+    if not ip_address_from_header:
+        logger.debug(
+            "Rate limit header '{}' not found. Falling back to source IP.",
+            header_name,
+        )
+        return get_remote_address(request)
+
+    # Extract and validate IP
+    try:
+        extracted_ip = _extract_hostname_from_ip(ip_address_from_header)
+        if extracted_ip and validate_client_ip(extracted_ip):
+            return extracted_ip
+        raise ValueError("IP failed validation")
+    except ValueError:
+        if strict:
+            logger.error(
+                "Invalid IP '{}' in header '{}'. Rejecting request.",
+                ip_address_from_header,
+                header_name,
+            )
+            raise InvalidClientIPError(
+                detail="Invalid IP address format",
+                header_value=ip_address_from_header,
+                header_name=header_name,
+            )
+        # Non-strict path: fall back silently to source IP
+        return get_remote_address(request)
+
+
+def get_client_ip_from_header(request: Request) -> str:
+    """
+    Extracts the client IP from the configured CDN header.
+
+    If the header is not configured or is missing, it falls back to the
+    source IP on the request.
+
+    Raises InvalidClientIPError if header contains invalid IP format.
+    """
+    return _resolve_client_ip_from_header(request, strict=True)
+
+
+def safe_rate_limit_key(request: Request) -> str:
+    """
+    Safe key function for SlowAPI limiter.
+
+    Must never raise. If the configured header is missing or malformed,
+    fall back to the connection source IP for rate limiting purposes.
+    """
+    return _resolve_client_ip_from_header(request, strict=False)
+
+
+class RateLimitIPValidationMiddleware(BaseHTTPMiddleware):
+    """
+    Pre-validate the configured client IP header when rate limiting is enabled.
+
+    If the header is present but invalid, short-circuit the request with 422.
+    This keeps SlowAPI's middleware path free of exceptions from the key function.
+    """
+
+    async def dispatch(self, request: Request, call_next):  # type: ignore
+        if is_rate_limit_enabled:
+            try:
+                # Triggers parsing/validation; raises on invalid header
+                get_client_ip_from_header(request)
+            except InvalidClientIPError:
+                return JSONResponse(
+                    status_code=422, content={"detail": "Invalid client IP header"}
+                )
+        return await call_next(request)
+
+
+# Used for rate limiting with Slow API
+# Decorate individual routes to deviate from the default rate limits
+is_rate_limit_enabled = (
+    CONFIG.security.rate_limit_client_ip_header is not None
+    and CONFIG.security.rate_limit_client_ip_header != ""
+)
+fides_limiter = Limiter(
+    storage_uri=CONFIG.redis.connection_url_unencoded,
+    application_limits=[
+        CONFIG.security.request_rate_limit
+    ],  # Creates ONE shared bucket for all endpoints
+    headers_enabled=True,
+    key_prefix=CONFIG.security.rate_limit_prefix,
+    key_func=safe_rate_limit_key,
+    retry_after="http-date",
+    in_memory_fallback_enabled=False,  # Fall back to no rate limiting if Redis unavailable
+    enabled=is_rate_limit_enabled,
+)

fides/common/api/scope_registry.py

@@ -30,6 +30,7 @@ DATA_SUBJECT = "data_subject"
 DATA_USE = "data_use"
 DATASET = "dataset"
 DELETE = "delete"
+PRIVACY_REQUEST_REDACTION_PATTERNS = "privacy-request-redaction-patterns"
 ENCRYPTION = "encryption"
 MESSAGING_TEMPLATE = "messaging-template"
 EVALUATION = "evaluation"
@@ -134,6 +135,11 @@ DATA_USE_UPDATE = f"{DATA_USE}:{UPDATE}"
 DATA_USE_DELETE = f"{DATA_USE}:{DELETE}"
 
 DATASET_CREATE_OR_UPDATE = f"{DATASET}:{CREATE_OR_UPDATE}"
+
+PRIVACY_REQUEST_REDACTION_PATTERNS_READ = f"{PRIVACY_REQUEST_REDACTION_PATTERNS}:{READ}"
+PRIVACY_REQUEST_REDACTION_PATTERNS_UPDATE = (
+    f"{PRIVACY_REQUEST_REDACTION_PATTERNS}:{UPDATE}"
+)
 DATASET_DELETE = f"{DATASET}:{DELETE}"
 DATASET_READ = f"{DATASET}:{READ}"
 DATASET_TEST = f"{DATASET}:{TEST}"
@@ -288,6 +294,8 @@ SCOPE_DOCS = {
     DATA_USE_UPDATE: "Update data uses",
     DATASET_CREATE_OR_UPDATE: "Create or modify datasets",
     DATASET_DELETE: "Delete datasets",
+    PRIVACY_REQUEST_REDACTION_PATTERNS_READ: "View privacy request redaction patterns",
+    PRIVACY_REQUEST_REDACTION_PATTERNS_UPDATE: "Update privacy request redaction patterns",
     DATASET_READ: "View datasets",
     DATASET_TEST: "Run a standalone privacy request test for a dataset",
     ENCRYPTION_EXEC: "Encrypt data",

fides/common/api/v1/urn_registry.py

@@ -116,6 +116,9 @@ PRIVACY_REQUEST_TRANSFER_TO_PARENT = (
     "/privacy-request/transfer/{privacy_request_id}/{rule_key}"
 )
 
+# Privacy Request Redaction Patterns URLs
+PRIVACY_REQUEST_REDACTION_PATTERNS = "/privacy-request/redaction-patterns"
+
 # Privacy Request pre-approve URLs
 PRIVACY_REQUEST_PRE_APPROVE = "/privacy-request/{privacy_request_id}/pre-approve"
 PRIVACY_REQUEST_PRE_APPROVE_ELIGIBLE = PRIVACY_REQUEST_PRE_APPROVE + "/eligible"

fides/config/redis_settings.py

@@ -181,8 +181,24 @@ class RedisSettings(FidesSettings):
         description="A full connection URL to the read-only Redis cache. If not specified, this URL is automatically assembled from the read_only_host, read_only_port, read_only_password and read_only_db_index specified above.",
         exclude=True,
     )
+    connection_url_unencoded: Optional[str] = Field(
+        default=None,
+        description="A full connection URL to the Redis cache with the password unencoded. If not specified, this URL is automatically assembled from the host, port, password and db_index specified above.",
+        exclude=True,
+    )
+    read_only_connection_url_unencoded: Optional[str] = Field(
+        default=None,
+        description="A full connection URL to the read-only Redis cache with the password unencoded. If not specified, this URL is automatically assembled from the read_only_host, read_only_port, read_only_password and read_only_db_index specified above.",
+        exclude=True,
+    )
 
-    @field_validator("connection_url", "read_only_connection_url", mode="before")
+    @field_validator(
+        "connection_url",
+        "read_only_connection_url",
+        "connection_url_unencoded",
+        "read_only_connection_url_unencoded",
+        mode="before",
+    )
     @classmethod
     def assemble_connection_url(
         cls,
@@ -195,7 +211,14 @@ class RedisSettings(FidesSettings):
             return v
 
         # Determine which set of settings to use based on field name
-        is_read_only = info.field_name == "read_only_connection_url"
+        is_read_only = info.field_name in (
+            "read_only_connection_url",
+            "read_only_connection_url_unencoded",
+        )
+        is_unencoded = info.field_name in (
+            "connection_url_unencoded",
+            "read_only_connection_url_unencoded",
+        )
 
         # Extract settings - fallbacks already resolved by field validators for read-only fields
         user = (
@@ -244,7 +267,8 @@ class RedisSettings(FidesSettings):
         # redis://<user>:<password>@<host>
         auth_prefix = ""
         if password or user:
-            auth_prefix = f"{quote_plus(user)}:{quote_plus(password)}@"
+            encoded_password = password if is_unencoded else quote_plus(password)
+            auth_prefix = f"{quote_plus(user)}:{encoded_password}@"
 
         # For host, we don't have a fallback - read replica should be a different host
         host = (