ethyca-fides 2.69.0rc10__py2.py3-none-any.whl → 2.69.1__py2.py3-none-any.whl

fides/api/oauth/utils.py

@@ -4,16 +4,16 @@ import json
 from datetime import datetime
 from functools import update_wrapper
 from types import FunctionType
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple, cast
 
-from fastapi import Depends, HTTPException, Security
+from fastapi import Depends, HTTPException, Request, Security
 from fastapi.security import SecurityScopes
 from jose import exceptions, jwe
 from jose.constants import ALGORITHMS
 from loguru import logger
 from pydantic import ValidationError
 from sqlalchemy.orm import Session
-from starlette.status import HTTP_404_NOT_FOUND
+from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
 
 from fides.api.api.deps import get_db
 from fides.api.common_exceptions import AuthenticationError, AuthorizationError
@@ -30,7 +30,7 @@ from fides.api.models.fides_user_permissions import FidesUserPermissions
 from fides.api.models.policy import PolicyPreWebhook
 from fides.api.models.pre_approval_webhook import PreApprovalWebhook
 from fides.api.models.privacy_request import RequestTask
-from fides.api.oauth.roles import get_scopes_from_roles
+from fides.api.oauth.roles import ROLES_TO_SCOPES_MAPPING, get_scopes_from_roles
 from fides.api.request_context import set_user_id
 from fides.api.schemas.external_https import (
     DownloadTokenJWE,
@@ -80,6 +80,28 @@ def is_callback_token_expired(issued_at: Optional[datetime]) -> bool:
     ).total_seconds() / 60.0 > CONFIG.execution.privacy_request_delay_timeout
 
 
+def is_token_invalidated(issued_at: datetime, client: ClientDetail) -> bool:
+    """
+    Return True if the token should be considered invalid due to security events
+    (e.g., user password reset) that occurred after the token was issued.
+
+    Any errors accessing related objects are logged and treated as non-invalidating.
+    """
+    try:
+        if (
+            client.user is not None
+            and client.user.password_reset_at is not None
+            and issued_at < client.user.password_reset_at
+        ):
+            return True
+        return False
+    except Exception as exc:
+        logger.exception(
+            "Unable to evaluate password reset timestamp for client user: {}", exc
+        )
+        return False
+
+
 def _get_webhook_jwe_or_error(
     security_scopes: SecurityScopes, authorization: str = Security(oauth2_scheme)
 ) -> WebhookJWE:
@@ -225,7 +247,7 @@ async def get_current_user(
             created_at=datetime.utcnow(),
         )
 
-    return client.user  # type: ignore[attr-defined]
+    return cast(FidesUser, client.user)
 
 
 def verify_callback_oauth_policy_pre_webhook(
@@ -370,8 +392,10 @@ def extract_token_and_load_client(
         logger.debug("Auth token expired.")
         raise AuthorizationError(detail="Not Authorized for this action")
 
+    issued_at_dt = datetime.fromisoformat(issued_at)
+
     if is_token_expired(
-        datetime.fromisoformat(issued_at),
+        issued_at_dt,
         token_duration_override or CONFIG.security.oauth_access_token_expire_minutes,
     ):
         raise AuthorizationError(detail="Not Authorized for this action")
@@ -394,6 +418,12 @@ def extract_token_and_load_client(
         logger.debug("Auth token belongs to an invalid client_id.")
         raise AuthorizationError(detail="Not Authorized for this action")
 
+    # Invalidate tokens issued prior to the user's most recent password reset.
+    # This ensures any existing sessions are expired immediately after a password change.
+    if is_token_invalidated(issued_at_dt, client):
+        logger.debug("Auth token issued before latest password reset.")
+        raise AuthorizationError(detail="Not Authorized for this action")
+
     # Populate request-scoped context with the authenticated user identifier.
     # Prefer the linked user_id; fall back to the client id when this is the
     # special root client (which has no associated FidesUser row).
@@ -476,6 +506,87 @@ def has_scope_subset(user_scopes: List[str], endpoint_scopes: SecurityScopes) ->
     return set(endpoint_scopes.scopes).issubset(user_scopes)
 
 
+def get_client_effective_scopes(client: "ClientDetail") -> List[str]:
+    """
+    Get all scopes available to a client, including both direct scopes and role-derived scopes.
+
+    Args:
+        client: The ClientDetail instance
+
+    Returns:
+        List of scope strings that the client has access to
+    """
+    effective_scopes = set()
+
+    # Add direct scopes
+    if client.scopes:
+        effective_scopes.update(client.scopes)
+
+    # Add role-derived scopes
+    if client.roles:
+        for role in client.roles:
+            role_scopes = ROLES_TO_SCOPES_MAPPING.get(role, [])
+            effective_scopes.update(role_scopes)
+
+    # Add user permission scopes if client is associated with a user
+    # Note: client.user is available via SQLAlchemy backref from FidesUser.client relationship
+    user = getattr(client, "user", None)  # Use getattr to avoid mypy attr-defined error
+    if user and hasattr(user, "permissions") and user.permissions:
+        effective_scopes.update(user.permissions.total_scopes)
+
+    return sorted(list(effective_scopes))
+
+
+def verify_client_can_assign_scopes(
+    request: "Request",
+    requesting_client: "ClientDetail",
+    scopes: List[str],
+    db: "Session",
+) -> None:
+    """
+    Verify that a requesting client has permission to assign the given scopes.
+
+    Raises HTTPException if the client lacks permission.
+    Root client is exempt from this check.
+
+    Args:
+        request: FastAPI request object containing Authorization header
+        requesting_client: The client making the request
+        scopes: List of scopes to be assigned
+        db: Database session
+
+    Raises:
+        HTTPException: If the client lacks permission to assign the scopes
+    """
+    # Root client can assign any scope
+    if requesting_client.id == CONFIG.security.oauth_root_client_id:
+        return
+
+    # Get the actual token scopes (not the client's database scopes)
+    authorization = request.headers.get("Authorization", "").replace("Bearer ", "")
+    token_data, _ = extract_token_and_load_client(authorization, db)
+
+    # Get token's effective scopes
+    token_scopes = token_data.get("scopes", [])
+
+    # Check if user has the scopes via roles as well
+    has_scope_via_role = has_permissions(
+        token_data=token_data,
+        client=requesting_client,
+        endpoint_scopes=SecurityScopes(scopes),
+    )
+
+    # If they don't have all scopes via direct assignment or roles, check individual scopes
+    if not has_scope_via_role:
+        unauthorized_scopes = set(scopes) - set(token_scopes)
+
+        if unauthorized_scopes:
+            raise HTTPException(
+                status_code=HTTP_403_FORBIDDEN,
+                detail=f"Cannot assign scopes that you do not have. Missing scopes: {sorted(unauthorized_scopes)}",
+            )
+
+
 def create_temporary_user_for_login_flow(config: FidesConfig) -> FidesUser:
     """
     Create a temporary FidesUser in-memory with an attached in-memory ClientDetail

fides/api/schemas/privacy_request_redaction_patterns.py

@@ -0,0 +1,55 @@
+import re
+from typing import List
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class PrivacyRequestRedactionPatternsRequest(BaseModel):
+    """Request schema for updating privacy request redaction patterns."""
+
+    patterns: List[str] = Field(
+        description="List of regex patterns used to redact dataset, collection, and field names in privacy request package reports",
+        max_length=100,  # Limit number of patterns
+    )
+
+    @field_validator("patterns")
+    @classmethod
+    def validate_patterns(cls, patterns: List[str]) -> List[str]:
+        """Validate regex patterns with ReDoS protection via Pydantic's default rust-regex engine."""
+        if not patterns:
+            return patterns
+
+        validated_patterns = []
+        for i, pattern in enumerate(patterns):
+            if not isinstance(pattern, str):
+                raise ValueError(f"Pattern at index {i} must be a string")
+
+            pattern = pattern.strip()
+            if not pattern:
+                raise ValueError(f"Pattern at index {i} cannot be empty")
+
+            # Reasonable length limit for regex patterns
+            if len(pattern) > 500:
+                raise ValueError(
+                    f"Pattern at index {i} is too long (max 500 characters)"
+                )
+
+            # Pydantic's rust-regex engine provides ReDoS protection
+            try:
+                re.compile(pattern, re.IGNORECASE)
+            except re.error as e:
+                raise ValueError(f"Invalid regex pattern at index {i}: {e}")
+
+            validated_patterns.append(pattern)
+
+        return validated_patterns
+
+
+class PrivacyRequestRedactionPatternsResponse(BaseModel):
+    """Response schema for privacy request redaction patterns."""
+
+    patterns: List[str] = Field(
+        description="List of regex patterns used to redact dataset, collection, and field names in privacy request package reports"
+    )
+
+    model_config = {"from_attributes": True}

fides/api/service/privacy_request/dsr_package/dsr_data_preprocessor.py

@@ -0,0 +1,231 @@
+import re
+from typing import Any, Dict, List, Literal, Optional, Set, Tuple
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.models.privacy_request_redaction_pattern import (
+    PrivacyRequestRedactionPattern,
+)
+from fides.api.service.privacy_request.dsr_package.utils import (
+    get_redaction_entities_map_db,
+)
+
+
+class DSRDataPreprocessor:
+    """
+    Processes DSR data to apply name redaction before report generation.
+    """
+
+    def __init__(self, db: Session):
+        self.db = db
+        self.redaction_patterns: List[str] = (
+            PrivacyRequestRedactionPattern.get_patterns(db) or []
+        )
+        self.entities_to_redact: Set[str] = get_redaction_entities_map_db(db)
+
+    def process_dsr_data(self, dsr_data: dict[str, Any]) -> dict[str, Any]:
+        """Process the DSR data to apply all redaction upfront."""
+        if not self.redaction_patterns and not self.entities_to_redact:
+            return dsr_data
+
+        # First pass: collect and map dataset names
+        dataset_mapping = self._create_dataset_mapping(dsr_data)
+
+        # Second pass: process data with redaction
+        processed_data = {}
+        collection_indices: Dict[str, Dict[str, int]] = (
+            {}
+        )  # Track collection indices within each dataset
+
+        for key, rows in dsr_data.items():
+            # The "attachment" key is used to pass in the privacy request's attachments into `dsr_data`.
+            # We don't need to redact these.
+            if key == "attachments":
+                processed_data[key] = rows
+                continue
+
+            dataset_name, collection_name = self._parse_key(key, rows)
+
+            # Get redacted dataset name
+            redacted_dataset = dataset_mapping.get(dataset_name, dataset_name)
+
+            # Get redacted collection name (index per dataset)
+            if dataset_name not in collection_indices:
+                collection_indices[dataset_name] = {}
+
+            if collection_name not in collection_indices[dataset_name]:
+                collection_indices[dataset_name][collection_name] = (
+                    len(collection_indices[dataset_name]) + 1
+                )
+
+            collection_index = collection_indices[dataset_name][collection_name]
+            redacted_collection = self._redact_name(
+                "collection", collection_name, collection_index, dataset_name
+            )
+
+            # Process rows
+            new_key = f"{redacted_dataset}:{redacted_collection}"
+            processed_data[new_key] = [
+                self._process_row(row, dataset_name, collection_name) for row in rows
+            ]
+
+        return processed_data
+
+    def _create_dataset_mapping(self, dsr_data: dict[str, Any]) -> Dict[str, str]:
+        """Create dataset name mapping with ordered numbering for redacted datasets."""
+
+        # Extract unique dataset names in order of appearance
+        dataset_names = []
+        for key, rows in dsr_data.items():
+            if key not in ["attachments", "dataset"]:
+                dataset_name, _ = self._parse_key(key, rows)
+                dataset_names.append(dataset_name)
+
+        unique_datasets = list(dict.fromkeys(dataset_names))
+
+        # Create mapping using position-based numbering for redacted datasets
+        mapping = {}
+
+        # Handle regular datasets
+        for index, name in enumerate(unique_datasets, 1):
+            hierarchical_key = self._build_hierarchical_key("dataset", name)
+            if self._should_redact(name, hierarchical_key):
+                mapping[name] = f"dataset_{index}"
+            else:
+                mapping[name] = name  # Keep original name
+
+        # Special "dataset" and "attachment" cases comes last
+        # These keys are reserved for additional data and do not need to be redacted
+        if "dataset" in dsr_data.keys():
+            mapping["dataset"] = "dataset"
+
+        if "attachments" in dsr_data.keys():
+            mapping["attachments"] = "attachments"
+
+        return mapping
+
+    def _parse_key(self, key: str, rows: List[dict]) -> Tuple[str, str]:
+        """Parse a key into dataset and collection names."""
+        if ":" in key:
+            parts = key.split(":", 1)
+            return parts[0], parts[1]
+
+        # Fallback logic
+        for row in rows:
+            if "system_name" in row:
+                return row["system_name"], key
+        return "manual", key
+
+    def _should_redact(self, name: str, hierarchical_key: str) -> bool:
+        """Check if a name should be redacted."""
+        if hierarchical_key in self.entities_to_redact:
+            return True
+
+        for pattern in self.redaction_patterns:
+            try:
+                if re.search(pattern, name, re.IGNORECASE):
+                    return True
+            except re.error:
+                logger.warning(f"Invalid regex pattern: {pattern}")
+
+        return False
+
+    def _redact_name(
+        self,
+        name_type: Literal["dataset", "collection", "field"],
+        name: str,
+        index: int,
+        dataset_name: Optional[str] = None,
+        collection_name: Optional[str] = None,
+    ) -> str:
+        """Apply redaction to a name based on patterns and configurations."""
+        hierarchical_key = self._build_hierarchical_key(
+            name_type, name, dataset_name, collection_name
+        )
+
+        if self._should_redact(name, hierarchical_key):
+            return f"{name_type}_{index}"
+
+        return name
+
+    def _build_hierarchical_key(
+        self,
+        name_type: Literal["dataset", "collection", "field"],
+        name: str,
+        dataset_name: Optional[str] = None,
+        collection_name: Optional[str] = None,
+    ) -> str:
+        """Build hierarchical key for entity lookup."""
+        if name_type == "dataset":
+            return name
+        if name_type == "collection" and dataset_name:
+            return f"{dataset_name}.{name}"
+        if name_type == "field" and dataset_name and collection_name:
+            return f"{dataset_name}.{collection_name}.{name}"
+        return name
+
+    def _process_row(
+        self, row: dict[str, Any], dataset_name: str, collection_name: str
+    ) -> dict[str, Any]:
+        """Process a single row with field redaction."""
+        processed = {}
+
+        # Use enumerate for positional indexing (matches original)
+        for field_index, (field_name, value) in enumerate(row.items(), start=1):
+            redacted_field = self._redact_name(
+                "field", field_name, field_index, dataset_name, collection_name
+            )
+
+            # Process nested values
+            base_path = f"{dataset_name}.{collection_name}.{field_name}"
+            processed[redacted_field] = self._process_nested_value(value, base_path)
+
+        return processed
+
+    def _process_nested_value(
+        self,
+        value: Any,
+        current_path: str,
+        field_index_counter: Optional[Dict[str, Dict[str, int]]] = None,
+    ) -> Any:
+        """Recursively process nested values matching original logic."""
+        if field_index_counter is None:
+            field_index_counter = {}
+
+        if isinstance(value, dict):
+            processed = {}
+            level_key = f"{current_path}_fields"
+
+            if level_key not in field_index_counter:
+                field_index_counter[level_key] = {}
+
+            for field_name, field_value in value.items():
+                full_path = f"{current_path}.{field_name}"
+
+                # Get or create index for this field at this level
+                if field_name not in field_index_counter[level_key]:
+                    field_index_counter[level_key][field_name] = (
+                        len(field_index_counter[level_key]) + 1
+                    )
+
+                if self._should_redact(field_name, full_path):
+                    redacted_name = (
+                        f"field_{field_index_counter[level_key][field_name]}"
+                    )
+                else:
+                    redacted_name = field_name
+
+                processed[redacted_name] = self._process_nested_value(
+                    field_value, full_path, field_index_counter
+                )
+
+            return processed
+
+        if isinstance(value, list):
+            return [
+                self._process_nested_value(item, current_path, field_index_counter)
+                for item in value
+            ]
+
+        return value

fides/api/service/privacy_request/dsr_package/dsr_report_builder.py

@@ -5,13 +5,18 @@ import time as time_module
 import zipfile
 from io import BytesIO
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from typing import Any, Optional
 
 import jinja2
 from jinja2 import Environment, FileSystemLoader
 from loguru import logger
+from sqlalchemy.orm import object_session
 
-from fides.api.schemas.policy import ActionType
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.service.privacy_request.dsr_package.dsr_data_preprocessor import (
+    DSRDataPreprocessor,
+)
+from fides.api.service.privacy_request.dsr_package.utils import map_privacy_request
 from fides.api.service.storage.util import (
     _get_datasets_from_dsr_data,
     create_attachment_info_dict,
@@ -31,12 +36,9 @@ TEXT_COLOR = "#4A5568"
 HEADER_COLOR = "#FAFAFA"
 BORDER_COLOR = "#E2E8F0"
 
-if TYPE_CHECKING:
-    from fides.api.models.privacy_request import PrivacyRequest  # pragma: no cover
-
 
 # pylint: disable=too-many-instance-attributes
-class DsrReportBuilder:
+class DSRReportBuilder:
     """
     Manages populating HTML templates from the given data and adding the generated
     pages to a zip file in a way that the pages can be navigated between.
@@ -48,7 +50,7 @@ class DsrReportBuilder:
     - data/dataset_name/collection_name/item_index.html: the detail page for the item
     - attachments/index.html: the index page for the attachments
 
-    Args:
+        Args:
         privacy_request: the privacy request object
         dsr_data: the DSR data
     """
@@ -86,9 +88,18 @@ class DsrReportBuilder:
         }
         self.main_links: dict[str, Any] = {}  # used to track the generated pages
 
+        # Process the DSR data for redaction
+        db = object_session(privacy_request)
+        if db is not None:
+            processor = DSRDataPreprocessor(db)
+            processed_dsr_data = processor.process_dsr_data(dsr_data)
+        else:
+            # Fallback if no database session available
+            processed_dsr_data = dsr_data
+
         # report data to populate the templates
-        self.request_data = _map_privacy_request(privacy_request)
-        self.dsr_data = dsr_data
+        self.request_data = map_privacy_request(privacy_request)
+        self.dsr_data = processed_dsr_data
         self.enable_streaming = enable_streaming
 
         # Track used filenames per dataset to prevent conflicts within the same dataset
@@ -150,10 +161,6 @@ class DsrReportBuilder:
         """
         Generates a page for each collection in the dataset and an index page for the dataset.
         Tracks the generated links to build a root level index after each collection has been processed.
-
-        Args:
-            dataset_name: the name of the dataset to add
-            collections: the collections to add to the dataset
         """
         # track links to collection indexes
         collection_links = {}
@@ -325,19 +332,17 @@ class DsrReportBuilder:
         return corrected_attachment_info
 
     def _add_collection(
-        self, rows: list[dict[str, Any]], dataset_name: str, collection_name: str
+        self,
+        rows: list[dict[str, Any]],
+        dataset_name: str,
+        collection_name: str,
     ) -> None:
         """
         Adds a collection to the zip file.
-
-        Args:
-            rows: the rows to add to the collection
-            dataset_name: the name of the dataset to add the collection to
-            collection_name: the name of the collection to add
         """
         items_content = []
 
-        for index, collection_item in enumerate(rows, 1):
+        for item_index, collection_item in enumerate(rows, 1):
             # Create a deep copy of the item data to avoid modifying the original DSR data
             # This ensures the comprehensive attachments index can access unmodified attachments
             item_data = copy.deepcopy(collection_item)
@@ -377,11 +382,13 @@ class DsrReportBuilder:
                             # Replace the field value with processed attachment links
                             item_data[field_name] = attachment_links
 
-            # Add item content to the list
+            # Add item content to the list with item heading
+            # Field names are already redacted in the processed data
+            item_heading = f"{collection_name} (item #{item_index})"
             items_content.append(
                 {
-                    "index": index,
-                    "heading": f"{collection_name} (item #{index})",
+                    "index": item_index,
+                    "heading": item_heading,
                     "data": item_data,
                 }
             )
@@ -518,6 +525,7 @@ class DsrReportBuilder:
             # Add Additional Data if it exists
             if "dataset" in datasets:
                 self._add_dataset("dataset", datasets["dataset"])
+                # Use a more friendly name for the link but keep the dataset name for the path
                 self.main_links["Additional Data"] = "data/dataset/index.html"
 
             # Add comprehensive attachments index that includes ALL attachments
@@ -569,27 +577,3 @@ class DsrReportBuilder:
             "DSR report generation complete."
         )
         return self.baos
-
-
-def _map_privacy_request(privacy_request: "PrivacyRequest") -> dict[str, Any]:
-    """Creates a map with a subset of values from the privacy request"""
-    request_data: dict[str, Any] = {}
-    request_data["id"] = privacy_request.id
-
-    action_type: Optional[ActionType] = privacy_request.policy.get_action_type()
-    if action_type:
-        request_data["type"] = action_type.value
-
-    request_data["identity"] = {
-        key: value
-        for key, value in privacy_request.get_persisted_identity()
-        .labeled_dict(include_default_labels=True)
-        .items()
-        if value["value"] is not None
-    }
-
-    if privacy_request.requested_at:
-        request_data["requested_at"] = privacy_request.requested_at.strftime(
-            "%m/%d/%Y %H:%M %Z"
-        )
-    return request_data