ethyca-fides 2.69.1b1__py2.py3-none-any.whl → 2.69.1rc0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-29T09:50:25-0700",
+ "date": "2025-09-03T19:56:24+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "0ffb56145a84fca7b13bb51fa603c2d883250703",
- "version": "2.69.1b1"
+ "full-revisionid": "792b5cd4c5cb409b37e792476fd5b9d15be7ddb6",
+ "version": "2.69.1rc0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/78dbe23d8204_adding_privacy_request_redaction_patterns.py

@@ -0,0 +1,52 @@
+"""adding privacy requeste redaction patterns
+
+Revision ID: 78dbe23d8204
+Revises: b1a2c3d4e5f6
+Create Date: 2025-08-30 05:40:17.816172
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "78dbe23d8204"
+down_revision = "b1a2c3d4e5f6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "privacy_request_redaction_pattern",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("pattern", sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("pattern"),
+    )
+    op.create_index(
+        op.f("ix_privacy_request_redaction_pattern_id"),
+        "privacy_request_redaction_pattern",
+        ["id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_privacy_request_redaction_pattern_id"),
+        table_name="privacy_request_redaction_pattern",
+    )
+    op.drop_table("privacy_request_redaction_pattern")

fides/api/api/v1/api.py

@@ -17,6 +17,7 @@ from fides.api.api.v1.endpoints import (
     policy_webhook_endpoints,
     pre_approval_webhook_endpoints,
     privacy_request_endpoints,
+    privacy_request_redaction_patterns_endpoints,
     registration_endpoints,
     saas_config_endpoints,
     storage_endpoints,
@@ -41,6 +42,7 @@ api_router.include_router(policy_endpoints.router)
 api_router.include_router(policy_webhook_endpoints.router)
 api_router.include_router(pre_approval_webhook_endpoints.router)
 api_router.include_router(privacy_request_endpoints.router)
+api_router.include_router(privacy_request_redaction_patterns_endpoints.router)
 api_router.include_router(identity_verification_endpoints.router)
 api_router.include_router(storage_endpoints.router)
 api_router.include_router(messaging_endpoints.router)

fides/api/api/v1/endpoints/dsr_package_link.py

@@ -21,7 +21,7 @@ from fides.api.schemas.privacy_request import PrivacyRequestStatus
 from fides.api.schemas.storage.storage import StorageType
 from fides.api.service.storage.streaming.s3 import S3StorageClient
 from fides.api.util.api_router import APIRouter
-from fides.api.util.endpoint_utils import fides_limiter
+from fides.api.util.rate_limit import fides_limiter
 from fides.common.api.v1.urn_registry import PRIVACY_CENTER_DSR_PACKAGE, V1_URL_PREFIX
 from fides.config import CONFIG
 
@@ -62,7 +62,7 @@ def raise_error(status_code: int, detail: str) -> None:
     PRIVACY_CENTER_DSR_PACKAGE,
     status_code=HTTP_302_FOUND,
 )
-@fides_limiter.limit(CONFIG.security.public_request_rate_limit)
+@fides_limiter.limit(CONFIG.security.request_rate_limit)
 def get_access_results_urls(
     privacy_request_id: str,
     token: str,
@@ -155,7 +155,9 @@ def get_access_results_urls(
 
     try:
         # Use S3StorageClient for cleaner presigned URL generation
-        s3_storage_client = S3StorageClient(storage_config.secrets)
+        s3_storage_client = S3StorageClient(
+            storage_config.details.get("auth_method"), storage_config.secrets
+        )
         result_url = s3_storage_client.generate_presigned_url(
             bucket=bucket_name,
             key=file_name,

fides/api/api/v1/endpoints/oauth_endpoints.py

@@ -8,8 +8,8 @@ from loguru import logger
 from sqlalchemy.orm import Session
 from starlette.status import (
     HTTP_400_BAD_REQUEST,
+    HTTP_403_FORBIDDEN,
     HTTP_404_NOT_FOUND,
-    HTTP_422_UNPROCESSABLE_ENTITY,
 )
 
 from fides.api.api.deps import get_db
@@ -26,7 +26,7 @@ from fides.api.models.client import ClientDetail
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.fides_user import FidesUser
 from fides.api.oauth.roles import ROLES_TO_SCOPES_MAPPING
-from fides.api.oauth.utils import verify_oauth_client
+from fides.api.oauth.utils import verify_client_can_assign_scopes, verify_oauth_client
 from fides.api.schemas.client import ClientCreatedResponse
 from fides.api.schemas.oauth import AccessToken, OAuth2ClientCredentialsRequestForm
 from fides.api.service.authentication.authentication_strategy import (
@@ -37,7 +37,7 @@ from fides.api.service.authentication.authentication_strategy_oauth2_authorizati
 )
 from fides.api.util.api_router import APIRouter
 from fides.api.util.connection_util import connection_status
-from fides.api.util.endpoint_utils import fides_limiter
+from fides.api.util.rate_limit import fides_limiter
 from fides.common.api.scope_registry import (
     CLIENT_CREATE,
     CLIENT_DELETE,
@@ -123,22 +123,28 @@ async def acquire_access_token(
 
 @router.post(
     CLIENT,
-    dependencies=[Security(verify_oauth_client, scopes=[CLIENT_CREATE])],
     response_model=ClientCreatedResponse,
 )
 def create_client(
     *,
+    request: Request,
     db: Session = Depends(get_db),
     scopes: List[str] = Body([]),
+    requesting_client: ClientDetail = Security(
+        verify_oauth_client, scopes=[CLIENT_CREATE]
+    ),
 ) -> ClientCreatedResponse:
     """Creates a new client and returns the credentials. Only direct scopes can be added to the client via this endpoint."""
     logger.info("Creating new client")
     if not all(scope in SCOPE_REGISTRY for scope in scopes):
         raise HTTPException(
-            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            status_code=HTTP_403_FORBIDDEN,
             detail=f"Invalid Scope. Scopes must be one of {SCOPE_REGISTRY}.",
         )
 
+    # Security check: Verify that the requesting client has all the scopes they're trying to assign
+    verify_client_can_assign_scopes(request, requesting_client, scopes, db)
+
     client, secret = ClientDetail.create_client_and_secret(
         db,
         CONFIG.security.oauth_client_id_length_bytes,
@@ -180,13 +186,16 @@ def get_client_scopes(client_id: str, db: Session = Depends(get_db)) -> List[str
 
 @router.put(
     CLIENT_SCOPE,
-    dependencies=[Security(verify_oauth_client, scopes=[CLIENT_UPDATE])],
     response_model=None,
 )
 def set_client_scopes(
     client_id: str,
     scopes: List[str],
+    request: Request,
     db: Session = Depends(get_db),
+    requesting_client: ClientDetail = Security(
+        verify_oauth_client, scopes=[CLIENT_UPDATE]
+    ),
 ) -> None:
     """Overwrites the client's directly-assigned scopes with those provided.
     Roles cannot be edited via this endpoint.
@@ -197,10 +206,13 @@ def set_client_scopes(
 
     if not all(elem in SCOPE_REGISTRY for elem in scopes):
         raise HTTPException(
-            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            status_code=HTTP_403_FORBIDDEN,
             detail=f"Invalid Scope. Scopes must be one of {SCOPE_REGISTRY}.",
         )
 
+    # Security check: Verify that the requesting client has all the scopes they're trying to assign
+    verify_client_can_assign_scopes(request, requesting_client, scopes, db)
+
     logger.info("Updating client scopes")
     client.update(db, data={"scopes": scopes})
 

fides/api/api/v1/endpoints/privacy_request_redaction_patterns_endpoints.py

@@ -0,0 +1,95 @@
+from fastapi import Depends, HTTPException, Security
+from fastapi.routing import APIRouter
+from loguru import logger
+from sqlalchemy.orm import Session
+from starlette.status import HTTP_200_OK
+
+from fides.api.api.deps import get_db
+from fides.api.models.privacy_request_redaction_pattern import (
+    PrivacyRequestRedactionPattern,
+)
+from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.privacy_request_redaction_patterns import (
+    PrivacyRequestRedactionPatternsRequest,
+    PrivacyRequestRedactionPatternsResponse,
+)
+from fides.common.api.scope_registry import (
+    PRIVACY_REQUEST_REDACTION_PATTERNS_READ,
+    PRIVACY_REQUEST_REDACTION_PATTERNS_UPDATE,
+)
+from fides.common.api.v1.urn_registry import (
+    PRIVACY_REQUEST_REDACTION_PATTERNS,
+    V1_URL_PREFIX,
+)
+
+router = APIRouter(
+    tags=["Privacy Request Redaction Patterns"],
+    prefix=V1_URL_PREFIX,
+)
+
+
+@router.get(
+    PRIVACY_REQUEST_REDACTION_PATTERNS,
+    status_code=HTTP_200_OK,
+    response_model=PrivacyRequestRedactionPatternsResponse,
+    dependencies=[
+        Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_REDACTION_PATTERNS_READ])
+    ],
+)
+def get_privacy_request_redaction_patterns(
+    *, db: Session = Depends(get_db)
+) -> PrivacyRequestRedactionPatternsResponse:
+    """
+    Get the current privacy request redaction patterns configuration.
+
+    Returns the list of regex patterns used to mask dataset, collection,
+    and field names in privacy request package reports.
+    """
+    logger.info("Getting privacy request redaction patterns configuration")
+
+    patterns = PrivacyRequestRedactionPattern.get_patterns(db)
+    return PrivacyRequestRedactionPatternsResponse(patterns=patterns)
+
+
+@router.put(
+    PRIVACY_REQUEST_REDACTION_PATTERNS,
+    status_code=HTTP_200_OK,
+    response_model=PrivacyRequestRedactionPatternsResponse,
+    dependencies=[
+        Security(
+            verify_oauth_client, scopes=[PRIVACY_REQUEST_REDACTION_PATTERNS_UPDATE]
+        )
+    ],
+)
+def update_privacy_request_redaction_patterns(
+    *,
+    db: Session = Depends(get_db),
+    request: PrivacyRequestRedactionPatternsRequest,
+) -> PrivacyRequestRedactionPatternsResponse:
+    """
+    Update the privacy request redaction patterns configuration.
+
+    This is a complete replacement of the patterns list. To clear all patterns,
+    send an empty list.
+    """
+    logger.info(
+        "Updating privacy request redaction patterns configuration with {} patterns",
+        len(request.patterns),
+    )
+
+    try:
+        updated = PrivacyRequestRedactionPattern.replace_patterns(
+            db=db, patterns=request.patterns
+        )
+
+        logger.info("Successfully updated privacy request redaction patterns")
+        return PrivacyRequestRedactionPatternsResponse(patterns=updated)
+
+    except Exception as exc:
+        logger.exception(
+            "Failed to update privacy request redaction patterns: {}", str(exc)
+        )
+        raise HTTPException(
+            status_code=500,
+            detail="Failed to update privacy request redaction patterns",
+        ) from exc

fides/api/api/v1/endpoints/user_endpoints.py

@@ -59,7 +59,7 @@ from fides.api.schemas.user import (
 )
 from fides.api.service.deps import get_user_service
 from fides.api.util.api_router import APIRouter
-from fides.api.util.endpoint_utils import fides_limiter
+from fides.api.util.rate_limit import fides_limiter
 from fides.common.api.scope_registry import (
     SCOPE_REGISTRY,
     SYSTEM_MANAGER_DELETE,
@@ -117,7 +117,7 @@ def verify_user_read_scopes(
     db: Session = Depends(get_db),
 ) -> ClientDetail:
     """
-    Custom dependency that verifies the user has either user:read or user:read-own scope.
+    Custom dependency that verifies the user has either USER_READ or USER_READ_OWN scope.
     Returns the client if authorized.
     """
     token_data, client = extract_token_and_load_client(authorization, db)
@@ -159,7 +159,7 @@ async def update_user(
 ) -> FidesUser:
     """
     Update a user given a `user_id`. If the user is not updating their own data,
-    they need the user:update scope
+    they need the USER_UPDATE scope
     """
     user = FidesUser.get(db=db, object_id=user_id)
     if not user:
@@ -208,6 +208,17 @@ def update_user_password(
 
     current_user.update_password(db=db, new_password=data.new_password)
 
+    # Delete the user's associated OAuth client to invalidate all existing sessions
+    if current_user.client:
+        try:
+            current_user.client.delete(db)
+        except Exception as exc:
+            logger.exception(
+                "Unable to delete user client during password reset for user {}: {}",
+                current_user.id,
+                exc,
+            )
+
     logger.info("Updated user with id: '{}'.", current_user.id)
     return current_user
 
@@ -236,6 +247,18 @@ def force_update_password(
         )
 
     user.update_password(db=db, new_password=data.new_password)
+
+    # Delete the user's associated OAuth client to invalidate all existing sessions
+    if user.client:
+        try:
+            user.client.delete(db)
+        except Exception as exc:
+            logger.exception(
+                "Unable to delete user client during admin-forced password reset for user {}: {}",
+                user.id,
+                exc,
+            )
+
     logger.info("Updated user with id: '{}'.", user.id)
     return user
 
@@ -548,7 +571,7 @@ def get_user(
     client: ClientDetail = Security(verify_user_read_scopes),
     authorization: str = Security(oauth2_scheme),
 ) -> FidesUser:
-    """Returns a User based on an Id. Users with user:read-own scope can only access their own data. Users with user:read can access other's data."""
+    """Returns a User based on an Id. Users with USER_READ_OWN scope can only access their own data."""
     user: Optional[FidesUser] = FidesUser.get_by_key_or_id(db, data={"id": user_id})
     if user is None:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="User not found")

fides/api/app_setup.py

@@ -48,9 +48,13 @@ from fides.api.service.saas_request.override_implementations import *
 from fides.api.util.api_router import APIRouter
 from fides.api.util.cache import get_cache
 from fides.api.util.consent_util import create_default_tcf_purpose_overrides_on_startup
-from fides.api.util.endpoint_utils import fides_limiter
 from fides.api.util.errors import FidesError
 from fides.api.util.logger import setup as setup_logging
+from fides.api.util.rate_limit import (
+    RateLimitIPValidationMiddleware,
+    fides_limiter,
+    is_rate_limit_enabled,
+)
 from fides.config import CONFIG
 from fides.config.config_proxy import ConfigProxy
 
@@ -88,7 +92,17 @@ def create_fides_app(
     for handler in ExceptionHandlers.get_handlers():
         # Starlette bug causing this to fail mypy
         fastapi_app.add_exception_handler(RedisNotConfigured, handler)  # type: ignore
-    fastapi_app.add_middleware(SlowAPIMiddleware)
+
+    if is_rate_limit_enabled:
+        # Validate header before SlowAPI processes the request
+        fastapi_app.add_middleware(RateLimitIPValidationMiddleware)
+        # Required for default rate limiting to work
+        fastapi_app.add_middleware(SlowAPIMiddleware)
+    else:
+        logger.warning(
+            "Rate limiting client IPs is disabled because the FIDES__SECURITY__RATE_LIMIT_CLIENT_IP_HEADER env var is not configured."
+        )
+
     fastapi_app.add_middleware(
         GZipMiddleware, minimum_size=1000, compresslevel=5
     )  # minimum_size is in bytes

fides/api/db/base.py

@@ -66,6 +66,9 @@ from fides.api.models.privacy_preference import (
     ServedNoticeHistory,
 )
 from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.models.privacy_request_redaction_pattern import (
+    PrivacyRequestRedactionPattern,
+)
 from fides.api.models.property import (
     MessagingTemplateToProperty,
     PrivacyExperienceConfigProperty,

fides/api/main.py

@@ -18,6 +18,8 @@ from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
 from fideslog.sdk.python.event import AnalyticsEvent
 from loguru import logger
 from pyinstrument import Profiler
+from slowapi import _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
 from starlette.background import BackgroundTask
 from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
 from uvicorn import Config, Server
@@ -60,6 +62,7 @@ from fides.api.ui import (
 )
 from fides.api.util.endpoint_utils import API_PREFIX
 from fides.api.util.logger import _log_exception
+from fides.api.util.rate_limit import safe_rate_limit_key
 from fides.cli.utils import FIDES_ASCII_ART
 from fides.config import CONFIG, check_required_webserver_config_values
 
@@ -388,3 +391,22 @@ async def request_validation_exception_handler(
             "detail": jsonable_encoder(exc.errors(), exclude={"input", "url", "ctx"})
         },
     )
+
+
+@app.exception_handler(RateLimitExceeded)
+async def rate_limit_handler(request: Request, exc: RateLimitExceeded) -> Response:
+    """Log rate limit violations and delegate to default handler."""
+    client_ip = safe_rate_limit_key(
+        request
+    )  # non exception-raising, falls back to source IP
+
+    # Log the rate limit event
+    logger.warning(
+        "Rate limit exceeded - IP: %s, Path: %s, Method: %s",
+        client_ip,
+        request.url.path,
+        request.method,
+    )
+
+    # Use the default handler to generate the proper response
+    return _rate_limit_exceeded_handler(request, exc)

fides/api/models/client.py

@@ -51,6 +51,7 @@ class ClientDetail(Base):
     user_id = Column(
         String, ForeignKey(FidesUser.id_field_path), nullable=True, unique=True
     )
+    user: Optional["FidesUser"]
 
     @classmethod
     def create_client_and_secret(

fides/api/models/privacy_request_redaction_pattern.py

@@ -0,0 +1,64 @@
+from typing import List, Set
+
+from sqlalchemy import Column, String
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session
+
+from fides.api.db.base_class import Base
+
+
+class PrivacyRequestRedactionPattern(Base):
+    """
+    Stores one regex pattern per row for masking dataset, collection, and field names
+    in DSR package reports.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "privacy_request_redaction_pattern"
+
+    # One pattern per row; unique to prevent duplicates
+    pattern = Column(String, nullable=False, unique=True)
+
+    @classmethod
+    def get_patterns(cls, db: Session) -> List[str]:
+        """
+        Get the current list of masking patterns ordered by creation time.
+
+        Returns:
+            List of regex pattern strings, or None if no patterns are configured
+        """
+        rows = db.query(cls).order_by(cls.created_at.asc()).all()
+        if not rows:
+            return []
+        return [row.pattern for row in rows]
+
+    @classmethod
+    def replace_patterns(cls, db: Session, patterns: List[str]) -> List[str]:
+        """
+        Replace the set of stored patterns with the provided list using set reconciliation.
+
+        - Adds patterns that are not present
+        - Removes patterns that are no longer desired
+        - Returns the resulting canonical list in deterministic order
+        """
+        # Normalize: trim whitespace, remove empties and duplicates
+        desired: Set[str] = {p.strip() for p in patterns if p and p.strip()}
+
+        # Fetch existing patterns from DB as a set
+        existing: Set[str] = set(p for (p,) in db.query(cls.pattern).all())
+
+        to_add = desired - existing
+        to_remove = existing - desired
+
+        if to_remove:
+            db.query(cls).filter(cls.pattern.in_(list(to_remove))).delete(
+                synchronize_session=False
+            )
+
+        if to_add:
+            db.bulk_save_objects([cls(pattern=p) for p in to_add])
+
+        db.commit()
+
+        return sorted(desired)