ethyca-fides 2.64.6b1__py2.py3-none-any.whl → 2.65.0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-02T08:50:36-0700",
+ "date": "2025-07-08T13:15:47-0300",
  "dirty": false,
  "error": null,
- "full-revisionid": "5b2b01716a352fdf042c622bf1b996a7490b400b",
- "version": "2.64.6b1"
+ "full-revisionid": "34ee2ee50c7518572e66a4441078263e59ba4030",
+ "version": "2.65.0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/641f6bcd424f_add_user_id_to_manual_task_log.py

@@ -0,0 +1,28 @@
+"""add user_id to manual_task_log
+
+Revision ID: 641f6bcd424f
+Revises: 41a634d8c669
+Create Date: 2025-07-02 03:19:40.171201
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "641f6bcd424f"
+down_revision = "41a634d8c669"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("manual_task_log", sa.Column("user_id", sa.String(), nullable=True))
+    op.create_index(
+        op.f("ix_manual_task_log_user_id"), "manual_task_log", ["user_id"], unique=False
+    )
+
+
+def downgrade():
+    op.drop_index(op.f("ix_manual_task_log_user_id"), table_name="manual_task_log")
+    op.drop_column("manual_task_log", "user_id")

fides/api/api/v1/endpoints/generic_overrides.py

@@ -1,6 +1,8 @@
+from functools import partial
 from typing import Dict, List, Optional, Type, Union
 
 from fastapi import Depends, HTTPException, Query, Response, Security
+from fastapi.concurrency import run_in_threadpool
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 from fastapi_pagination import Page, Params
@@ -224,9 +226,17 @@ async def list_dataset_paginated(
 
     pagination_params = Params(page=page or 1, size=size or 50)
     results = await async_paginate(db, filtered_query, pagination_params)
-    results.items = [  # type: ignore[attr-defined]
-        DatasetResponse.model_validate(result.__dict__) for result in results.items  # type: ignore[attr-defined]
-    ]
+
+    validated_items = []
+    for result in results.items:  # type: ignore[attr-defined]
+        # run pydantic validation in a threadpool to avoid blocking the main thread
+        validated_item = await run_in_threadpool(
+            partial(DatasetResponse.model_validate, result.__dict__)
+        )
+        validated_items.append(validated_item)
+
+    results.items = validated_items  # type: ignore[attr-defined]
+
     return results
 
 

fides/api/email_templates/get_email_template.py

@@ -8,6 +8,7 @@ from fides.api.email_templates.template_names import (
     CONSENT_REQUEST_EMAIL_FULFILLMENT,
     CONSENT_REQUEST_VERIFICATION_TEMPLATE,
     EMAIL_ERASURE_REQUEST_FULFILLMENT,
+    EXTERNAL_USER_WELCOME,
     PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE,
     PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE,
     PRIVACY_REQUEST_ERROR_NOTIFICATION_TEMPLATE,
@@ -29,7 +30,7 @@ template_env = Environment(
 )
 
 
-def get_email_template(  # pylint: disable=too-many-return-statements
+def get_email_template(  # pylint: disable=too-many-return-statements, too-many-branches
     action_type: MessagingActionType,
 ) -> Template:
     if action_type == MessagingActionType.CONSENT_REQUEST:
@@ -56,6 +57,8 @@ def get_email_template(  # pylint: disable=too-many-return-statements
         return template_env.get_template(TEST_MESSAGE_TEMPLATE)
     if action_type == MessagingActionType.USER_INVITE:
         return template_env.get_template(USER_INVITE)
+    if action_type == MessagingActionType.EXTERNAL_USER_WELCOME:
+        return template_env.get_template(EXTERNAL_USER_WELCOME)
 
     logger.error("No corresponding template linked to the {}", action_type)
     raise EmailTemplateUnhandledActionType(

fides/api/email_templates/template_names.py

@@ -10,3 +10,4 @@ PRIVACY_REQUEST_REVIEW_DENY_TEMPLATE = "privacy_request_review_deny.html"
 PRIVACY_REQUEST_REVIEW_APPROVE_TEMPLATE = "privacy_request_review_approve.html"
 TEST_MESSAGE_TEMPLATE = "test_message.html"
 USER_INVITE = "user_invite.html"
+EXTERNAL_USER_WELCOME = "external_user_welcome.html"

fides/api/email_templates/templates/external_user_welcome.html

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Welcome to Privacy Center</title>
+</head>
+<body>
+<main>
+    <p>Hello {{ display_name }},</p>
+
+    <p>An account has been created for you by the privacy team at {{ org_name }} so that you can assist them in responding to data subject requests, as required by the GDPR and many US state laws, for users that have data in your systems.</p>
+
+    <p>A portal is available to view and take action on these requests at: <a href="{{ portal_link }}">{{ portal_link }}</a>. Through this portal you can upload or provide user data for users who have requested a copy of their information and attest that users have been removed from your system in the case of an erasure request.</p>
+
+    <p>The provided link is valid for 30 days and you will be asked to verify your email address each time you access the portal.</p>
+
+    <p>If you have any questions, please reach out first to your contacts at: {{ org_name }}</p>
+
+    <p>Regards,<br/>
+    {{ org_name }} Privacy Team</p>
+</main>
+</body>
+</html> 

fides/api/models/fides_user_respondent_email_verification.py

@@ -19,16 +19,16 @@ CONFIG = get_config()
 if TYPE_CHECKING:
     from fides.api.models.fides_user import FidesUser
 
-# Access links stay active for 45 days - the same as the DSR expiration. A new link is generated for each email.
+# Access links stay active for 30 days for external login functionality. A new link is generated for each email.
 # The emails are created for new DSRs which are assigned to the respondent.
-ACCESS_LINK_TTL_DAYS = 45
+ACCESS_LINK_TTL_DAYS = 30
 
 
 class FidesUserRespondentEmailVerification(Base, IdentityVerificationMixin):
     """Model for handling email verification for external respondents.
 
     This handles two types of verification:
-    1. Access links - long-lived (45 days) for initial access
+    1. Access links - long-lived (30 days) for initial access
     2. Verification codes - short-lived (1 hour) for actual verification
 
     When an email is sent to an external respondent, a new verification is created with a new access token is created.

fides/api/models/manual_task.py

@@ -20,6 +20,7 @@ from sqlalchemy.sql import func
 
 from fides.api.db.base_class import Base, FidesBase
 from fides.api.db.util import EnumColumn
+from fides.api.request_context import get_user_id
 from fides.api.schemas.base_class import FidesSchema
 
 if TYPE_CHECKING:
@@ -880,6 +881,10 @@ class ManualTaskLog(Base):
         ForeignKey("manual_task_instance.id", ondelete="CASCADE"),
         nullable=True,
     )
+    # The user responsible for the action being logged.  This may be `None`
+    # for system-initiated events or for legacy records created before this
+    # column existed.
+    user_id = Column(String, nullable=True, index=True)
     status = Column(String, nullable=False)
     message = Column(String, nullable=True)
     details = Column(JSONB, nullable=True)
@@ -890,6 +895,7 @@ class ManualTaskLog(Base):
         Index("ix_manual_task_log_instance_id", "instance_id"),
         Index("ix_manual_task_log_status", "status"),
         Index("ix_manual_task_log_task_id", "task_id"),
+        Index("ix_manual_task_log_user_id", "user_id"),
     )
 
     # Relationships
@@ -908,6 +914,7 @@ class ManualTaskLog(Base):
         task_id: str,
         status: "ManualTaskLogStatus",
         message: str,
+        user_id: Optional[str] = None,
         config_id: Optional[str] = None,
         instance_id: Optional[str] = None,
         details: Optional[dict[str, Any]] = None,
@@ -921,10 +928,12 @@ class ManualTaskLog(Base):
             message: Optional message describing the event
             details: Optional additional details about the event
         """
+
         data = {
             "task_id": task_id,
             "config_id": config_id,
             "instance_id": instance_id,
+            "user_id": user_id or get_user_id(),
             "status": status,
             "message": message,
             "details": details,
@@ -937,6 +946,7 @@ class ManualTaskLog(Base):
         db: Session,
         task_id: str,
         message: str,
+        user_id: Optional[str] = None,
         config_id: Optional[str] = None,
         instance_id: Optional[str] = None,
         details: Optional[dict[str, Any]] = None,
@@ -954,12 +964,14 @@ class ManualTaskLog(Base):
         Returns:
             The created error log entry
         """
+
         return cls.create_log(
             db=db,
             status=ManualTaskLogStatus.error,
             task_id=task_id,
             config_id=config_id,
             instance_id=instance_id,
+            user_id=user_id or get_user_id(),
             message=message,
             details=details,
         )

fides/api/oauth/utils.py

@@ -31,6 +31,7 @@ from fides.api.models.policy import PolicyPreWebhook
 from fides.api.models.pre_approval_webhook import PreApprovalWebhook
 from fides.api.models.privacy_request import RequestTask
 from fides.api.oauth.roles import get_scopes_from_roles
+from fides.api.request_context import set_user_id
 from fides.api.schemas.external_https import RequestTaskJWE, WebhookJWE
 from fides.api.schemas.oauth import OAuth2ClientCredentialsBearer
 from fides.common.api.v1.urn_registry import TOKEN, V1_URL_PREFIX
@@ -325,6 +326,16 @@ def extract_token_and_load_client(
         logger.debug("Auth token belongs to an invalid client_id.")
         raise AuthorizationError(detail="Not Authorized for this action")
 
+    # Populate request-scoped context with the authenticated user identifier.
+    # Prefer the linked user_id; fall back to the client id when this is the
+    # special root client (which has no associated FidesUser row).
+    ctx_user_id = client.user_id
+    if not ctx_user_id and client.id == CONFIG.security.oauth_root_client_id:
+        ctx_user_id = CONFIG.security.oauth_root_client_id
+
+    if ctx_user_id:
+        set_user_id(ctx_user_id)
+
     return token_data, client
 
 

fides/api/request_context.py

@@ -0,0 +1,76 @@
+"""
+Utilities for storing and accessing request-scoped context that must be
+accessible across the entire application stack (endpoints -> services ->
+helpers -> decorators) without having to thread additional parameters through
+all call-sites.
+
+Currently we only capture the authenticated `user_id` but additional fields
+(e.g. correlation_id, locale, feature_flags) can be added in the future by
+expanding the `RequestContext` dataclass.
+
+A `contextvars.ContextVar` is used instead of a module-level global to ensure
+that values are local to the current execution context (async task, thread or
+Celery worker) and therefore safe for concurrent workloads.
+"""
+
+from __future__ import annotations
+
+import contextvars
+from dataclasses import dataclass
+from typing import Any, Optional
+
+__all__ = [
+    "get_user_id",
+    "set_user_id",
+    "reset_request_context",
+]
+
+
+@dataclass
+class RequestContext:
+    user_id: Optional[str] = None
+
+
+# A single ContextVar holding the current request context.
+_ctx: contextvars.ContextVar[RequestContext] = contextvars.ContextVar(
+    "request_context",
+    default=RequestContext(),
+)
+
+
+def get_request_context() -> RequestContext:
+    """Return the current `RequestContext` for this request.
+
+    - The context is populated during authentication by `extract_token_and_load_client()`,
+      which calls `fides.api.request_context.set_request_context` with
+      the authenticated `user_id`.
+    - A `ContextVar` keeps the data isolated per request. Each coroutine
+      or thread sees its own copy and it is discarded at the end of the request.
+    - The returned object is the live context; treat it as read-only and use
+      `set_request_context` to mutate values.
+    """
+    return _ctx.get()
+
+
+def set_request_context(**kwargs: Any) -> None:
+    """Mutate the current context in place using `key=value` pairs."""
+    ctx = _ctx.get()
+    for key, value in kwargs.items():
+        if hasattr(ctx, key):
+            setattr(ctx, key, value)
+
+
+def reset_request_context() -> None:
+    """Remove all context, mainly for test clean-up."""
+    _ctx.set(RequestContext())
+
+
+def get_user_id() -> Optional[str]:
+    """Return the user_id from the current request context."""
+    ctx = get_request_context()
+    return ctx.user_id
+
+
+def set_user_id(user_id: str) -> None:
+    """Set the user_id in the current request context."""
+    set_request_context(user_id=user_id)

fides/api/schemas/messaging/messaging.py

@@ -69,6 +69,7 @@ class MessagingActionType(str, Enum):
     PRIVACY_REQUEST_REVIEW_DENY = "privacy_request_review_deny"
     PRIVACY_REQUEST_REVIEW_APPROVE = "privacy_request_review_approve"
     USER_INVITE = "user_invite"
+    EXTERNAL_USER_WELCOME = "external_user_welcome"
     TEST_MESSAGE = "test_message"
 
 
@@ -182,6 +183,17 @@ class UserInviteBodyParams(BaseModel):
     invite_code: str
 
 
+class ExternalUserWelcomeBodyParams(BaseModel):
+    """Body params required to send a welcome email to external users"""
+
+    username: str
+    first_name: Optional[str] = None
+    last_name: Optional[str] = None
+    privacy_center_url: str
+    access_token: str
+    org_name: str
+
+
 class FidesopsMessage(
     BaseModel,
 ):
@@ -198,6 +210,7 @@ class FidesopsMessage(
             ErasureRequestBodyParams,
             ErrorNotificationBodyParams,
             UserInviteBodyParams,
+            ExternalUserWelcomeBodyParams,
         ]
     ] = None
 
@@ -217,6 +230,7 @@ class FidesopsMessage(
             MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT: ErasureRequestBodyParams,
             MessagingActionType.PRIVACY_REQUEST_ERROR_NOTIFICATION: ErrorNotificationBodyParams,
             MessagingActionType.USER_INVITE: UserInviteBodyParams,
+            MessagingActionType.EXTERNAL_USER_WELCOME: ExternalUserWelcomeBodyParams,
         }
 
         valid_body_params = valid_body_params_for_action_type.get(

fides/api/schemas/saas/strategy_configuration.py

@@ -30,6 +30,12 @@ class FilterPostProcessorConfiguration(StrategyConfiguration):
     case_sensitive: bool = True
 
 
+class ExtractForExecutionLogPostProcessorConfiguration(StrategyConfiguration):
+    """Configuration for extracting data from response body and adding to execution log messages"""
+
+    path: Optional[str] = None
+
+
 class OffsetPaginationConfiguration(StrategyConfiguration):
     """
     Increases the value of the query param `incremental_param` by the `increment_by` until the `limit` is hit

fides/api/service/connectors/saas_connector.py

@@ -423,6 +423,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             response_data,
             identity_data,
             cast(Optional[List[PostProcessorStrategy]], saas_request.postprocessors),
+            response,
         )
 
         logger.info(
@@ -456,6 +457,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         response_data: Union[List[Dict[str, Any]], Dict[str, Any]],
         identity_data: Dict[str, Any],
         postprocessors: Optional[List[PostProcessorStrategy]],
+        response: Optional[Response] = None,
     ) -> List[Row]:
         """
         Runs the raw response through all available postprocessors for the request,
@@ -481,7 +483,9 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                     processed_data,
                     identity_data,
                     privacy_request,
+                    response,
                 )
+
             except Exception as exc:
                 raise PostProcessingException(
                     f"Exception occurred during the '{postprocessor.strategy}' postprocessor "  # type: ignore
@@ -567,6 +571,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             rows,
             privacy_request.get_cached_identity_data(),
             cast(Optional[List[PostProcessorStrategy]], masking_request.postprocessors),
+            None,
         )
 
         client = self.create_client()
@@ -584,9 +589,46 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                     )
                     continue
                 raise exc
-            client.send(prepared_request, masking_request.ignore_errors)
+            response = client.send(prepared_request, masking_request.ignore_errors)
             rows_updated += 1
 
+            # Run post-processors against the response from the masking request so that
+            # processors such as `extract_for_execution_log` can inspect the API
+            # response body (e.g. confirmation, ticket IDs, etc.).  We ignore the
+            # returned rows because masking responses are not used downstream.
+            try:
+                handled_response = self._handle_errored_response(
+                    masking_request, response
+                )
+                response_data = self._unwrap_response_data(
+                    masking_request, handled_response
+                )
+
+                # Only attempt post-processing if we have post-processors and the response body
+                # is JSON-serializable (dict or list of dicts).
+                if masking_request.postprocessors and isinstance(
+                    response_data, (dict, list)
+                ):
+                    self.process_response_data(
+                        response_data,
+                        privacy_request.get_cached_identity_data(),
+                        cast(
+                            Optional[List[PostProcessorStrategy]],
+                            masking_request.postprocessors,
+                        ),
+                        handled_response,
+                    )
+            except (
+                PostProcessingException,
+                Exception,
+            ) as exc:  # pylint: disable=broad-except
+                # We do not want a post-processing failure to prevent the masking
+                # operation itself from succeeding.
+                logger.warning(
+                    "Post-processing of masking request response failed: {}",
+                    exc,
+                )
+
         self.unset_connector_state()
 
         awaiting_async_callback: bool = bool(

fides/api/service/execution_context.py

@@ -0,0 +1,59 @@
+"""
+Execution context for collecting log messages during privacy request processing.
+
+Uses Python's contextvars for thread-safe, execution-scoped message collection.
+"""
+
+import contextvars
+from contextlib import contextmanager
+from typing import Generator, List, Optional
+
+from loguru import logger
+
+# Context variable for current execution messages
+_execution_messages: contextvars.ContextVar[Optional[List[str]]] = (
+    contextvars.ContextVar("execution_messages")
+)
+
+
+@contextmanager
+def collect_execution_log_messages() -> Generator[List[str], None, None]:
+    """
+    Context manager for collecting execution log messages.
+
+    Usage:
+        with collect_execution_log_messages() as messages:
+            # ... do work that calls add_execution_log_message()
+            pass
+        # messages now contains all collected messages
+
+    Returns:
+        List[str]: Messages collected during execution
+    """
+    messages: List[str] = []
+    token = _execution_messages.set(messages)
+    try:
+        yield messages
+    finally:
+        _execution_messages.reset(token)
+
+
+def add_execution_log_message(message: str) -> None:
+    """Add a message to the current execution context"""
+    try:
+        messages = _execution_messages.get()
+        if messages is not None and message.strip():
+            messages.append(message.strip())
+    except LookupError:
+        logger.error(
+            f"Attempted to call add_execution_log_message outside of execution context, unable to add message: {message}"
+        )
+
+
+def get_execution_log_messages() -> List[str]:
+    """Get messages from current execution context (for testing/debugging)"""
+    try:
+        messages = _execution_messages.get()
+        return messages.copy() if messages is not None else []
+    except LookupError:
+        return []

fides/api/service/messaging/message_dispatch_service.py

@@ -26,6 +26,7 @@ from fides.api.schemas.messaging.messaging import (
     EmailForActionType,
     ErasureRequestBodyParams,
     ErrorNotificationBodyParams,
+    ExternalUserWelcomeBodyParams,
     FidesopsMessage,
     MessagingActionType,
     MessagingMethod,
@@ -176,6 +177,7 @@ def dispatch_message(
             ErasureRequestBodyParams,
             UserInviteBodyParams,
             ErrorNotificationBodyParams,
+            ExternalUserWelcomeBodyParams,
         ]
     ] = None,
     subject_override: Optional[str] = None,
@@ -351,7 +353,7 @@ def _render(template_str: str, variables: Optional[Dict] = None) -> str:
     return template_str
 
 
-def _build_email(  # pylint: disable=too-many-return-statements
+def _build_email(  # pylint: disable=too-many-return-statements, too-many-branches
     config_proxy: ConfigProxy,
     action_type: MessagingActionType,
     body_params: Any,
@@ -463,6 +465,36 @@ def _build_email(  # pylint: disable=too-many-return-statements
                 }
             ),
         )
+    if action_type == MessagingActionType.EXTERNAL_USER_WELCOME:
+        base_template = get_email_template(action_type)
+        # Generate display name for personalization
+        display_name = body_params.username
+        if body_params.first_name:
+            display_name = body_params.first_name
+            if body_params.last_name:
+                display_name = f"{body_params.first_name} {body_params.last_name}"
+
+        portal_link = (
+            f"{body_params.privacy_center_url}?access_token={body_params.access_token}"
+        )
+
+        variables = {
+            "username": body_params.username,
+            "display_name": display_name,
+            "first_name": body_params.first_name,
+            "last_name": body_params.last_name,
+            "org_name": body_params.org_name,
+            "portal_link": portal_link,
+            "privacy_center_url": body_params.privacy_center_url,
+            "access_token": body_params.access_token,
+        }
+
+        return EmailForActionType(
+            subject="Welcome to our Privacy Center",
+            body=base_template.render(variables),
+            template_variables=variables,
+        )
+
     logger.error("Message action type {} is not implemented", action_type)
     raise MessageDispatchException(
         f"Message action type {action_type} is not implemented"

fides/api/service/processors/post_processor_strategy/__init__.py

@@ -1,4 +1,5 @@
 from fides.api.service.processors.post_processor_strategy import (
+    post_processor_strategy_extract_for_execution_log,
     post_processor_strategy_filter,
     post_processor_strategy_unwrap,
 )

fides/api/service/processors/post_processor_strategy/post_processor_strategy.py

@@ -1,6 +1,8 @@
 from abc import abstractmethod
 from typing import Any, Dict, List, Optional, Union
 
+from requests import Response
+
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.service.strategy import Strategy
 
@@ -14,5 +16,13 @@ class PostProcessorStrategy(Strategy):
         data: Any,
         identity_data: Optional[Dict[str, Any]] = None,
         privacy_request: Optional[PrivacyRequest] = None,
+        response: Optional[Response] = None,
     ) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
-        """Process data from SaaS connector"""
+        """Process data from SaaS connector
+
+        Args:
+            data: The response data (potentially unwrapped/processed)
+            identity_data: Cached identity data for the request
+            privacy_request: The privacy request object
+            response: The raw HTTP response object (includes headers, status, etc.)
+        """