ethyca-fides 2.64.6b2__py2.py3-none-any.whl → 2.65.0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-02T09:46:43-0700",
+ "date": "2025-07-08T13:15:47-0300",
  "dirty": false,
  "error": null,
- "full-revisionid": "b016bc2f2307de41b4d22942b4b2f7ef0eaa8b3a",
- "version": "2.64.6b2"
+ "full-revisionid": "34ee2ee50c7518572e66a4441078263e59ba4030",
+ "version": "2.65.0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/641f6bcd424f_add_user_id_to_manual_task_log.py

@@ -0,0 +1,28 @@
+"""add user_id to manual_task_log
+
+Revision ID: 641f6bcd424f
+Revises: 41a634d8c669
+Create Date: 2025-07-02 03:19:40.171201
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "641f6bcd424f"
+down_revision = "41a634d8c669"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("manual_task_log", sa.Column("user_id", sa.String(), nullable=True))
+    op.create_index(
+        op.f("ix_manual_task_log_user_id"), "manual_task_log", ["user_id"], unique=False
+    )
+
+
+def downgrade():
+    op.drop_index(op.f("ix_manual_task_log_user_id"), table_name="manual_task_log")
+    op.drop_column("manual_task_log", "user_id")

fides/api/api/v1/endpoints/generic_overrides.py

@@ -1,6 +1,8 @@
+from functools import partial
 from typing import Dict, List, Optional, Type, Union
 
 from fastapi import Depends, HTTPException, Query, Response, Security
+from fastapi.concurrency import run_in_threadpool
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 from fastapi_pagination import Page, Params
@@ -224,9 +226,17 @@ async def list_dataset_paginated(
 
     pagination_params = Params(page=page or 1, size=size or 50)
     results = await async_paginate(db, filtered_query, pagination_params)
-    results.items = [  # type: ignore[attr-defined]
-        DatasetResponse.model_validate(result.__dict__) for result in results.items  # type: ignore[attr-defined]
-    ]
+
+    validated_items = []
+    for result in results.items:  # type: ignore[attr-defined]
+        # run pydantic validation in a threadpool to avoid blocking the main thread
+        validated_item = await run_in_threadpool(
+            partial(DatasetResponse.model_validate, result.__dict__)
+        )
+        validated_items.append(validated_item)
+
+    results.items = validated_items  # type: ignore[attr-defined]
+
     return results
 
 

fides/api/models/manual_task.py

@@ -20,6 +20,7 @@ from sqlalchemy.sql import func
 
 from fides.api.db.base_class import Base, FidesBase
 from fides.api.db.util import EnumColumn
+from fides.api.request_context import get_user_id
 from fides.api.schemas.base_class import FidesSchema
 
 if TYPE_CHECKING:
@@ -880,6 +881,10 @@ class ManualTaskLog(Base):
         ForeignKey("manual_task_instance.id", ondelete="CASCADE"),
         nullable=True,
     )
+    # The user responsible for the action being logged.  This may be `None`
+    # for system-initiated events or for legacy records created before this
+    # column existed.
+    user_id = Column(String, nullable=True, index=True)
     status = Column(String, nullable=False)
     message = Column(String, nullable=True)
     details = Column(JSONB, nullable=True)
@@ -890,6 +895,7 @@ class ManualTaskLog(Base):
         Index("ix_manual_task_log_instance_id", "instance_id"),
         Index("ix_manual_task_log_status", "status"),
         Index("ix_manual_task_log_task_id", "task_id"),
+        Index("ix_manual_task_log_user_id", "user_id"),
     )
 
     # Relationships
@@ -908,6 +914,7 @@ class ManualTaskLog(Base):
         task_id: str,
         status: "ManualTaskLogStatus",
         message: str,
+        user_id: Optional[str] = None,
         config_id: Optional[str] = None,
         instance_id: Optional[str] = None,
         details: Optional[dict[str, Any]] = None,
@@ -921,10 +928,12 @@ class ManualTaskLog(Base):
             message: Optional message describing the event
             details: Optional additional details about the event
         """
+
         data = {
             "task_id": task_id,
             "config_id": config_id,
             "instance_id": instance_id,
+            "user_id": user_id or get_user_id(),
             "status": status,
             "message": message,
             "details": details,
@@ -937,6 +946,7 @@ class ManualTaskLog(Base):
         db: Session,
         task_id: str,
         message: str,
+        user_id: Optional[str] = None,
         config_id: Optional[str] = None,
         instance_id: Optional[str] = None,
         details: Optional[dict[str, Any]] = None,
@@ -954,12 +964,14 @@ class ManualTaskLog(Base):
         Returns:
             The created error log entry
         """
+
         return cls.create_log(
             db=db,
             status=ManualTaskLogStatus.error,
             task_id=task_id,
             config_id=config_id,
             instance_id=instance_id,
+            user_id=user_id or get_user_id(),
             message=message,
             details=details,
         )

fides/api/oauth/utils.py

@@ -31,6 +31,7 @@ from fides.api.models.policy import PolicyPreWebhook
 from fides.api.models.pre_approval_webhook import PreApprovalWebhook
 from fides.api.models.privacy_request import RequestTask
 from fides.api.oauth.roles import get_scopes_from_roles
+from fides.api.request_context import set_user_id
 from fides.api.schemas.external_https import RequestTaskJWE, WebhookJWE
 from fides.api.schemas.oauth import OAuth2ClientCredentialsBearer
 from fides.common.api.v1.urn_registry import TOKEN, V1_URL_PREFIX
@@ -325,6 +326,16 @@ def extract_token_and_load_client(
         logger.debug("Auth token belongs to an invalid client_id.")
         raise AuthorizationError(detail="Not Authorized for this action")
 
+    # Populate request-scoped context with the authenticated user identifier.
+    # Prefer the linked user_id; fall back to the client id when this is the
+    # special root client (which has no associated FidesUser row).
+    ctx_user_id = client.user_id
+    if not ctx_user_id and client.id == CONFIG.security.oauth_root_client_id:
+        ctx_user_id = CONFIG.security.oauth_root_client_id
+
+    if ctx_user_id:
+        set_user_id(ctx_user_id)
+
     return token_data, client
 
 

fides/api/request_context.py

@@ -0,0 +1,76 @@
+"""
+Utilities for storing and accessing request-scoped context that must be
+accessible across the entire application stack (endpoints -> services ->
+helpers -> decorators) without having to thread additional parameters through
+all call-sites.
+
+Currently we only capture the authenticated `user_id` but additional fields
+(e.g. correlation_id, locale, feature_flags) can be added in the future by
+expanding the `RequestContext` dataclass.
+
+A `contextvars.ContextVar` is used instead of a module-level global to ensure
+that values are local to the current execution context (async task, thread or
+Celery worker) and therefore safe for concurrent workloads.
+"""
+
+from __future__ import annotations
+
+import contextvars
+from dataclasses import dataclass
+from typing import Any, Optional
+
+__all__ = [
+    "get_user_id",
+    "set_user_id",
+    "reset_request_context",
+]
+
+
+@dataclass
+class RequestContext:
+    user_id: Optional[str] = None
+
+
+# A single ContextVar holding the current request context.
+_ctx: contextvars.ContextVar[RequestContext] = contextvars.ContextVar(
+    "request_context",
+    default=RequestContext(),
+)
+
+
+def get_request_context() -> RequestContext:
+    """Return the current `RequestContext` for this request.
+
+    - The context is populated during authentication by `extract_token_and_load_client()`,
+      which calls `fides.api.request_context.set_request_context` with
+      the authenticated `user_id`.
+    - A `ContextVar` keeps the data isolated per request. Each coroutine
+      or thread sees its own copy and it is discarded at the end of the request.
+    - The returned object is the live context; treat it as read-only and use
+      `set_request_context` to mutate values.
+    """
+    return _ctx.get()
+
+
+def set_request_context(**kwargs: Any) -> None:
+    """Mutate the current context in place using `key=value` pairs."""
+    ctx = _ctx.get()
+    for key, value in kwargs.items():
+        if hasattr(ctx, key):
+            setattr(ctx, key, value)
+
+
+def reset_request_context() -> None:
+    """Remove all context, mainly for test clean-up."""
+    _ctx.set(RequestContext())
+
+
+def get_user_id() -> Optional[str]:
+    """Return the user_id from the current request context."""
+    ctx = get_request_context()
+    return ctx.user_id
+
+
+def set_user_id(user_id: str) -> None:
+    """Set the user_id in the current request context."""
+    set_request_context(user_id=user_id)

fides/api/schemas/saas/strategy_configuration.py

@@ -30,6 +30,12 @@ class FilterPostProcessorConfiguration(StrategyConfiguration):
     case_sensitive: bool = True
 
 
+class ExtractForExecutionLogPostProcessorConfiguration(StrategyConfiguration):
+    """Configuration for extracting data from response body and adding to execution log messages"""
+
+    path: Optional[str] = None
+
+
 class OffsetPaginationConfiguration(StrategyConfiguration):
     """
     Increases the value of the query param `incremental_param` by the `increment_by` until the `limit` is hit

fides/api/service/connectors/saas_connector.py

@@ -423,6 +423,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             response_data,
             identity_data,
             cast(Optional[List[PostProcessorStrategy]], saas_request.postprocessors),
+            response,
         )
 
         logger.info(
@@ -456,6 +457,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         response_data: Union[List[Dict[str, Any]], Dict[str, Any]],
         identity_data: Dict[str, Any],
         postprocessors: Optional[List[PostProcessorStrategy]],
+        response: Optional[Response] = None,
     ) -> List[Row]:
         """
         Runs the raw response through all available postprocessors for the request,
@@ -481,7 +483,9 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                     processed_data,
                     identity_data,
                     privacy_request,
+                    response,
                 )
+
             except Exception as exc:
                 raise PostProcessingException(
                     f"Exception occurred during the '{postprocessor.strategy}' postprocessor "  # type: ignore
@@ -567,6 +571,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             rows,
             privacy_request.get_cached_identity_data(),
             cast(Optional[List[PostProcessorStrategy]], masking_request.postprocessors),
+            None,
         )
 
         client = self.create_client()
@@ -584,9 +589,46 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                     )
                     continue
                 raise exc
-            client.send(prepared_request, masking_request.ignore_errors)
+            response = client.send(prepared_request, masking_request.ignore_errors)
             rows_updated += 1
 
+            # Run post-processors against the response from the masking request so that
+            # processors such as `extract_for_execution_log` can inspect the API
+            # response body (e.g. confirmation, ticket IDs, etc.).  We ignore the
+            # returned rows because masking responses are not used downstream.
+            try:
+                handled_response = self._handle_errored_response(
+                    masking_request, response
+                )
+                response_data = self._unwrap_response_data(
+                    masking_request, handled_response
+                )
+
+                # Only attempt post-processing if we have post-processors and the response body
+                # is JSON-serializable (dict or list of dicts).
+                if masking_request.postprocessors and isinstance(
+                    response_data, (dict, list)
+                ):
+                    self.process_response_data(
+                        response_data,
+                        privacy_request.get_cached_identity_data(),
+                        cast(
+                            Optional[List[PostProcessorStrategy]],
+                            masking_request.postprocessors,
+                        ),
+                        handled_response,
+                    )
+            except (
+                PostProcessingException,
+                Exception,
+            ) as exc:  # pylint: disable=broad-except
+                # We do not want a post-processing failure to prevent the masking
+                # operation itself from succeeding.
+                logger.warning(
+                    "Post-processing of masking request response failed: {}",
+                    exc,
+                )
+
         self.unset_connector_state()
 
         awaiting_async_callback: bool = bool(

fides/api/service/execution_context.py

@@ -0,0 +1,59 @@
+"""
+Execution context for collecting log messages during privacy request processing.
+
+Uses Python's contextvars for thread-safe, execution-scoped message collection.
+"""
+
+import contextvars
+from contextlib import contextmanager
+from typing import Generator, List, Optional
+
+from loguru import logger
+
+# Context variable for current execution messages
+_execution_messages: contextvars.ContextVar[Optional[List[str]]] = (
+    contextvars.ContextVar("execution_messages")
+)
+
+
+@contextmanager
+def collect_execution_log_messages() -> Generator[List[str], None, None]:
+    """
+    Context manager for collecting execution log messages.
+
+    Usage:
+        with collect_execution_log_messages() as messages:
+            # ... do work that calls add_execution_log_message()
+            pass
+        # messages now contains all collected messages
+
+    Returns:
+        List[str]: Messages collected during execution
+    """
+    messages: List[str] = []
+    token = _execution_messages.set(messages)
+    try:
+        yield messages
+    finally:
+        _execution_messages.reset(token)
+
+
+def add_execution_log_message(message: str) -> None:
+    """Add a message to the current execution context"""
+    try:
+        messages = _execution_messages.get()
+        if messages is not None and message.strip():
+            messages.append(message.strip())
+    except LookupError:
+        logger.error(
+            f"Attempted to call add_execution_log_message outside of execution context, unable to add message: {message}"
+        )
+
+
+def get_execution_log_messages() -> List[str]:
+    """Get messages from current execution context (for testing/debugging)"""
+    try:
+        messages = _execution_messages.get()
+        return messages.copy() if messages is not None else []
+    except LookupError:
+        return []

fides/api/service/processors/post_processor_strategy/__init__.py

@@ -1,4 +1,5 @@
 from fides.api.service.processors.post_processor_strategy import (
+    post_processor_strategy_extract_for_execution_log,
     post_processor_strategy_filter,
     post_processor_strategy_unwrap,
 )

fides/api/service/processors/post_processor_strategy/post_processor_strategy.py

@@ -1,6 +1,8 @@
 from abc import abstractmethod
 from typing import Any, Dict, List, Optional, Union
 
+from requests import Response
+
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.service.strategy import Strategy
 
@@ -14,5 +16,13 @@ class PostProcessorStrategy(Strategy):
         data: Any,
         identity_data: Optional[Dict[str, Any]] = None,
         privacy_request: Optional[PrivacyRequest] = None,
+        response: Optional[Response] = None,
     ) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
-        """Process data from SaaS connector"""
+        """Process data from SaaS connector
+
+        Args:
+            data: The response data (potentially unwrapped/processed)
+            identity_data: Cached identity data for the request
+            privacy_request: The privacy request object
+            response: The raw HTTP response object (includes headers, status, etc.)
+        """

fides/api/service/processors/post_processor_strategy/post_processor_strategy_extract_for_execution_log.py

@@ -0,0 +1,73 @@
+from typing import Any, Dict, List, Optional, Union
+
+import pydash
+from loguru import logger
+from requests import Response
+
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.saas.strategy_configuration import (
+    ExtractForExecutionLogPostProcessorConfiguration,
+)
+from fides.api.service.execution_context import add_execution_log_message
+from fides.api.service.processors.post_processor_strategy.post_processor_strategy import (
+    PostProcessorStrategy,
+)
+
+
+class ExtractForExecutionLogPostProcessorStrategy(PostProcessorStrategy):
+    """Postprocessor that extracts data from API response contents and adds it to execution log messages"""
+
+    name = "extract_for_execution_log"
+    configuration_model = ExtractForExecutionLogPostProcessorConfiguration
+
+    def __init__(self, configuration: ExtractForExecutionLogPostProcessorConfiguration):
+        self.configuration = configuration
+
+    def process(
+        self,
+        data: Union[List[Dict[str, Any]], Dict[str, Any]],
+        identity_data: Optional[Dict[str, Any]] = None,
+        privacy_request: Optional[PrivacyRequest] = None,
+        response: Optional[Response] = None,
+    ) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
+        """
+        Extract data from response contents and add to execution log messages.
+        Data is passed through unchanged (non-destructive postprocessor).
+        """
+        config = self.configuration
+
+        try:
+            # Only process if response.contents is available
+            if response is None or not response.content:
+                return data
+
+            if config.path is None:
+                # No path specified - use entire response contents as string
+                message = str(response.content)
+            else:
+                # Attempt to parse the response content as JSON before extracting
+                try:
+                    json_data: Any = response.json()
+                except ValueError:
+                    logger.debug(
+                        "Unable to parse response content as JSON for path extraction"
+                    )
+                    return data
+
+                extracted_value = pydash.get(json_data, config.path)
+                if extracted_value is None:
+                    logger.debug(f"No data found at path '{config.path}'")
+                    return data
+
+                message = str(extracted_value)
+
+            # Add message to execution context
+            add_execution_log_message(message)
+            logger.info(message)
+
+        except Exception as e:
+            error_msg = f"Error in extract_for_execution_log postprocessor: {e}"
+            logger.error(error_msg)
+            add_execution_log_message(error_msg)
+
+        return data

fides/api/service/processors/post_processor_strategy/post_processor_strategy_factory.py

@@ -9,6 +9,9 @@ from fides.api.schemas.saas.strategy_configuration import StrategyConfiguration
 from fides.api.service.processors.post_processor_strategy.post_processor_strategy import (
     PostProcessorStrategy,
 )
+from fides.api.service.processors.post_processor_strategy.post_processor_strategy_extract_for_execution_log import (
+    ExtractForExecutionLogPostProcessorStrategy,
+)
 from fides.api.service.processors.post_processor_strategy.post_processor_strategy_filter import (
     FilterPostProcessorStrategy,
 )
@@ -24,6 +27,7 @@ class SupportedPostProcessorStrategies(Enum):
 
     unwrap = UnwrapPostProcessorStrategy
     filter = FilterPostProcessorStrategy
+    extract_for_execution_log = ExtractForExecutionLogPostProcessorStrategy
 
     @classmethod
     def __contains__(cls, item: str) -> bool:

fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py

@@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional, Union
 
 import pydash
 from loguru import logger
+from requests import Response
 
 from fides.api.common_exceptions import FidesopsException
 from fides.api.models.privacy_request import PrivacyRequest
@@ -54,6 +55,7 @@ class FilterPostProcessorStrategy(PostProcessorStrategy):
         data: Union[List[Dict[str, Any]], Dict[str, Any]],
         identity_data: Optional[Dict[str, Any]] = None,
         privacy_request: Optional[PrivacyRequest] = None,
+        response: Optional[Response] = None,
     ) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
         """
         - data: A list or a dict

fides/api/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py

@@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional, Union
 
 import pydash
 from loguru import logger
+from requests import Response
 
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.saas.strategy_configuration import (
@@ -44,6 +45,7 @@ class UnwrapPostProcessorStrategy(PostProcessorStrategy):
         data: Union[List[Dict[str, Any]], Dict[str, Any]],
         identity_data: Optional[Dict[str, Any]] = None,
         privacy_request: Optional[PrivacyRequest] = None,
+        response: Optional[Response] = None,
     ) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
         """
         :param data: A list or dict

fides/api/task/graph_task.py

@@ -42,6 +42,7 @@ from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest, Reque
 from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.service.connectors.base_connector import BaseConnector
+from fides.api.service.execution_context import collect_execution_log_messages
 from fides.api.task.consolidate_query_matches import consolidate_query_matches
 from fides.api.task.filter_element_match import filter_element_match
 from fides.api.task.refine_target_path import FieldPathNodeInput
@@ -419,7 +420,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         self,
         action_type: ActionType,
         ex: Optional[BaseException] = None,
-        success_override_msg: Optional[BaseException] = None,
+        success_override_msg: Optional[str] = None,
     ) -> None:
         """On completion activities"""
         if ex:
@@ -615,19 +616,29 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             formatted_input_data: NodeInput = self.pre_process_input_data(
                 *inputs, group_dependent_fields=True
             )
-            output: List[Row] = self.connector.retrieve_data(
-                self.execution_node,
-                self.resources.policy,
-                self.resources.request,
-                self.resources.privacy_request_task,
-                formatted_input_data,
-            )
+
+            # Use execution context to capture postprocessor messages
+            with collect_execution_log_messages() as messages:
+                output: List[Row] = self.connector.retrieve_data(
+                    self.execution_node,
+                    self.resources.policy,
+                    self.resources.request,
+                    self.resources.privacy_request_task,
+                    formatted_input_data,
+                )
+
             filtered_output: List[Row] = self.access_results_post_processing(
                 self.pre_process_input_data(*inputs, group_dependent_fields=False),
                 output,
             )
-            self.log_end(ActionType.access)
-            return filtered_output
+
+        # Include postprocessor messages in success message if available
+        success_message = None
+        if messages:
+            success_message = "\n".join(messages)
+
+        self.log_end(ActionType.access, success_override_msg=success_message)
+        return filtered_output
 
     @retry(action_type=ActionType.erasure, default_return=0)
     def erasure_request(
@@ -683,13 +694,16 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             )
             return 0
 
-        output = self.connector.mask_data(
-            self.execution_node,
-            self.resources.policy,
-            self.resources.request,
-            self.resources.privacy_request_task,
-            retrieved_data,
-        )
+        # Use execution context to capture postprocessor messages
+        with collect_execution_log_messages() as messages:
+            output = self.connector.mask_data(
+                self.execution_node,
+                self.resources.policy,
+                self.resources.request,
+                self.resources.privacy_request_task,
+                retrieved_data,
+            )
+
         if self.request_task.id:
             # For DSR 3.0, largely for testing. DSR 3.0 uses Request Task status
             # instead of presence of cached erasure data to know if we should rerun a node
@@ -700,7 +714,13 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         self.resources.cache_erasure(
             self.key.value, output
         )  # Cache that the erasure was performed in case we need to restart
-        self.log_end(ActionType.erasure)
+
+        # Include postprocessor messages in success message if available
+        success_message = None
+        if messages:
+            success_message = "\n".join(messages)
+
+        self.log_end(ActionType.erasure, success_override_msg=success_message)
         return output
 
     @retry(action_type=ActionType.consent, default_return=False)