ethyca-fides 2.71.1b1__py2.py3-none-any.whl → 2.74.0rc1__py2.py3-none-any.whl

fides/api/service/async_dsr/utils.py

@@ -0,0 +1,130 @@
+"""
+Pure utility functions for async DSR operations.
+
+This module contains utility functions with no business logic dependencies.
+These are helper functions that can be used across the async DSR system.
+"""
+
+from enum import Enum
+
+# Type checking imports
+from typing import TYPE_CHECKING, List, cast
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.common_exceptions import PrivacyRequestError
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.privacy_request.request_task import AsyncTaskType, RequestTask
+from fides.api.schemas.saas.saas_config import SaaSRequest
+
+if TYPE_CHECKING:
+    from fides.api.service.connectors.query_configs.saas_query_config import (
+        SaaSQueryConfig,
+    )
+
+
+def has_async_requests(query_config: "SaaSQueryConfig") -> bool:
+    """Check if any read/update/delete requests have async configuration"""
+    all_requests: List[SaaSRequest] = cast(
+        List[SaaSRequest], query_config.get_read_requests_by_identity()
+    )
+    masking_request = query_config.get_masking_request()
+    if masking_request:
+        all_requests.append(masking_request)
+    return any(request.async_config is not None for request in all_requests)
+
+
+def is_polling_continuation(request_task: RequestTask) -> bool:
+    """Check if this is a polling continuation (not initial request)"""
+    async_type_check = request_task.async_type == AsyncTaskType.polling
+    sub_requests_count = len(request_task.sub_requests)
+
+    logger.warning(
+        f"is_polling_continuation for task {request_task.id}: "
+        f"async_type={request_task.async_type} (polling={async_type_check}), "
+        f"sub_requests_count={sub_requests_count}"
+    )
+
+    return async_type_check and sub_requests_count > 0
+
+
+def is_callback_completion(request_task: RequestTask) -> bool:
+    """Check if this is a completed callback request"""
+    return bool(request_task.callback_succeeded)
+
+
+def is_async_request(
+    request_task: RequestTask, query_config: "SaaSQueryConfig"
+) -> bool:
+    """
+    Check if this is an async request (callback, continuation, or initial async).
+
+    This is the main entry point for async detection.
+    """
+    return (
+        is_callback_completion(request_task)
+        or is_polling_continuation(request_task)
+        or has_async_requests(query_config)
+    )
+
+
+class AsyncPhase(Enum):
+    """Enum representing different phases of async DSR processing."""
+
+    callback_completion = "callback_completion"
+    polling_continuation = "polling_continuation"
+    initial_async = "initial_async"
+    sync = "sync"
+
+
+def get_async_phase(
+    request_task: RequestTask, query_config: "SaaSQueryConfig"
+) -> AsyncPhase:
+    """
+    Classify the phase of async request for routing purposes.
+
+    Returns:
+        AsyncPhase enum value representing the current phase:
+        - callback_completion: Callback has completed
+        - polling_continuation: Continuing an existing polling process
+        - initial_async: Initial async request setup
+        - sync: Not an async request
+    """
+    if is_callback_completion(request_task):
+        return AsyncPhase.callback_completion
+    if is_polling_continuation(request_task):
+        return AsyncPhase.polling_continuation
+    if has_async_requests(query_config):
+        return AsyncPhase.initial_async
+    return AsyncPhase.sync
+
+
+def get_connection_config_from_task(
+    db: Session, request_task: RequestTask
+) -> ConnectionConfig:
+    """
+    Get ConnectionConfig from a RequestTask.
+
+    This utility function retrieves the connection configuration
+    associated with a request task by looking up the dataset configuration.
+    """
+    dataset_config = DatasetConfig.filter(
+        db=db,
+        conditions=(DatasetConfig.fides_key == request_task.dataset_name),
+    ).first()
+    if not dataset_config:
+        raise PrivacyRequestError(
+            f"DatasetConfig with fides_key {request_task.dataset_name} not found."
+        )
+
+    connection_config = ConnectionConfig.get(
+        db=db, object_id=dataset_config.connection_config_id
+    )
+    if not connection_config:
+        raise PrivacyRequestError(
+            f"ConnectionConfig with id {dataset_config.connection_config_id} not found."
+        )
+
+    return connection_config

fides/api/service/connectors/bigquery_connector.py

@@ -7,7 +7,7 @@ from sqlalchemy.orm import Session
 from sqlalchemy.sql import Executable  # type: ignore
 from sqlalchemy.sql.elements import TextClause
 
-from fides.api.common_exceptions import ConnectionException
+from fides.api.common_exceptions import ConnectionException, TableNotFound
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionTestStatus
 from fides.api.models.policy import Policy
@@ -194,23 +194,41 @@ class BigQueryConnector(SQLConnector):
         update_or_delete_ct = 0
         client = self.client()
 
-        if query_config.uses_delete_masking_strategy():
-            delete_stmts = query_config.generate_delete(client, input_data or {})
-            logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
-            update_or_delete_ct += self._execute_statements_with_sql_dry_run(
-                delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
-            )
-        else:
-            for row in rows:
-                update_or_delete_stmts: List[Executable] = query_config.generate_update(
-                    row, policy, privacy_request, client
-                )
-                logger.debug(
-                    f"Generated {len(update_or_delete_stmts)} UPDATE statements"
-                )
+        try:
+            if query_config.uses_delete_masking_strategy():
+                delete_stmts = query_config.generate_delete(client, input_data or {})
+                logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
                 update_or_delete_ct += self._execute_statements_with_sql_dry_run(
-                    update_or_delete_stmts,
+                    delete_stmts,
                     self.should_dry_run(SqlDryRunMode.erasure),
                     client,
                 )
+            else:
+                for row in rows:
+                    update_or_delete_stmts: List[Executable] = (
+                        query_config.generate_update(
+                            row, policy, privacy_request, client
+                        )
+                    )
+                    logger.debug(
+                        f"Generated {len(update_or_delete_stmts)} UPDATE statements"
+                    )
+                    update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                        update_or_delete_stmts,
+                        self.should_dry_run(SqlDryRunMode.erasure),
+                        client,
+                    )
+        except Exception as exc:
+            # Check if table exists using qualified table name
+            qualified_table_name = self.get_qualified_table_name(node)
+            if not self.table_exists(qualified_table_name):
+                # For data erasure, we can always skip the collection since other collections
+                # don't depend on erasure results
+                skip_msg = (
+                    f"Table '{qualified_table_name}' did not exist during data erasure."
+                )
+                raise TableNotFound(skip_msg) from exc
+            # Table exists or can't check - re-raise original exception
+            raise
+
         return update_or_delete_ct

fides/api/service/connectors/fides/fides_client.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+from asyncio import sleep
+from datetime import datetime
 from typing import Any, Dict, List, Optional
 
 import httpx
 from httpx import AsyncClient, Client, HTTPStatusError, Request, RequestError, Timeout
 from loguru import logger
 
+from fides.api.common_exceptions import PrivacyRequestNotFound
 from fides.api.schemas.privacy_request import (
     PrivacyRequestCreate,
     PrivacyRequestResponse,
@@ -13,11 +16,11 @@ from fides.api.schemas.privacy_request import (
 )
 from fides.api.schemas.redis_cache import Identity
 from fides.api.schemas.user import UserLogin
-from fides.api.service.privacy_request.request_service import poll_server_for_completion
 from fides.api.util.collection_util import Row
 from fides.api.util.errors import FidesError
 from fides.api.util.wrappers import sync
 from fides.common.api.v1 import urn_registry as urls
+from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS
 
 COMPLETION_STATUSES = [
     PrivacyRequestStatus.complete,
@@ -27,6 +30,65 @@ COMPLETION_STATUSES = [
 ]
 
 
+def get_async_client() -> AsyncClient:
+    """Return an async client used to make API requests"""
+    return AsyncClient()
+
+
+async def poll_server_for_completion(
+    privacy_request_id: str,
+    server_url: str,
+    token: str,
+    *,
+    poll_interval_seconds: int = 30,
+    timeout_seconds: int = 1800,  # 30 minutes
+    client: AsyncClient | None = None,
+) -> PrivacyRequestResponse:
+    """Poll a server for privacy request completion.
+
+    Requests will report complete with if they have a status of canceled, complete,
+    denied, or error. By default the polling will time out if not completed in 30
+    minutes, time can be overridden by setting the timeout_seconds.
+    """
+    url = f"{server_url}{urls.V1_URL_PREFIX}{PRIVACY_REQUESTS}?request_id={privacy_request_id}"
+    start_time = datetime.now()
+    elapsed_time = 0.0
+    while elapsed_time < timeout_seconds:
+        if client:
+            response = await client.get(
+                url, headers={"Authorization": f"Bearer {token}"}
+            )
+        else:
+            async_client = get_async_client()
+            response = await async_client.get(
+                url, headers={"Authorization": f"Bearer {token}"}
+            )
+        response.raise_for_status()
+
+        # Privacy requests are returned paginated. Since this is searching for a specific
+        # privacy request there should only be one value present in items.
+        items = response.json()["items"]
+        if not items:
+            raise PrivacyRequestNotFound(
+                f"No privacy request found with id '{privacy_request_id}'"
+            )
+        status = PrivacyRequestResponse(**items[0])
+        if status.status and status.status in (
+            PrivacyRequestStatus.complete,
+            PrivacyRequestStatus.canceled,
+            PrivacyRequestStatus.error,
+            PrivacyRequestStatus.denied,
+        ):
+            return status
+
+        await sleep(poll_interval_seconds)
+        time_delta = datetime.now() - start_time
+        elapsed_time = time_delta.seconds
+    raise TimeoutError(
+        f"Timeout of {timeout_seconds} seconds has been exceeded while waiting for privacy request {privacy_request_id}"
+    )
+
+
 class FidesClient:
     """
     A helper client to broker communications between Fides servers.

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -10,7 +10,6 @@ from uuid import uuid4
 import pydash
 from fideslang.models import FidesDatasetReference
 from loguru import logger
-from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import FidesopsException
 from fides.api.graph.execution import ExecutionNode
@@ -47,6 +46,7 @@ from fides.api.util.saas_util import (
     ISO_8601_DATETIME,
     MASKED_OBJECT_FIELDS,
     PRIVACY_REQUEST_ID,
+    PRIVACY_REQUEST_OBJECT,
     REPLY_TO,
     REPLY_TO_TOKEN,
     UUID,
@@ -83,12 +83,10 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         """
         Returns the appropriate request configs based on the current collection and identity
         """
-        collection_name = self.node.address.collection
-
         try:
-            requests = self.endpoints[collection_name].requests
+            requests = self.endpoints[self.collection_name].requests
         except KeyError:
-            logger.error("The '{}' endpoint is not defined", collection_name)
+            logger.error("The '{}' endpoint is not defined", self.collection_name)
             return []
 
         if not requests.read:
@@ -140,7 +138,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             )
         return request
 
-    def get_masking_request(self, db: Session) -> Optional[SaaSRequest]:
+    def get_masking_request(self) -> Optional[SaaSRequest]:
         """
         Returns a tuple of the preferred action and SaaSRequest to use for masking.
         An update request is preferred, but we can use a gdpr delete endpoint or
@@ -212,6 +210,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         input_data: Dict[str, List[Any]],
         policy: Optional[Policy],
         read_request: SaaSRequest,
+        privacy_request: Optional[PrivacyRequest] = None,
     ) -> List[Tuple[SaaSRequestParams, Dict[str, Any]]]:
         """
         Takes the identity and reference values from input_data and combines them
@@ -224,6 +223,10 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         self.current_request = read_request
 
         request_params = []
+
+        if privacy_request:
+            input_data[PRIVACY_REQUEST_OBJECT] = [privacy_request.to_safe_dict()]
+
         param_value_maps = self.generate_param_value_maps(input_data, read_request)
 
         for param_value_map in param_value_maps:
@@ -277,6 +280,114 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             name: value for name, value in self.secrets.items() if name in param_names
         }
 
+    def _safe_extract_value(self, value_list: Optional[List[Any]]) -> Optional[Any]:
+        """
+        Safely extract the first value from a list
+        """
+        return value_list[0] if value_list else None
+
+    def _process_param_values_for_query(
+        self,
+        request: SaaSRequest,
+        input_data: Dict[str, List[Any]],
+        param_values: Dict[str, Any],
+    ) -> None:
+        """
+        Process param_values for query generation on access requests.
+        """
+        for param_value in request.param_values or []:
+            if param_value.references or param_value.identity:
+                input_list = input_data.get(param_value.name)
+                if input_list:
+                    param_values[param_value.name] = self._safe_extract_value(
+                        input_list
+                    )
+            elif param_value.connector_param:
+                connector_value = pydash.get(input_data, param_value.connector_param)
+                param_values[param_value.name] = self._safe_extract_value(
+                    connector_value
+                )
+
+    def _process_param_values_for_update(
+        self,
+        request: SaaSRequest,
+        collection_values: Dict[str, Row],
+        identity_data: Dict[str, Any],
+        input_data: Optional[Dict[str, List[Any]]] = None,
+        param_values: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """
+        Process param_values for update generation (erasure requests).
+        Modifies param_values in place.
+
+        Args:
+            request: The SaaS request configuration
+            collection_values: Collection data for reference resolution
+            identity_data: Identity data for parameter resolution
+            input_data: Optional upstream data from other collections
+        """
+        if param_values is None:
+            param_values = {}
+
+        for param_value in request.param_values or []:
+            if param_value.references:
+                # we resolve the param reference here for consistency,
+                # i.e. as if it may be a pointer to an `external_reference`.
+                # Cross-collection references are now supported by looking up data
+                # from upstream collections passed via input_data.
+                reference: FidesDatasetReference = SaaSConfig.resolve_param_reference(
+                    param_value.references[0], self.secrets
+                )
+                param_values[param_value.name] = pydash.get(
+                    collection_values, reference.field
+                )
+                if param_values[param_value.name] is None and input_data:
+                    logger.info(
+                        "Reference not found in collection, checking input data from upstream tasks"
+                    )
+                    # If the reference is not found in the collection (None), check the input data from upstream tasks
+                    input_list = input_data.get(param_value.name)
+                    if input_list:
+                        param_values[param_value.name] = self._safe_extract_value(
+                            input_list
+                        )
+
+            elif param_value.identity:
+                param_values[param_value.name] = pydash.get(
+                    identity_data, param_value.identity
+                )
+            elif param_value.connector_param:
+                param_values[param_value.name] = pydash.get(
+                    self.secrets, param_value.connector_param
+                )
+
+    def _add_standard_parameters(
+        self,
+        param_values: Dict[str, Any],
+    ) -> None:
+        """
+        Add standard parameters that are common across query and update operations.
+        """
+
+        # common parameters for read and update
+        if self.privacy_request:
+            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        param_values[UUID] = str(uuid4())
+        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
+        param_values[FIELD_LIST] = ",".join(
+            [
+                field.name
+                for field in self.top_level_field_map().values()
+                if field.data_type() != "None"
+            ]
+        )
+
+        if self.request_task and self.request_task.id:
+            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
+                self.request_task
+            )
+            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+
     @staticmethod
     def _generate_product_list(*args: Dict[str, Any]) -> List[Dict[str, Any]]:
         """
@@ -326,22 +437,19 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
                 f"The 'read' action is not defined for the '{self.collection_name}' "
                 f"endpoint in {self.node.connection_key}"
             )
-
-        # create the source of param values to populate the various placeholders
-        # in the path, headers, query_params, and body
         param_values: Dict[str, Any] = {}
-        for param_value in self.current_request.param_values or []:
-            if param_value.references or param_value.identity:
-                input_list = input_data.get(param_value.name)
-                if input_list:
-                    param_values[param_value.name] = input_list[0]
-            elif param_value.connector_param:
-                param_values[param_value.name] = pydash.get(
-                    input_data, param_value.connector_param
-                )[0]
+        # Process param values using utility method
+        self._process_param_values_for_query(
+            self.current_request, input_data, param_values
+        )
 
-        if self.privacy_request:
-            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        privacy_request_object = input_data.get(PRIVACY_REQUEST_OBJECT)
+        if (
+            isinstance(privacy_request_object, list)
+            and len(privacy_request_object) > 0
+            and isinstance(privacy_request_object[0], dict)
+        ):
+            param_values[PRIVACY_REQUEST_OBJECT] = privacy_request_object[0]
 
         custom_privacy_request_fields = input_data.get(CUSTOM_PRIVACY_REQUEST_FIELDS)
         if (
@@ -352,20 +460,9 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
                 0
             ]
 
-        param_values[UUID] = str(uuid4())
-        param_values[ISO_8601_DATETIME] = datetime.now().date().isoformat()
-        param_values[FIELD_LIST] = ",".join(
-            [
-                field.name
-                for field in self.top_level_field_map().values()
-                if field.data_type() != "None"
-            ]
-        )
-        if self.request_task and self.request_task.id:
-            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
-                self.request_task
-            )
-            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+        # Add standard parameters
+        # For generate_query, we don't have a PrivacyRequest object, so we use self.privacy_request
+        self._add_standard_parameters(param_values)
 
         # map param values to placeholders in path, headers, and query params
         saas_request_params: SaaSRequestParams = saas_util.map_param_values(
@@ -377,17 +474,26 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         return saas_request_params
 
     def generate_update_stmt(
-        self, row: Row, policy: Policy, request: PrivacyRequest
+        self,
+        row: Row,
+        policy: Policy,
+        request: PrivacyRequest,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> SaaSRequestParams:
         """
         This returns the method, path, header, query, and body params needed to make an API call.
         The fields in the row are masked according to the policy and added to the request body
         if specified by the body field of the masking request.
+
+        Args:
+            row: The current row data from this collection
+            policy: The privacy policy being applied
+            request: The privacy request being processed
+            input_data: Optional upstream data from other collections for cross-collection references
         """
-        session = Session.object_session(request)
-        current_request: SaaSRequest = self.get_masking_request(session)  # type: ignore
+        current_request: SaaSRequest = self.get_masking_request()  # type: ignore
         param_values: Dict[str, Any] = self.generate_update_param_values(
-            row, policy, request, current_request
+            row, policy, request, current_request, input_data
         )
 
         return self.generate_update_request_params(param_values, current_request)
@@ -416,6 +522,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         policy: Policy,
         privacy_request: PrivacyRequest,
         saas_request: SaaSRequest,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> Dict[str, Any]:
         """
         A utility that generates the update request param values
@@ -426,10 +533,16 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         the fields in the provided row that have been masked according
         to provided policy. The `all_object_fields` key maps to a JSON structure
         that holds all values, including those that have not been masked.
+
+        Args:
+            row: The current row data from this collection
+            policy: The privacy policy being applied
+            privacy_request: The privacy request being processed
+            saas_request: The SaaS request configuration
+            input_data: Optional upstream data from other collections for cross-collection references
         """
 
-        collection_name: str = self.node.address.collection
-        collection_values: Dict[str, Row] = {collection_name: row}
+        collection_values: Dict[str, Row] = {self.collection_name: row}
         identity_data: Dict[str, Any] = privacy_request.get_cached_identity_data()
         custom_privacy_request_fields: Dict[str, Any] = (
             privacy_request.get_cached_custom_privacy_request_fields()
@@ -438,46 +551,15 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         # create the source of param values to populate the various placeholders
         # in the path, headers, query_params, and body
         param_values: Dict[str, Any] = {}
-        for param_value in saas_request.param_values or []:
-            if param_value.references:
-                # we resolve the param reference here for consistency,
-                # i.e. as if it may be a pointer to an `external_reference`.
-                # however, `references` in update requests can, currently, only reference
-                # the same collection the same collection, and so it is highly unlikely
-                # that this would be an external reference at this point.
-                reference: FidesDatasetReference = SaaSConfig.resolve_param_reference(
-                    param_value.references[0], self.secrets
-                )
-                param_values[param_value.name] = pydash.get(
-                    collection_values, reference.field
-                )
-            elif param_value.identity:
-                param_values[param_value.name] = pydash.get(
-                    identity_data, param_value.identity
-                )
-            elif param_value.connector_param:
-                param_values[param_value.name] = pydash.get(
-                    self.secrets, param_value.connector_param
-                )
-
-        if self.privacy_request:
-            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        # Process param values using utility method
+        self._process_param_values_for_update(
+            saas_request, collection_values, identity_data, input_data, param_values
+        )
 
+        param_values[PRIVACY_REQUEST_OBJECT] = privacy_request.to_safe_dict()
         param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields
-        param_values[UUID] = str(uuid4())
-        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
-        param_values[FIELD_LIST] = ",".join(
-            [
-                field.name
-                for field in self.top_level_field_map().values()
-                if field.data_type() != "None"
-            ]
-        )
-        if self.request_task and self.request_task.id:
-            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
-                self.request_task
-            )
-            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+        # Add standard parameters
+        self._add_standard_parameters(param_values)
         # remove any row values for fields marked as read-only, these will be omitted from all update maps
         for field_path, field in self.field_map().items():
             if field.read_only:
@@ -503,7 +585,6 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
         param_values[MASKED_OBJECT_FIELDS] = masked_object
         param_values[ALL_OBJECT_FIELDS] = complete_object
-
         return param_values
 
     def generate_update_request_params(