ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/service/async_dsr/utils.py

@@ -0,0 +1,130 @@
+"""
+Pure utility functions for async DSR operations.
+
+This module contains utility functions with no business logic dependencies.
+These are helper functions that can be used across the async DSR system.
+"""
+
+from enum import Enum
+
+# Type checking imports
+from typing import TYPE_CHECKING, List, cast
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.common_exceptions import PrivacyRequestError
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.privacy_request.request_task import AsyncTaskType, RequestTask
+from fides.api.schemas.saas.saas_config import SaaSRequest
+
+if TYPE_CHECKING:
+    from fides.api.service.connectors.query_configs.saas_query_config import (
+        SaaSQueryConfig,
+    )
+
+
+def has_async_requests(query_config: "SaaSQueryConfig") -> bool:
+    """Check if any read/update/delete requests have async configuration"""
+    all_requests: List[SaaSRequest] = cast(
+        List[SaaSRequest], query_config.get_read_requests_by_identity()
+    )
+    masking_request = query_config.get_masking_request()
+    if masking_request:
+        all_requests.append(masking_request)
+    return any(request.async_config is not None for request in all_requests)
+
+
+def is_polling_continuation(request_task: RequestTask) -> bool:
+    """Check if this is a polling continuation (not initial request)"""
+    async_type_check = request_task.async_type == AsyncTaskType.polling
+    sub_requests_count = len(request_task.sub_requests)
+
+    logger.warning(
+        f"is_polling_continuation for task {request_task.id}: "
+        f"async_type={request_task.async_type} (polling={async_type_check}), "
+        f"sub_requests_count={sub_requests_count}"
+    )
+
+    return async_type_check and sub_requests_count > 0
+
+
+def is_callback_completion(request_task: RequestTask) -> bool:
+    """Check if this is a completed callback request"""
+    return bool(request_task.callback_succeeded)
+
+
+def is_async_request(
+    request_task: RequestTask, query_config: "SaaSQueryConfig"
+) -> bool:
+    """
+    Check if this is an async request (callback, continuation, or initial async).
+
+    This is the main entry point for async detection.
+    """
+    return (
+        is_callback_completion(request_task)
+        or is_polling_continuation(request_task)
+        or has_async_requests(query_config)
+    )
+
+
+class AsyncPhase(Enum):
+    """Enum representing different phases of async DSR processing."""
+
+    callback_completion = "callback_completion"
+    polling_continuation = "polling_continuation"
+    initial_async = "initial_async"
+    sync = "sync"
+
+
+def get_async_phase(
+    request_task: RequestTask, query_config: "SaaSQueryConfig"
+) -> AsyncPhase:
+    """
+    Classify the phase of async request for routing purposes.
+
+    Returns:
+        AsyncPhase enum value representing the current phase:
+        - callback_completion: Callback has completed
+        - polling_continuation: Continuing an existing polling process
+        - initial_async: Initial async request setup
+        - sync: Not an async request
+    """
+    if is_callback_completion(request_task):
+        return AsyncPhase.callback_completion
+    if is_polling_continuation(request_task):
+        return AsyncPhase.polling_continuation
+    if has_async_requests(query_config):
+        return AsyncPhase.initial_async
+    return AsyncPhase.sync
+
+
+def get_connection_config_from_task(
+    db: Session, request_task: RequestTask
+) -> ConnectionConfig:
+    """
+    Get ConnectionConfig from a RequestTask.
+
+    This utility function retrieves the connection configuration
+    associated with a request task by looking up the dataset configuration.
+    """
+    dataset_config = DatasetConfig.filter(
+        db=db,
+        conditions=(DatasetConfig.fides_key == request_task.dataset_name),
+    ).first()
+    if not dataset_config:
+        raise PrivacyRequestError(
+            f"DatasetConfig with fides_key {request_task.dataset_name} not found."
+        )
+
+    connection_config = ConnectionConfig.get(
+        db=db, object_id=dataset_config.connection_config_id
+    )
+    if not connection_config:
+        raise PrivacyRequestError(
+            f"ConnectionConfig with id {dataset_config.connection_config_id} not found."
+        )
+
+    return connection_config

fides/api/service/connectors/base_erasure_email_connector.py

@@ -59,11 +59,18 @@ class BaseErasureEmailConnector(BaseEmailConnector):
         self, user_identities: Dict[str, Any], privacy_request: PrivacyRequest
     ) -> bool:
         """Schedules an erasure email for erasure privacy requests containing the required identities"""
+
+        # do not send erasure emails if there is more than 1 action type for the request.
+        # we use consent + erasure types to facilitate Do Not Sell features.
+        multiple_action_types = len(privacy_request.policy.get_all_action_types()) > 1
+
         erasure_rules: List[Rule] = privacy_request.policy.get_rules_for_action(
             action_type=ActionType.erasure
         )
+
         return bool(
             erasure_rules
+            and not multiple_action_types
             and filter_user_identities_for_connector(self.config, user_identities)
         )
 

fides/api/service/connectors/bigquery_connector.py

@@ -7,7 +7,7 @@ from sqlalchemy.orm import Session
 from sqlalchemy.sql import Executable  # type: ignore
 from sqlalchemy.sql.elements import TextClause
 
-from fides.api.common_exceptions import ConnectionException
+from fides.api.common_exceptions import ConnectionException, TableNotFound
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionTestStatus
 from fides.api.models.policy import Policy
@@ -194,23 +194,41 @@ class BigQueryConnector(SQLConnector):
         update_or_delete_ct = 0
         client = self.client()
 
-        if query_config.uses_delete_masking_strategy():
-            delete_stmts = query_config.generate_delete(client, input_data or {})
-            logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
-            update_or_delete_ct += self._execute_statements_with_sql_dry_run(
-                delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
-            )
-        else:
-            for row in rows:
-                update_or_delete_stmts: List[Executable] = query_config.generate_update(
-                    row, policy, privacy_request, client
-                )
-                logger.debug(
-                    f"Generated {len(update_or_delete_stmts)} UPDATE statements"
-                )
+        try:
+            if query_config.uses_delete_masking_strategy():
+                delete_stmts = query_config.generate_delete(client, input_data or {})
+                logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
                 update_or_delete_ct += self._execute_statements_with_sql_dry_run(
-                    update_or_delete_stmts,
+                    delete_stmts,
                     self.should_dry_run(SqlDryRunMode.erasure),
                     client,
                 )
+            else:
+                for row in rows:
+                    update_or_delete_stmts: List[Executable] = (
+                        query_config.generate_update(
+                            row, policy, privacy_request, client
+                        )
+                    )
+                    logger.debug(
+                        f"Generated {len(update_or_delete_stmts)} UPDATE statements"
+                    )
+                    update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                        update_or_delete_stmts,
+                        self.should_dry_run(SqlDryRunMode.erasure),
+                        client,
+                    )
+        except Exception as exc:
+            # Check if table exists using qualified table name
+            qualified_table_name = self.get_qualified_table_name(node)
+            if not self.table_exists(qualified_table_name):
+                # For data erasure, we can always skip the collection since other collections
+                # don't depend on erasure results
+                skip_msg = (
+                    f"Table '{qualified_table_name}' did not exist during data erasure."
+                )
+                raise TableNotFound(skip_msg) from exc
+            # Table exists or can't check - re-raise original exception
+            raise
+
         return update_or_delete_ct

fides/api/service/connectors/fides/fides_client.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+from asyncio import sleep
+from datetime import datetime
 from typing import Any, Dict, List, Optional
 
 import httpx
 from httpx import AsyncClient, Client, HTTPStatusError, Request, RequestError, Timeout
 from loguru import logger
 
+from fides.api.common_exceptions import PrivacyRequestNotFound
 from fides.api.schemas.privacy_request import (
     PrivacyRequestCreate,
     PrivacyRequestResponse,
@@ -13,11 +16,11 @@ from fides.api.schemas.privacy_request import (
 )
 from fides.api.schemas.redis_cache import Identity
 from fides.api.schemas.user import UserLogin
-from fides.api.service.privacy_request.request_service import poll_server_for_completion
 from fides.api.util.collection_util import Row
 from fides.api.util.errors import FidesError
 from fides.api.util.wrappers import sync
 from fides.common.api.v1 import urn_registry as urls
+from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS
 
 COMPLETION_STATUSES = [
     PrivacyRequestStatus.complete,
@@ -27,6 +30,65 @@ COMPLETION_STATUSES = [
 ]
 
 
+def get_async_client() -> AsyncClient:
+    """Return an async client used to make API requests"""
+    return AsyncClient()
+
+
+async def poll_server_for_completion(
+    privacy_request_id: str,
+    server_url: str,
+    token: str,
+    *,
+    poll_interval_seconds: int = 30,
+    timeout_seconds: int = 1800,  # 30 minutes
+    client: AsyncClient | None = None,
+) -> PrivacyRequestResponse:
+    """Poll a server for privacy request completion.
+
+    Requests will report complete with if they have a status of canceled, complete,
+    denied, or error. By default the polling will time out if not completed in 30
+    minutes, time can be overridden by setting the timeout_seconds.
+    """
+    url = f"{server_url}{urls.V1_URL_PREFIX}{PRIVACY_REQUESTS}?request_id={privacy_request_id}"
+    start_time = datetime.now()
+    elapsed_time = 0.0
+    while elapsed_time < timeout_seconds:
+        if client:
+            response = await client.get(
+                url, headers={"Authorization": f"Bearer {token}"}
+            )
+        else:
+            async_client = get_async_client()
+            response = await async_client.get(
+                url, headers={"Authorization": f"Bearer {token}"}
+            )
+        response.raise_for_status()
+
+        # Privacy requests are returned paginated. Since this is searching for a specific
+        # privacy request there should only be one value present in items.
+        items = response.json()["items"]
+        if not items:
+            raise PrivacyRequestNotFound(
+                f"No privacy request found with id '{privacy_request_id}'"
+            )
+        status = PrivacyRequestResponse(**items[0])
+        if status.status and status.status in (
+            PrivacyRequestStatus.complete,
+            PrivacyRequestStatus.canceled,
+            PrivacyRequestStatus.error,
+            PrivacyRequestStatus.denied,
+        ):
+            return status
+
+        await sleep(poll_interval_seconds)
+        time_delta = datetime.now() - start_time
+        elapsed_time = time_delta.seconds
+    raise TimeoutError(
+        f"Timeout of {timeout_seconds} seconds has been exceeded while waiting for privacy request {privacy_request_id}"
+    )
+
+
 class FidesClient:
     """
     A helper client to broker communications between Fides servers.

fides/api/service/connectors/http_connector.py

@@ -3,6 +3,9 @@ from typing import Any, Dict, List, Optional
 
 import requests
 from loguru import logger
+from oauthlib.oauth2 import BackendApplicationClient
+from requests.auth import HTTPBasicAuth
+from requests_oauthlib import OAuth2Session
 from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR
 
 from fides.api.common_exceptions import ClientUnsuccessfulException
@@ -41,25 +44,58 @@ class HTTPSConnector(BaseConnector[None]):
     ) -> Optional[Dict[str, Any]]:
         """Calls a client-defined endpoint and returns the data that it responds with"""
         config = HttpsSchema(**self.configuration.secrets or {})
-        headers = self.build_authorization_header()
-        headers.update(additional_headers)
+
+        def _request_with_oauth() -> requests.Response:
+            """Helper function to make a request with OAuth2 authentication"""
+            client_id = oauth_config.client_id
+            client_secret = oauth_config.client_secret
+            scopes = oauth_config.scope.split(" ") or []
+            auth = HTTPBasicAuth(client_id, client_secret)
+            client = BackendApplicationClient(client_id=client_id)
+            session_client = OAuth2Session(client=client, scope=scopes)
+            try:
+                # Fetch the access token from the token URL
+                session_client.fetch_token(token_url=oauth_config.token_url, auth=auth)
+            except Exception as e:
+                logger.error(f"Error fetching OAuth2 token: {e}")
+                raise ClientUnsuccessfulException(
+                    status_code=HTTP_500_INTERNAL_SERVER_ERROR
+                )
+
+            return session_client.post(
+                url=config.url, headers=additional_headers, json=request_body
+            )
+
+        def _request_without_oauth() -> requests.Response:
+            """Helper function to make a request without OAuth2 authentication"""
+            headers = self.build_authorization_header()
+            headers.update(additional_headers)
+            return requests.post(url=config.url, headers=headers, json=request_body)
+
+        oauth_config = self.configuration.oauth_config
 
         try:
-            response = requests.post(url=config.url, headers=headers, json=request_body)
+            response = (
+                _request_with_oauth()
+                if oauth_config is not None
+                else _request_without_oauth()
+            )
+
+            if not response_expected:
+                return {}
+
+            if not response.ok:
+                logger.error("Invalid response received from webhook.")
+                raise ClientUnsuccessfulException(status_code=response.status_code)
+
+            return json.loads(response.text)
+
         except requests.ConnectionError:
-            logger.info("Requests connection error received.")
+            logger.error("HTTPS client received a connection error.")
             raise ClientUnsuccessfulException(
                 status_code=HTTP_500_INTERNAL_SERVER_ERROR
             )
 
-        if not response_expected:
-            return {}
-
-        if not response.ok:
-            logger.error("Invalid response received from webhook.")
-            raise ClientUnsuccessfulException(status_code=response.status_code)
-        return json.loads(response.text)
-
     def test_connection(self) -> Optional[ConnectionTestStatus]:
         """
         Override to skip connection test

fides/api/service/connectors/microsoft_sql_server_connector.py

@@ -25,7 +25,7 @@ class MicrosoftSQLServerConnector(SQLConnector):
     def build_uri(self) -> URL:
         """
         Build URI of format
-        mssql+pymssql://[username]:[password]@[host]:[port]/[dbname]
+        mssql+pymssql://[username]:[password]@[host]:[port]/[dbname]?read_only=[read_only_connection]
         Returns URL obj, since SQLAlchemy's create_engine method accepts either a URL obj or a string
         """
 
@@ -38,6 +38,11 @@ class MicrosoftSQLServerConnector(SQLConnector):
             host=config.host,
             port=config.port,
             database=config.dbname,
+            query=(
+                {"read_only": str(config.read_only_connection)}
+                if config.read_only_connection
+                else None
+            ),
         )
 
         return url

fides/api/service/connectors/mongodb_connector.py

@@ -27,31 +27,96 @@ class MongoDBConnector(BaseConnector[MongoClient]):
 
     def build_uri(self) -> str:
         """
-        Builds URI of format mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
+        Builds URI of format:
+        - mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
+        - mongodb+srv://[username:password@]host[/[defaultauthdb][?options]] (for SRV)
         """
 
         config = MongoDBSchema(**self.configuration.secrets or {})
 
+        # Determine scheme based on SRV setting
+        scheme = "mongodb+srv" if config.use_srv else "mongodb"
+
+        # Build authentication part
         user_pass: str = ""
-        default_auth_db: str = ""
         if config.username and config.password:
             user_pass = f"{quote_plus(config.username)}:{quote_plus(config.password)}@"
-            if config.defaultauthdb:
-                default_auth_db = f"/{config.defaultauthdb}"
 
-        port: str = f":{config.port}" if config.port else ""
-        url = f"mongodb://{user_pass}{config.host}{port}{default_auth_db}"
+        # Build host/port part
+        if config.use_srv:
+            # SRV connections use hostname only, no port
+            host_part = config.host
+        else:
+            # Standard connections include port if specified
+            port = f":{config.port}" if config.port else ""
+            host_part = f"{config.host}{port}"
+
+        # Build database path
+        default_auth_db = f"/{config.defaultauthdb}" if config.defaultauthdb else ""
+
+        # Build query parameters
+        params = []
+
+        # Determine SSL behavior
+        ssl_active = self._determine_ssl_enabled(config)
+        if ssl_active is True:
+            params.append("ssl=true")
+        elif ssl_active is False and config.use_srv:
+            # Explicitly disable SSL for SRV (override default)
+            params.append("ssl=false")
+
+        query = f"?{'&'.join(params)}" if params else ""
+
+        url = f"{scheme}://{user_pass}{host_part}{default_auth_db}{query}"
         return url
 
+    def _determine_ssl_enabled(self, config: MongoDBSchema) -> Optional[bool]:
+        """Determine if SSL should be enabled based on configuration"""
+        if config.ssl_enabled is not None:
+            # Explicit setting takes precedence
+            return config.ssl_enabled
+        if config.use_srv:
+            # SRV defaults to SSL enabled
+            return True
+
+        # Standard connections default to SSL disabled
+        return False
+
     def create_client(self) -> MongoClient:
         """Returns a client for a MongoDB instance"""
-        uri = (self.configuration.secrets or {}).get("url") or self.build_uri()
+        secrets = self.configuration.secrets or {}
+        uri = secrets.get("url") or self.build_uri()
+
+        # Log connection details (without credentials) for debugging
+        if secrets.get("url"):
+            # URL-based connection - determine scheme and SSL from URL or defaults
+            scheme = "mongodb+srv" if "mongodb+srv://" in uri else "mongodb"
+            # For URL connections, SSL is determined by the URI scheme or explicit setting
+            ssl_status = (
+                "enabled"
+                if "ssl=true" in uri or secrets.get("ssl_enabled")
+                else "disabled"
+            )
+        else:
+            # Individual parameter connection - validate using schema
+            config = MongoDBSchema(**secrets)
+            scheme = "mongodb+srv" if config.use_srv else "mongodb"
+            ssl_status = (
+                "enabled" if self._determine_ssl_enabled(config) else "disabled"
+            )
+
+        logger.info(
+            "Connecting to MongoDB using {} scheme with SSL {}", scheme, ssl_status
+        )
+
         try:
             return MongoClient(
                 uri, serverSelectionTimeoutMS=5000, uuidRepresentation="standard"
             )
-        except ValueError:
-            raise ConnectionException("Value Error connecting to MongoDB.")
+        except ValueError as exc:
+            raise ConnectionException(f"Value Error connecting to MongoDB: {exc}")
+        except Exception as exc:
+            raise ConnectionException(f"Error connecting to MongoDB: {exc}")
 
     def query_config(self, node: ExecutionNode) -> QueryConfig[Any]:
         """Query wrapper corresponding to the input traversal_node."""