ethyca-fides 2.68.1b1__py2.py3-none-any.whl → 2.68.1b3__py2.py3-none-any.whl

fides/api/service/storage/streaming/storage_client_factory.py

@@ -0,0 +1,60 @@
+"""Factory for creating provider-specific storage clients."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fides.api.service.storage.streaming.base_storage_client import BaseStorageClient
+from fides.api.service.storage.streaming.s3.s3_storage_client import S3StorageClient
+
+
+class StorageClientFactory:
+    """Factory for creating provider-specific storage clients.
+
+    This factory creates the appropriate storage client implementation based on
+    the storage type, providing a clean separation between the main client and
+    provider-specific logic.
+    """
+
+    @staticmethod
+    def create_client(storage_type: str, storage_secrets: Any) -> BaseStorageClient:
+        """Create a provider-specific storage client.
+
+        Args:
+            storage_type: Type of storage ('s3', 'gcs', 'azure')
+            storage_secrets: Storage credentials and configuration.
+                           Will be passed to the specific storage client implementation.
+
+        Returns:
+            Provider-specific storage client implementation
+
+        Raises:
+            ValueError: If the storage type is not supported
+        """
+        normalized_type = StorageClientFactory._normalize_storage_type(storage_type)
+
+        if normalized_type == "s3":
+            return S3StorageClient(storage_secrets)
+        if normalized_type == "gcs":
+            raise NotImplementedError("GCS storage is not yet supported")
+        raise ValueError(f"Unsupported storage type: {storage_type}")
+
+    @staticmethod
+    def _normalize_storage_type(storage_type: str) -> str:
+        """Normalize storage type to standard values.
+
+        Args:
+            storage_type: Raw storage type string
+
+        Returns:
+            Normalized storage type string
+        """
+        storage_type = storage_type.lower()
+
+        # Normalize storage type
+        if storage_type in {"gcs", "gcp", "google"}:
+            return "gcs"
+        if storage_type == "s3":
+            return "s3"
+
+        return storage_type

fides/api/task/graph_task.py

@@ -13,7 +13,7 @@ from sqlalchemy.orm import Session
 from fides.api.api.deps import get_autoclose_db_session as get_db
 from fides.api.common_exceptions import (
     ActionDisabled,
-    AwaitingAsyncTaskCallback,
+    AwaitingAsyncTask,
     CollectionDisabled,
     NotSupportedForCollection,
     PrivacyRequestErasureEmailSendRequired,
@@ -114,17 +114,17 @@ def retry(
                         self.log_start(action_type)
                     # Run access or erasure request
                     return func(*args, **kwargs)
-                except AwaitingAsyncTaskCallback as ex:
+                except AwaitingAsyncTask as ex:
                     traceback.print_exc()
                     logger.warning(
-                        "Request Task {} {} {} awaiting async callback",
+                        "Request Task {} {} {} awaiting async continuation",
                         self.request_task.id if self.request_task.id else None,
                         method_name,
                         self.execution_node.address,
                     )
                     # Log the awaiting processing status and exit without retrying.
                     self.log_awaiting_processing(action_type, ex)
-                    # Request Task put in "awaiting_processing" status and exited, awaiting Async Callback
+                    # Request Task put in "awaiting_processing" status and exited, awaiting for Async continuation
                     return None
                 except PrivacyRequestErasureEmailSendRequired as exc:
                     traceback.print_exc()

fides/api/task/manual/manual_task_graph_task.py

@@ -3,7 +3,7 @@ from typing import Any, Optional
 from loguru import logger
 from pydantic.v1.utils import deep_update
 
-from fides.api.common_exceptions import AwaitingAsyncTaskCallback
+from fides.api.common_exceptions import AwaitingAsyncTask
 from fides.api.models.attachment import AttachmentType
 from fides.api.models.manual_task import (
     ManualTask,
@@ -63,6 +63,17 @@ class ManualTaskGraphTask(GraphTask):
         Calls _run_request with ACCESS configs.
         Returns data if submitted, raise AwaitingAsyncTaskCallback if not
         """
+        if self.resources.request.policy.get_action_type() == ActionType.erasure:
+            # We're in an erasure privacy request's access phase - complete access task immediately
+            # since access is just for data collection to support erasure, not for user data access
+            self.update_status(
+                "Access task completed immediately for erasure privacy request (data collection only)",
+                [],
+                ActionType.access,
+                ExecutionLogStatus.complete,
+            )
+            return []
+
         result = self._run_request(
             ManualTaskConfigurationType.access_privacy_request,
             ActionType.access,
@@ -127,7 +138,7 @@ class ManualTaskGraphTask(GraphTask):
         Execute manual task logic following the standard GraphTask pattern:
         1. Create ManualTaskInstances if they don't exist
         2. Check if all required submissions are present
-        3. Return data if submitted, raise AwaitingAsyncTaskCallback if not
+        3. Return data if submitted, raise AwaitingAsyncTask if not
         """
         manual_task = self._get_manual_task_or_none()
         if manual_task is None:
@@ -254,7 +265,7 @@ class ManualTaskGraphTask(GraphTask):
         base_msg = f"Manual task for {self.connection_key} requires user input"
         if awaiting_detail_message:
             base_msg = f"{base_msg}. {awaiting_detail_message}"
-        raise AwaitingAsyncTaskCallback(base_msg)
+        raise AwaitingAsyncTask(base_msg)
 
     def _ensure_manual_task_instances(
         self,
@@ -424,7 +435,6 @@ class ManualTaskGraphTask(GraphTask):
         ]
 
         if instances_to_remove:
-
             # Remove instances from the database
             for instance in instances_to_remove:
                 instance.delete(self.resources.session)

fides/api/util/connection_type.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Dict, Set
+from typing import Any, Dict, Optional, Set
 
 from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError
 from fides.api.models.connectionconfig import ConnectionType
@@ -141,7 +141,9 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]
     Note that this does not return actual secrets, instead we return the *types* of
     secret fields needed to authenticate.
     """
-    connection_system_types: list[ConnectionSystemTypeMap] = get_connection_types()
+    connection_system_types: list[ConnectionSystemTypeMap] = get_connection_types(
+        include_test_connections=True
+    )
     if not any(item.identifier == connection_type for item in connection_system_types):
         raise NoSuchConnectionTypeSecretSchemaError(
             f"No connection type found with name '{connection_type}'."
@@ -189,15 +191,15 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]
     return schema
 
 
-def get_connection_types(
-    search: str | None = None,
-    system_type: SystemType | None = None,
+def _is_match(elem: str, search: Optional[str] = None) -> bool:
+    """If a search query param was included, is it a substring of an available connector type?"""
+    return search.lower() in elem.lower() if search else True
+
+
+def get_saas_connection_types(
     action_types: Set[ActionType] = SUPPORTED_ACTION_TYPES,
+    search: Optional[str] = None,
 ) -> list[ConnectionSystemTypeMap]:
-    def is_match(elem: str) -> bool:
-        """If a search query param was included, is it a substring of an available connector type?"""
-        return search.lower() in elem.lower() if search else True
-
     def saas_request_type_filter(connection_type: str) -> bool:
         """
         If any of the request type filters are set to true,
@@ -216,6 +218,45 @@ def get_connection_types(
             action_type in template.supported_actions for action_type in action_types
         )
 
+    saas_connection_types: list[ConnectionSystemTypeMap] = []
+    saas_types: list[str] = sorted(
+        [
+            saas_type
+            for saas_type in ConnectorRegistry.connector_types()
+            if _is_match(saas_type, search) and saas_request_type_filter(saas_type)
+        ]
+    )
+
+    for item in saas_types:
+        connector_template = ConnectorRegistry.get_connector_template(item)
+        if connector_template is not None:
+            saas_connection_types.append(
+                ConnectionSystemTypeMap(
+                    identifier=item,
+                    type=SystemType.saas,
+                    human_readable=connector_template.human_readable,
+                    encoded_icon=connector_template.icon,
+                    authorization_required=connector_template.authorization_required,
+                    user_guide=connector_template.user_guide,
+                    supported_actions=connector_template.supported_actions,
+                )
+            )
+
+    return saas_connection_types
+
+
+# FIXME: this function needs a refactor
+def get_connection_types(
+    search: str | None = None,
+    system_type: SystemType | None = None,
+    action_types: Set[ActionType] = SUPPORTED_ACTION_TYPES,
+    include_test_connections: bool = False,
+) -> list[ConnectionSystemTypeMap]:
+    """
+    Returns a list of ConnectionSystemTypeMap objects that match the given search and system type.
+
+    If include_test_connections is True, test connections like test_website will be included in the response.
+    """
     connection_system_types: list[ConnectionSystemTypeMap] = []
     if (system_type == SystemType.database or system_type is None) and (
         ActionType.access in action_types or ActionType.erasure in action_types
@@ -238,7 +279,7 @@ def get_connection_types(
                     ConnectionType.sovrn,
                     ConnectionType.test_website,
                 ]
-                and is_match(conn_type.value)
+                and _is_match(conn_type.value, search)
             ],
             key=lambda x: x.value,
         )
@@ -254,28 +295,10 @@ def get_connection_types(
             ]
         )
     if system_type == SystemType.saas or system_type is None:
-        saas_types: list[str] = sorted(
-            [
-                saas_type
-                for saas_type in ConnectorRegistry.connector_types()
-                if is_match(saas_type) and saas_request_type_filter(saas_type)
-            ]
+        saas_connection_types = get_saas_connection_types(
+            action_types=action_types, search=search
         )
-
-        for item in saas_types:
-            connector_template = ConnectorRegistry.get_connector_template(item)
-            if connector_template is not None:
-                connection_system_types.append(
-                    ConnectionSystemTypeMap(
-                        identifier=item,
-                        type=SystemType.saas,
-                        human_readable=connector_template.human_readable,
-                        encoded_icon=connector_template.icon,
-                        authorization_required=connector_template.authorization_required,
-                        user_guide=connector_template.user_guide,
-                        supported_actions=connector_template.supported_actions,
-                    )
-                )
+        connection_system_types.extend(saas_connection_types)
 
     if (system_type == SystemType.manual or system_type is None) and (
         ActionType.access in action_types or ActionType.erasure in action_types
@@ -285,7 +308,7 @@ def get_connection_types(
                 manual_type.value
                 for manual_type in ConnectionType
                 if manual_type == ConnectionType.manual_webhook
-                and is_match(manual_type.value)
+                and _is_match(manual_type.value, search)
             ]
         )
         connection_system_types.extend(
@@ -307,7 +330,7 @@ def get_connection_types(
                 for email_type in ConnectionType
                 if email_type
                 in ERASURE_EMAIL_CONNECTOR_TYPES + CONSENT_EMAIL_CONNECTOR_TYPES
-                and is_match(email_type.value)
+                and _is_match(email_type.value, search)
                 and (  # include consent or erasure connectors if requested, respectively
                     (
                         ActionType.consent in action_types
@@ -339,4 +362,16 @@ def get_connection_types(
             ]
         )
 
+    if include_test_connections and (
+        system_type == SystemType.website or system_type is None
+    ):
+        connection_system_types.append(
+            ConnectionSystemTypeMap(
+                identifier=ConnectionType.test_website.value,
+                type=SystemType.website,
+                human_readable=ConnectionType.test_website.human_readable,
+                supported_actions=[],
+            )
+        )
+
     return connection_system_types

fides/config/execution_settings.py

@@ -61,6 +61,10 @@ class ExecutionSettings(FidesSettings):
         default=3,
         description="The number of times a privacy request will be requeued when its tasks are interrupted before being marked as error",
     )
+    async_tasks_status_polling_interval_seconds: int = Field(
+        default=3600,
+        description="Seconds between polling for async tasks to requeue",
+    )
     use_dsr_3_0: bool = Field(
         default=False,
         description="Temporary flag to switch to using DSR 3.0 to process your tasks.",

fides/data/sample_project/docker-compose.yml

@@ -87,7 +87,7 @@ services:
 
   fides-db:
     container_name: fides-db
-    image: postgres:12
+    image: postgres:16
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 15s
@@ -104,14 +104,14 @@ services:
 
   redis:
     container_name: fides-redis
-    image: redis:6.2-alpine
+    image: redis:8.0-alpine
     command: redis-server --requirepass redispassword
     ports:
       - "7379:6379"
 
   postgres-test:
     container_name: fides-postgres-example-db
-    image: postgres:12
+    image: postgres:16
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 15s

fides/service/privacy_request/privacy_request_service.py

@@ -7,6 +7,7 @@ from sqlalchemy.orm import Session
 from fides.api.common_exceptions import (
     FidesopsException,
     MessageDispatchException,
+    PrivacyRequestError,
     RedisNotConfigured,
 )
 from fides.api.models.audit_log import AuditLog, AuditLogAction
@@ -51,15 +52,6 @@ from fides.service.messaging.messaging_service import (
 )
 
 
-class PrivacyRequestError(Exception):
-    """Base exception for privacy request operations."""
-
-    def __init__(self, message: str, data: Optional[Dict] = None):
-        self.message = message
-        self.data = data
-        super().__init__(message)
-
-
 class PrivacyRequestService:
     def __init__(
         self,

fides/service/taxonomy/handlers/__init__.py

@@ -0,0 +1,11 @@
+"""
+Taxonomy handlers package.
+"""
+
+from .base import TaxonomyHandler
+from .legacy_handler import LegacyTaxonomyHandler
+
+__all__ = [
+    "TaxonomyHandler",
+    "LegacyTaxonomyHandler",
+]

fides/service/taxonomy/handlers/base.py

@@ -0,0 +1,42 @@
+"""
+Base taxonomy handler and core validation functions.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Type
+
+from sqlalchemy.orm import Session
+
+from fides.api.db.base_class import FidesBase
+
+
+class TaxonomyHandler(ABC):
+    """Abstract handler for taxonomy CRUD operations."""
+
+    def __init__(self, db: Session, taxonomy_type: str):
+        self.db = db
+        self.taxonomy_type = taxonomy_type
+
+    @abstractmethod
+    def get_model(self) -> Type[FidesBase]:
+        pass
+
+    @abstractmethod
+    def get_elements(self, active_only: bool, parent_key: Optional[str]) -> List[Any]:
+        pass
+
+    @abstractmethod
+    def get_element(self, fides_key: str) -> Optional[Any]:
+        pass
+
+    @abstractmethod
+    def create_element(self, element_data: Dict) -> Any:
+        pass
+
+    @abstractmethod
+    def update_element(self, fides_key: str, element_data: Dict) -> Optional[Any]:
+        pass
+
+    @abstractmethod
+    def delete_element(self, fides_key: str) -> None:
+        pass

fides/service/taxonomy/handlers/legacy_handler.py

@@ -0,0 +1,95 @@
+"""
+Legacy taxonomy handler for DataCategory, DataUse, and DataSubject models.
+"""
+
+from typing import Dict, List, Optional, Type, Union
+
+from sqlalchemy.orm import Query, Session
+
+from fides.api.models.sql_models import (  # type:ignore[attr-defined]
+    DataCategory,
+    DataSubject,
+    DataUse,
+)
+
+from .base import TaxonomyHandler
+
+
+class LegacyTaxonomyHandler(TaxonomyHandler):
+    """Handler for legacy taxonomy models (DataCategory, DataUse, DataSubject)."""
+
+    def __init__(self, db: Session, taxonomy_type: str) -> None:
+        super().__init__(db, taxonomy_type)
+        self.models = {
+            "data_categories": DataCategory,
+            "data_uses": DataUse,
+            "data_subjects": DataSubject,
+        }
+        if taxonomy_type not in self.models:
+            raise ValueError(
+                f"Taxonomy type '{taxonomy_type}' not supported. Supported types: {list(self.models.keys())}"
+            )
+
+    def get_elements(
+        self, active_only: bool, parent_key: Optional[str]
+    ) -> List[Union[DataCategory, DataUse, DataSubject]]:
+        model_class = self.get_model()
+        query = self._build_query(model_class, active_only, parent_key)
+        return query.all()
+
+    def get_element(
+        self, fides_key: str
+    ) -> Optional[Union[DataCategory, DataUse, DataSubject]]:
+        model_class = self.get_model()
+        element = (
+            self.db.query(model_class)
+            .filter(model_class.fides_key == fides_key)
+            .first()
+        )
+        return element
+
+    def create_element(
+        self, element_data: Dict
+    ) -> Union[DataCategory, DataUse, DataSubject]:
+        model_class = self.get_model()
+        element = model_class.create(self.db, data=element_data)
+        return element
+
+    def update_element(
+        self, fides_key: str, element_data: Dict
+    ) -> Optional[Union[DataCategory, DataUse, DataSubject]]:
+        model_class = self.get_model()
+        element = (
+            self.db.query(model_class)
+            .filter(model_class.fides_key == fides_key)
+            .first()
+        )
+
+        if not element:
+            return None
+
+        return element.update(db=self.db, data=element_data)
+
+    def delete_element(self, fides_key: str) -> None:
+        model_class = self.get_model()
+        element = (
+            self.db.query(model_class)
+            .filter(model_class.fides_key == fides_key)
+            .first()
+        )
+
+        if element:
+            self.db.delete(element)
+
+    def get_model(self) -> Type[Union[DataCategory, DataUse, DataSubject]]:
+        return self.models[self.taxonomy_type]
+
+    def _build_query(
+        self, model_class: Type, active_only: bool, parent_key: Optional[str]
+    ) -> Query:
+        query = self.db.query(model_class)
+        if active_only:
+            query = query.filter(model_class.active.is_(True))
+        if parent_key and hasattr(model_class, "parent_key"):
+            query = query.filter(model_class.parent_key == parent_key)
+        return query