ethyca-fides 2.69.3b0__py2.py3-none-any.whl → 2.69.3b1__py2.py3-none-any.whl

fides/api/models/system_group.py

@@ -0,0 +1,85 @@
+from enum import Enum
+
+from citext import CIText
+from sqlalchemy import ARRAY, Column
+from sqlalchemy import Enum as EnumColumn
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.fides_user import FidesUser
+from fides.api.models.sql_models import System  # type: ignore[attr-defined]
+
+
+class CustomTaxonomyColor(str, Enum):
+    WHITE = "taxonomy_white"
+    RED = "taxonomy_red"
+    ORANGE = "taxonomy_orange"
+    YELLOW = "taxonomy_yellow"
+    GREEN = "taxonomy_green"
+    BLUE = "taxonomy_blue"
+    PURPLE = "taxonomy_purple"
+    SANDSTONE = "sandstone"
+    MINOS = "minos"
+
+
+class SystemGroup(Base):
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "system_group"
+
+    fides_key = Column(
+        String,
+        ForeignKey("taxonomy_element.fides_key", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+        index=True,
+    )
+    label_color = Column(
+        EnumColumn(CustomTaxonomyColor, values_callable=lambda x: [i.value for i in x]),
+        nullable=True,
+    )
+    data_steward_username = Column(
+        CIText,
+        ForeignKey(FidesUser.username, ondelete="SET NULL"),
+        index=True,
+        nullable=True,
+    )
+    data_uses = Column(
+        ARRAY(String),
+        nullable=False,
+        server_default="{}",
+        default=list,
+    )  # a list of `fides_key`s of `DataUse` records
+    data_steward = relationship(
+        "FidesUser", foreign_keys=[data_steward_username], lazy="selectin"
+    )
+
+    systems = relationship(
+        System,
+        secondary="system_group_member",
+        back_populates="system_groups",
+        viewonly=True,
+    )
+
+
+class SystemGroupMember(Base):
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "system_group_member"
+
+    system_group_id = Column(
+        String(255), ForeignKey("system_group.id", ondelete="CASCADE"), index=True
+    )
+    system_id = Column(
+        String(255), ForeignKey("ctl_systems.id", ondelete="CASCADE"), index=True
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "system_group_id",
+            "system_id",
+            name="uq_system_group_member_group_system",
+        ),
+    )

fides/api/schemas/saas/saas_config.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Any, Dict, List, Optional, Set, Union
 
 from fideslang.models import FidesCollectionKey, FidesDatasetReference
@@ -69,7 +68,7 @@ class Strategy(BaseModel):
     """General shape for swappable strategies (ex: auth, processors, pagination, etc.)"""
 
     strategy: str
-    configuration: Dict[str, Any]
+    configuration: Optional[Dict[str, Any]] = None
 
 
 class ClientConfig(BaseModel):
@@ -90,20 +89,6 @@ class QueryParam(BaseModel):
     value: Union[int, str]
 
 
-class AsyncStrategy(Enum):
-    """Async strategies: supports callback for now, but could be
-    extended to polling as well in the future"""
-
-    callback = "callback"
-
-
-class AsyncConfig(BaseModel):
-    """Async config only has strategy for now, but could be
-    extended with other configuration options when we add polling"""
-
-    strategy: AsyncStrategy
-
-
 class SaaSRequest(BaseModel):
     """
     A single request with static or dynamic path, headers, query, and body params.
@@ -126,7 +111,7 @@ class SaaSRequest(BaseModel):
     grouped_inputs: Optional[List[str]] = []
     ignore_errors: Optional[Union[bool, List[int]]] = False
     rate_limit_config: Optional[RateLimitConfig] = None
-    async_config: Optional[AsyncConfig] = None
+    async_config: Optional[Strategy] = None
     skip_missing_param_values: Optional[bool] = (
         False  # Skip instead of raising an exception if placeholders can't be populated in body
     )

fides/api/schemas/saas/strategy_configuration.py

@@ -178,3 +178,15 @@ class OAuth2ClientCredentialsConfiguration(OAuth2BaseConfiguration):
     """
 
     refresh_request: Optional[SaaSRequest] = Field(exclude=True)
+
+
+class PollingAsyncDSRConfiguration(StrategyConfiguration):
+    """
+    Configuration for polling async DSR requests.
+    """
+
+    status_request: SaaSRequest
+    status_path: str
+    status_completed_value: Optional[str] = None
+    result_request: SaaSRequest
+    result_path: str

fides/api/schemas/system.py

@@ -2,6 +2,7 @@ from datetime import datetime
 from typing import Any, Dict, List, Optional, Sequence
 
 from fideslang.models import PrivacyDeclaration, System
+from fideslang.validation import FidesKey
 from pydantic import ConfigDict, Field
 from pydantic.main import BaseModel
 
@@ -61,3 +62,10 @@ class SystemHistoryResponse(BaseModel):
     after: Dict[str, Any]
     created_at: datetime
     model_config = ConfigDict(from_attributes=True)
+
+
+class AssignStewardRequest(BaseModel):
+    """Request schema for assigning a data steward to one or more systems."""
+
+    data_steward: str
+    system_keys: List[FidesKey]

fides/api/service/async_dsr/async_dsr_service.py

@@ -1,3 +1,5 @@
+from typing import Any, Dict
+
 from loguru import logger
 from sqlalchemy.orm import Session
 
@@ -5,7 +7,22 @@ from fides.api.common_exceptions import PrivacyRequestError
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.models.worker_task import ExecutionLogStatus
+from fides.api.schemas.policy import ActionType
 from fides.api.schemas.privacy_request import PrivacyRequestStatus
+from fides.api.service.async_dsr.async_dsr_strategy import AsyncDSRStrategy
+from fides.api.service.async_dsr.async_dsr_strategy_polling import (
+    PollingAsyncDSRStrategy,
+)
+from fides.api.service.connectors.query_configs.saas_query_config import SaaSQueryConfig
+from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
+from fides.api.service.connectors.saas_connector import SaaSConnector
+from fides.api.task.execute_request_tasks import (
+    create_graph_task,
+    log_task_queued,
+    queue_request_task,
+)
+from fides.api.task.graph_task import GraphTask
 from fides.api.task.task_resources import TaskResources
 
 
@@ -16,6 +33,9 @@ def requeue_polling_request(
     """Re-queue a Privacy request that polls async tasks for a given privacy request"""
     # Check that the privacy request is approved or in processing
     privacy_request: PrivacyRequest = async_task.privacy_request
+    logger.info(
+        f"Privacy request {privacy_request.id} status: {privacy_request.status.value}"
+    )
 
     if privacy_request.status not in [
         PrivacyRequestStatus.approved,
@@ -41,16 +61,23 @@ def requeue_polling_request(
         async_task,
         db,
     ) as resources:
-        # graph_task: GraphTask = create_graph_task(db, async_task, resources)
-        # Currently, upstream tasks and "input keys" (which are built by data dependencies)
-        # are the same, but they may not be the same in the future.
-        # upstream_tasks = async_task.upstream_tasks_objects(db)
-        # upstream_access_data: List[List[Row]] = _build_upstream_access_data(
-        #    graph_task.execution_node.input_keys, upstream_tasks
-        # )
-        # TODO: Implement the polling strategy
-        logger.info(f"found resources: {resources}")
-        return None
+        graph_task: GraphTask = create_graph_task(db, async_task, resources)
+
+        saas_connector: SaaSConnector = graph_task.connector  # type: ignore
+        saas_connector.set_privacy_request_state(
+            privacy_request,
+            graph_task.execution_node,
+            async_task,
+        )
+        query_config: SaaSQueryConfig = saas_connector.query_config(
+            graph_task.execution_node
+        )  # type: ignore
+        if async_task.action_type == ActionType.access:
+            logger.info(f"Executing read polling requests for {async_task.id}")
+
+            execute_read_polling_requests(db, async_task, query_config, saas_connector)
+        elif async_task.action_type == ActionType.erasure:
+            execute_erasure_polling_requests(db, async_task, query_config)
 
 
 def get_connection_config_from_task(
@@ -64,6 +91,7 @@ def get_connection_config_from_task(
         raise PrivacyRequestError(
             f"DatasetConfig with fides_key {request_task.dataset_name} not found."
         )
+
     connection_config = ConnectionConfig.get(
         db=db, object_id=dataset_config.connection_config_id
     )
@@ -73,3 +101,95 @@ def get_connection_config_from_task(
         )
 
     return connection_config
+
+
+def execute_read_polling_requests(
+    db: Session,
+    async_task: RequestTask,
+    query_config: SaaSQueryConfig,
+    connector: SaaSConnector,
+) -> None:
+    """Execute the read polling requests for a given privacy request"""
+    read_requests = query_config.get_read_requests_by_identity()
+    logger.info(f"Read requests: {read_requests}")
+    for read_request in read_requests:
+        if read_request.async_config:
+            strategy: PollingAsyncDSRStrategy = AsyncDSRStrategy.get_strategy(  # type: ignore
+                read_request.async_config.strategy,
+                read_request.async_config.configuration,
+            )
+            client: AuthenticatedClient = connector.create_client()
+
+            # Get missing parameters from available context
+            privacy_request: PrivacyRequest = async_task.privacy_request
+            secrets: Dict[str, Any] = connector.secrets
+            identity_data = {
+                **privacy_request.get_persisted_identity().labeled_dict(),
+                **privacy_request.get_cached_identity_data(),
+            }
+
+            status = strategy.get_status_request(
+                client,
+                secrets,
+                identity_data,
+            )
+
+            if status:
+                execute_read_result_request(
+                    db,
+                    async_task,
+                    strategy,
+                    client,
+                    secrets,
+                    identity_data,
+                )
+            else:
+                logger.info(f"Polling request - {async_task.id} still not Ready. ")
+                continue
+
+
+def execute_read_result_request(
+    db: Session,
+    async_task: RequestTask,
+    strategy: PollingAsyncDSRStrategy,
+    client: AuthenticatedClient,
+    secrets: Dict[str, Any],
+    identity_data: Dict[str, Any],
+) -> None:
+
+    result = strategy.get_result_request(
+        client,
+        secrets,
+        identity_data,
+    )
+    logger.info(f"Result: {result}")
+    if result:
+        # Save updated data back to the request task.
+        async_task.access_data = result
+        logger.info(
+            f"Polling request - {async_task.id} is ready. Added {len(result) if isinstance(result, list) else 1} results"
+        )
+
+    else:
+        async_task.access_data = []
+        logger.info(
+            f"Polling request - {async_task.id} is ready but returned no results"
+        )
+
+    async_task.callback_succeeded = (
+        True  # Setting this task as successful, so it wont loop anymore
+    )
+    async_task.update_status(db, ExecutionLogStatus.pending)
+    async_task.save(db)
+    log_task_queued(async_task, "callback")
+    queue_request_task(async_task, privacy_request_proceed=True)
+
+
+def execute_erasure_polling_requests(
+    db: Session,
+    async_task: RequestTask,
+    query_config: SaaSQueryConfig,
+) -> None:
+    """Execute the erasure polling requests for a given privacy request"""
+    # TODO: Implement erasure polling logic
+    logger.info(f"Erasure polling not yet implemented. Task {async_task.id} passed")

fides/api/service/async_dsr/async_dsr_strategy.py

@@ -0,0 +1,5 @@
+from fides.api.service.strategy import Strategy
+
+
+class AsyncDSRStrategy(Strategy):
+    """Abstract base class for async DSR strategies"""

fides/api/service/async_dsr/async_dsr_strategy_callback.py

@@ -0,0 +1,16 @@
+from fides.api.service.async_dsr.async_dsr_strategy import AsyncDSRStrategy
+
+
+class CallbackAsyncDSRStrategy(AsyncDSRStrategy):
+    """
+    Strategy for callback async DSR requests.
+
+    This strategy has no methods since the callback is handled by the callback endpoint
+    we still require the strategy to be defined to notify that the request is awaiting async processing
+    and follows the AsyncDSRStrategy interface.
+    """
+
+    name = "callback"
+
+    def __init__(self) -> None:
+        pass

fides/api/service/async_dsr/async_dsr_strategy_factory.py

@@ -0,0 +1,63 @@
+from enum import Enum
+from typing import Any, Dict, List
+
+from pydantic import ValidationError
+
+from fides.api.common_exceptions import NoSuchStrategyException
+from fides.api.common_exceptions import ValidationError as FidesopsValidationError
+from fides.api.schemas.saas.strategy_configuration import StrategyConfiguration
+from fides.api.service.async_dsr.async_dsr_strategy import AsyncDSRStrategy
+from fides.api.service.async_dsr.async_dsr_strategy_callback import (
+    CallbackAsyncDSRStrategy,
+)
+from fides.api.service.async_dsr.async_dsr_strategy_polling import (
+    PollingAsyncDSRStrategy,
+)
+
+
+class SupportedAsyncDSRStrategies(Enum):
+    """
+    The supported async DSR strategies for handling asynchronous data subject requests.
+    """
+
+    callback = CallbackAsyncDSRStrategy
+    polling = PollingAsyncDSRStrategy
+
+    @classmethod
+    def __contains__(cls, item: str) -> bool:
+        try:
+            cls[item]
+        except KeyError:
+            return False
+
+        return True
+
+
+def get_strategy(
+    strategy_name: str,
+    configuration: Dict[str, Any],
+) -> AsyncDSRStrategy:
+    """
+    Returns the strategy given the name and configuration.
+    Raises NoSuchStrategyException if the strategy does not exist
+    """
+    if not SupportedAsyncDSRStrategies.__contains__(strategy_name):
+        valid_strategies = ", ".join(get_strategy_names())
+        raise NoSuchStrategyException(
+            f"Strategy '{strategy_name}' does not exist. Valid strategies are [{valid_strategies}]"
+        )
+    strategy = SupportedAsyncDSRStrategies[strategy_name].value
+    if strategy.configuration_model is None:
+        return strategy()
+    try:
+        strategy_config: StrategyConfiguration = strategy.configuration_model(
+            **configuration
+        )
+        return strategy(configuration=strategy_config)
+    except ValidationError as e:
+        raise FidesopsValidationError(message=str(e))
+
+
+def get_strategy_names() -> List[str]:
+    """Returns all supported async DSR strategies"""
+    return [s.name for s in SupportedAsyncDSRStrategies]

fides/api/service/async_dsr/async_dsr_strategy_polling.py

@@ -0,0 +1,94 @@
+from typing import Any, Dict, List
+
+import pydash
+from requests import Response
+
+from fides.api.common_exceptions import PrivacyRequestError
+from fides.api.schemas.saas.strategy_configuration import PollingAsyncDSRConfiguration
+from fides.api.service.async_dsr.async_dsr_strategy import AsyncDSRStrategy
+from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
+from fides.api.util.collection_util import Row
+from fides.api.util.saas_util import map_param_values
+
+
+class PollingAsyncDSRStrategy(AsyncDSRStrategy):
+    """
+    Strategy for polling async DSR requests.
+    """
+
+    name = "polling"
+    configuration_model = PollingAsyncDSRConfiguration
+
+    def __init__(self, configuration: PollingAsyncDSRConfiguration):
+        self.status_request = configuration.status_request
+        self.status_path = configuration.status_path
+        self.status_completed_value = configuration.status_completed_value
+        self.result_request = configuration.result_request
+        self.result_path = configuration.result_path
+
+    def get_status_request(
+        self,
+        client: AuthenticatedClient,
+        secrets: Dict[str, Any],
+        identity_data: Dict[str, Any],
+    ) -> bool:
+        """Executes the status requests, and move forward if its true"""
+        param_values = secrets.copy()
+        param_values.update(identity_data)
+        prepared_status_request = map_param_values(
+            "status", "polling request", self.status_request, param_values
+        )
+
+        response: Response = client.send(prepared_status_request)
+
+        if response.ok:
+            status_path_value = pydash.get(response.json(), self.status_path)
+            # If the status path value is a boolean, return it
+            if isinstance(status_path_value, bool):
+                return status_path_value
+
+            # if the status path is a string, check if its the expected string value
+            if isinstance(status_path_value, str):
+                if status_path_value == self.status_completed_value:
+                    return True
+
+                return False
+
+            # if the status path is a list, check if the first element is the expected string value
+            if isinstance(status_path_value, list):
+                if status_path_value[0] == self.status_completed_value:
+                    return True
+
+                return False
+            # And if we cant recognize the type, raise an error
+            raise PrivacyRequestError(
+                f"Status request returned an unexpected value: {status_path_value}"
+            )
+
+        raise PrivacyRequestError(
+            f"Status request failed with status code {response.status_code}"
+        )
+
+    def get_result_request(
+        self,
+        client: AuthenticatedClient,
+        secrets: Dict[str, Any],
+        identity_data: Dict[str, Any],
+    ) -> List[Row]:
+        """Build request to get the result of the async DSR process"""
+        param_values = secrets.copy()
+        param_values.update(identity_data)
+        prepared_result_request = map_param_values(
+            "result",
+            "polling request",
+            self.result_request,
+            param_values,  # type: ignore
+        )
+        response: Response = client.send(prepared_result_request)
+        if response.ok:
+            result = pydash.get(response.json(), self.result_path)
+            return result
+
+        raise PrivacyRequestError(
+            f"Result request failed with status code {response.status_code}"
+        )

fides/api/service/connectors/saas_connector.py

@@ -14,12 +14,14 @@ from fides.api.common_exceptions import (
     AwaitingAsyncTask,
     FidesopsException,
     PostProcessingException,
+    PrivacyRequestError,
     SkippingConsentPropagation,
 )
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.models.privacy_request.request_task import AsyncTaskType
 from fides.api.schemas.consentable_item import (
     ConsentableItem,
     build_consent_item_hierarchy,
@@ -27,7 +29,6 @@ from fides.api.schemas.consentable_item import (
 from fides.api.schemas.limiter.rate_limit_config import RateLimitConfig
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.saas.saas_config import (
-    AsyncStrategy,
     ClientConfig,
     ConsentRequestMap,
     ParamValue,
@@ -264,17 +265,29 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             input_data[CUSTOM_PRIVACY_REQUEST_FIELDS] = [custom_privacy_request_fields]
 
         rows: List[Row] = []
-        awaiting_async_callback: bool = False
+        awaiting_async_processing: bool = False
+
         for read_request in read_requests:
             self.set_saas_request_state(read_request)
             if (
-                read_request.async_config
-                and read_request.async_config.strategy == AsyncStrategy.callback
+                read_request.async_config is not None
                 and request_task.id  # Only supported in DSR 3.0
             ):
                 # Asynchronous read request detected. We will exit below and put the
                 # Request Task in an "awaiting_processing" status.
-                awaiting_async_callback = True
+                awaiting_async_processing = True
+
+                # Validate async strategy with proper enum value checking
+                strategy_value = read_request.async_config.strategy
+                valid_strategies = [task_type.value for task_type in AsyncTaskType]
+
+                if strategy_value not in valid_strategies:
+                    raise PrivacyRequestError(
+                        f"Invalid async type '{strategy_value}' for request task {request_task.id}. "
+                        f"Valid types are: {valid_strategies}"
+                    )
+
+                request_task.async_type = AsyncTaskType(strategy_value)
 
             # check all the values specified by param_values are provided in input_data
             if self._missing_dataset_reference_values(
@@ -332,9 +345,14 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                 )
 
         self.unset_connector_state()
-        if awaiting_async_callback:
+        if awaiting_async_processing:
             # If a read request was marked to expect async results, original response data here is ignored.
             # We'll instead use the data received in the callback URL later.
+            # However for polling async request we want to save the request data for ids that we will use on the pollings status
+            if request_task.async_type == AsyncTaskType.polling:
+                # Saving the request task access data to use it on the polling status request
+                # TODO: Consider if we want to clean up the rows. Currently this is the concern of the GraphTask.
+                request_task.access_data = rows
             # Raising an AwaitingAsyncTask to put this task in an awaiting_processing state
             raise AwaitingAsyncTask()
 
@@ -633,7 +651,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
 
         awaiting_async_callback: bool = bool(
             masking_request.async_config
-            and masking_request.async_config.strategy == AsyncStrategy.callback
+            and masking_request.async_config.strategy == AsyncTaskType.callback.value
         ) and bool(
             request_task.id
         )  # Only supported in DSR 3.0

fides/api/service/deps.py

@@ -1,13 +1,16 @@
 from fastapi import Depends
+from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session
 
 from fides.api.api.deps import get_config, get_config_proxy, get_db
+from fides.api.db.ctl_session import get_async_db
 from fides.config import FidesConfig
 from fides.config.config_proxy import ConfigProxy
 from fides.service.dataset.dataset_config_service import DatasetConfigService
 from fides.service.dataset.dataset_service import DatasetService
 from fides.service.messaging.messaging_service import MessagingService
 from fides.service.privacy_request.privacy_request_service import PrivacyRequestService
+from fides.service.system.system_service import SystemService
 from fides.service.taxonomy.taxonomy_service import TaxonomyService
 from fides.service.user.user_service import UserService
 
@@ -46,3 +49,7 @@ def get_user_service(
 
 def get_taxonomy_service(db: Session = Depends(get_db)) -> TaxonomyService:
     return TaxonomyService(db)
+
+
+def get_system_service(db: AsyncSession = Depends(get_async_db)) -> SystemService:
+    return SystemService(db)