ethyca-fides 2.64.3b0__py2.py3-none-any.whl → 2.64.5rc1__py2.py3-none-any.whl

fides/api/schemas/manual_tasks/manual_task_status.py

@@ -0,0 +1,151 @@
+from datetime import datetime, timezone
+from enum import Enum as EnumType
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+
+class StatusTransitionNotAllowed(Exception):
+    """Exception raised when a status transition is not allowed."""
+
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class StatusType(str, EnumType):
+    """Enum for manual task status."""
+
+    pending = "pending"
+    in_progress = "in_progress"
+    completed = "completed"
+    failed = "failed"
+
+    @classmethod
+    def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
+        """Get valid transitions from the current status.
+
+        Args:
+            current_status: The current status
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        if current_status == cls.pending:
+            return [cls.in_progress, cls.failed, cls.completed]
+        if current_status == cls.in_progress:
+            return [cls.completed, cls.failed]
+        if current_status == cls.completed:
+            return []
+        if current_status == cls.failed:
+            return [cls.pending, cls.in_progress]
+        return []
+
+
+class StatusTransitionMixin:
+    """Mixin for handling status transitions.
+
+    This mixin provides methods for managing status transitions and completion tracking.
+    It can be used by any model that needs status management.
+    """
+
+    # These should be overridden by the implementing class
+    status: StatusType
+    completed_at: Optional[datetime]
+    completed_by_id: Optional[str]
+
+    def _get_valid_transitions(self) -> list[StatusType]:
+        """Get valid transitions from the current status.
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        return StatusType.get_valid_transitions(self.status)
+
+    def _validate_status_transition(self, new_status: StatusType) -> None:
+        """Validate that a status transition is allowed.
+
+        Args:
+            new_status: The new status to transition to
+
+        Raises:
+            StatusTransitionNotAllowed: If the transition is not allowed
+        """
+        # Don't allow transitions to the same status
+        if new_status == self.status:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition: already in status {new_status}"
+            )
+
+        # Get valid transitions for current status
+        valid_transitions = self._get_valid_transitions()
+        if new_status not in valid_transitions:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition from {self.status} to {new_status}. "
+                f"Valid transitions are: {valid_transitions}"
+            )
+
+    def update_status(
+        self, db: Session, new_status: StatusType, user_id: Optional[str] = None
+    ) -> None:
+        """Update the status with validation and completion handling.
+
+        Args:
+            db: Database session
+            new_status: New status to set
+            user_id: Optional user ID who is making the change
+        """
+        self._validate_status_transition(new_status)
+
+        if new_status == StatusType.completed:
+            self.completed_at = datetime.now(timezone.utc)
+            self.completed_by_id = user_id
+        elif new_status == StatusType.pending:
+            # Reset completion fields if going back to pending
+            self.completed_at = None
+            self.completed_by_id = None
+
+        self.status = new_status
+        db.add(self)
+        db.commit()
+
+    def mark_completed(self, db: Session, user_id: str) -> None:
+        """Mark as completed.
+
+        Args:
+            db: Database session
+            user_id: user ID who completed the task
+        """
+        self.update_status(db, StatusType.completed, user_id)
+
+    def mark_failed(self, db: Session) -> None:
+        """Mark as failed."""
+        self.update_status(db, StatusType.failed)
+
+    def start_progress(self, db: Session) -> None:
+        """Mark as in progress."""
+        self.update_status(db, StatusType.in_progress)
+
+    def reset_to_pending(self, db: Session) -> None:
+        """Reset to pending status."""
+        self.update_status(db, StatusType.pending)
+
+    @property
+    def is_completed(self) -> bool:
+        """Check if completed."""
+        return self.status == StatusType.completed
+
+    @property
+    def is_failed(self) -> bool:
+        """Check if failed."""
+        return self.status == StatusType.failed
+
+    @property
+    def is_in_progress(self) -> bool:
+        """Check if in progress."""
+        return self.status == StatusType.in_progress
+
+    @property
+    def is_pending(self) -> bool:
+        """Check if pending."""
+        return self.status == StatusType.pending

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -5,9 +5,8 @@ from fideslang.models import MaskingStrategies
 from loguru import logger
 from sqlalchemy import MetaData, Table, text
 from sqlalchemy.engine import Engine
-from sqlalchemy.sql import Delete, Update
+from sqlalchemy.sql import Delete, Update  # type: ignore
 from sqlalchemy.sql.elements import ColumnElement, TextClause
-from sqlalchemy_bigquery import BigQueryDialect
 
 from fides.api.graph.config import Field, FieldPath
 from fides.api.graph.execution import ExecutionNode
@@ -16,8 +15,6 @@ from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.namespace_meta.bigquery_namespace_meta import (
     BigQueryNamespaceMeta,
 )
-from fides.api.schemas.partitioning import TIME_BASED_REQUIRED_KEYS, combine_partitions
-from fides.api.schemas.partitioning.time_based_partitioning import TimeBasedPartitioning
 from fides.api.service.connectors.query_configs.query_config import (
     QueryStringWithoutTuplesOverrideQueryConfig,
 )
@@ -39,59 +36,40 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
     namespace_meta_schema = BigQueryNamespaceMeta
 
     @property
-    def partitioning(
-        self,
-    ) -> Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]:
-        # Overridden from base implementation to allow for _only_ BQ partitioning, for now
+    def partitioning(self) -> Optional[Dict]:
+        # Overriden from base implementation to allow for _only_ BQ partitioning, for now
         return self.node.collection.partitioning
 
     def get_partition_clauses(
         self,
     ) -> List[str]:
         """
-        Build a list of SQL `WHERE` clause strings for the collection's partitioning
-        configuration.
+        Returns the WHERE clauses specified in the partitioning spec
 
-        Supported modes:
-        1. `Legacy static` - A list of `where_clauses`.
-        2. `Single time-based` - A single time-based partitioning spec
-        3. `Multiple time-based` - A list of time-based partitioning specs, with different intervals
+        Currently, only where-clause based partitioning is supported.
 
-        Any other combination (e.g. mixing modes or missing keys) raises a
-        `ValueError` so that mis-configurations surface early.
-        """
+        TODO: derive partitions from a start/end/interval specification
 
-        partition_spec: Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]] = (
-            self.partitioning
-        )
+
+        NOTE: when we deprecate `where_clause` partitioning in favor of a more proper partitioning DSL,
+        we should be sure to still support the existing `where_clause` partition definition on
+        any in-progress DSRs so that they can run through to completion.
+        """
+        partition_spec = self.partitioning
         if not partition_spec:
-            logger.warning(
-                f"No partitioning specification found for node '{self.node.address}', skipping partition clauses"
+            logger.error(
+                "Partitioning clauses cannot be retrieved, no partitioning specification found"
             )
             return []
 
-        # Legacy mode using `where_clauses`
-        if isinstance(partition_spec, dict) and "where_clauses" in partition_spec:
-            if any(k in partition_spec for k in TIME_BASED_REQUIRED_KEYS):
-                # Mixed mode not allowed
-                raise ValueError(
-                    "Partitioning spec cannot define both `where_clauses` and time-based partitioning."
-                )
-            where = partition_spec.get("where_clauses") or []
-            if not where:
-                raise ValueError("`where_clauses` must be a non-empty list.")
-            return where
-
-        # Combine and de-duplicate boundary rows
-        combined_expressions = combine_partitions(partition_spec)  # type: ignore
-
-        dialect = BigQueryDialect()
-        where_clauses = [
-            str(expr.compile(dialect=dialect, compile_kwargs={"literal_binds": True}))
-            for expr in combined_expressions
-        ]
+        if where_clauses := partition_spec.get("where_clauses"):
+            return where_clauses
 
-        return where_clauses
+        # TODO: implement more advanced partitioning support!
+
+        raise ValueError(
+            "`where_clauses` must be specified in partitioning specification!"
+        )
 
     def _generate_table_name(self) -> str:
         """
@@ -169,7 +147,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         nested_result = unflatten_dict(merged_dict)
 
         # 5. Replace any arrays containing only None values with empty arrays
-        nested_result = replace_none_arrays(nested_result)
+        nested_result = replace_none_arrays(nested_result)  # type: ignore
 
         # 6. Only keep top-level keys that are in the update_value_map
         top_level_keys = {key.split(".")[0] for key in update_value_map}

fides/api/service/connectors/query_configs/query_config.py

@@ -1,7 +1,7 @@
 # pylint: disable=too-many-lines
 import re
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar, Union
+from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
 
 import pydash
 from loguru import logger
@@ -22,7 +22,6 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.policy import Policy, Rule
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.namespace_meta.namespace_meta import NamespaceMeta
-from fides.api.schemas.partitioning.time_based_partitioning import TimeBasedPartitioning
 from fides.api.schemas.policy import ActionType
 from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
 from fides.api.service.masking.strategy.masking_strategy_nullify import (
@@ -47,9 +46,7 @@ class QueryConfig(Generic[T], ABC):
         self.node = node
 
     @property
-    def partitioning(  # pylint: disable=useless-return
-        self,
-    ) -> Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]:
+    def partitioning(self) -> Optional[Dict]:  # pylint: disable=R1711
         # decided to de-scope partitioning support to only bigquery as this grew more complex,
         # but keeping more generic support stubbed out feels like a reasonable step.
         if self.node.collection.partitioning:

fides/api/util/connection_util.py

@@ -25,11 +25,6 @@ from fides.api.models.connectionconfig import (
     ConnectionType,
 )
 from fides.api.models.datasetconfig import DatasetConfig
-from fides.api.models.manual_task import (
-    ManualTask,
-    ManualTaskParentEntityType,
-    ManualTaskType,
-)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.models.sql_models import Dataset as CtlDataset  # type: ignore
@@ -248,9 +243,7 @@ def patch_connection_configs(
                     ).model_dump(mode="json")
                     connection_config.save(db=db)
                     created_or_updated.append(
-                        ConnectionConfigurationResponse.model_validate(
-                            connection_config
-                        )
+                        ConnectionConfigurationResponse(**connection_config.__dict__)
                     )
                     continue
 
@@ -275,24 +268,8 @@ def patch_connection_configs(
             connection_config = ConnectionConfig.create_or_update(
                 db, data=config_dict, check_name=False
             )
-
-            # Automatically create a ManualTask if this is a connection config of type manual_task
-            # and it doesn't already have one
-            if (
-                connection_config.connection_type == ConnectionType.manual_task
-                and not connection_config.manual_task
-            ):
-                ManualTask.create(
-                    db=db,
-                    data={
-                        "task_type": ManualTaskType.privacy_request,
-                        "parent_entity_id": connection_config.id,
-                        "parent_entity_type": ManualTaskParentEntityType.connection_config,
-                    },
-                )
-
             created_or_updated.append(
-                ConnectionConfigurationResponse.model_validate(connection_config)
+                ConnectionConfigurationResponse(**connection_config.__dict__)
             )
         except KeyOrNameAlreadyExists as exc:
             logger.warning(

fides/common/api/v1/urn_registry.py

@@ -155,10 +155,6 @@ AUTHORIZE = "/connection/{connection_key}/authorize"
 ACCESS_MANUAL_WEBHOOKS = "/access_manual_webhook"
 ACCESS_MANUAL_WEBHOOK = CONNECTION_BY_KEY + "/access_manual_webhook"
 
-# Manual Tasks
-MANUAL_TASKS = "/manual-tasks"
-MANUAL_TASK = CONNECTION_BY_KEY + "/manual-task"
-
 # Collection URLs
 DATASETS = "/dataset"
 DATASET_CONFIG = "/datasetconfig"