ethyca-fides 2.64.1b0__py2.py3-none-any.whl → 2.64.1b2__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/aadfe83c5644_add_manual_task_to_connectiontype_enum.py

@@ -0,0 +1,46 @@
+"""add_manual_task_to_connectiontype_enum
+
+Revision ID: aadfe83c5644
+Revises: 6a76a1fa4f3f
+Create Date: 2025-06-19 18:55:08.131278
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "aadfe83c5644"
+down_revision = "6a76a1fa4f3f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add manual_task to ConnectionType enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('attentive_email', 'bigquery', 'datahub', 'dynamodb', 'fides', 'generic_consent_email', 'generic_erasure_email', 'dynamic_erasure_email', 'google_cloud_sql_mysql', 'google_cloud_sql_postgres', 'https', 'manual', 'manual_webhook', 'manual_task', 'mariadb', 'mongodb', 'mssql', 'mysql', 'okta', 'postgres', 'rds_mysql', 'rds_postgres', 'redshift', 's3', 'saas', 'scylla', 'snowflake', 'sovrn', 'timescale', 'website')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")
+
+
+def downgrade():
+    # Remove manual_task from ConnectionType enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('attentive_email', 'bigquery', 'datahub', 'dynamodb', 'fides', 'generic_consent_email', 'generic_erasure_email', 'dynamic_erasure_email', 'google_cloud_sql_mysql', 'google_cloud_sql_postgres', 'https', 'manual', 'manual_webhook', 'mariadb', 'mongodb', 'mssql', 'mysql', 'okta', 'postgres', 'rds_mysql', 'rds_postgres', 'redshift', 's3', 'saas', 'scylla', 'snowflake', 'sovrn', 'timescale', 'website')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")

fides/api/db/base.py

@@ -36,6 +36,10 @@ from fides.api.models.manual_tasks.manual_task_config import (
     ManualTaskConfig,
     ManualTaskConfigField,
 )
+from fides.api.models.manual_tasks.manual_task_instance import (
+    ManualTaskInstance,
+    ManualTaskSubmission,
+)
 from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.messaging import MessagingConfig

fides/api/models/connectionconfig.py

@@ -53,6 +53,7 @@ class ConnectionType(enum.Enum):
     https = "https"
     manual = "manual"  # Deprecated - use manual_webhook instead
     manual_webhook = "manual_webhook"  # Runs upfront before the traversal
+    manual_task = "manual_task"  # Manual task integration
     mariadb = "mariadb"
     mongodb = "mongodb"
     mssql = "mssql"
@@ -88,6 +89,7 @@ class ConnectionType(enum.Enum):
             ConnectionType.google_cloud_sql_postgres.value: "Google Cloud SQL for Postgres",
             ConnectionType.https.value: "Policy Webhook",
             ConnectionType.manual_webhook.value: "Manual Process",
+            ConnectionType.manual_task.value: "Manual Task",
             ConnectionType.manual.value: "Manual Connector",
             ConnectionType.mariadb.value: "MariaDB",
             ConnectionType.mongodb.value: "MongoDB",
@@ -132,6 +134,7 @@ class ConnectionType(enum.Enum):
             ConnectionType.google_cloud_sql_postgres.value: SystemType.database,
             ConnectionType.https.value: SystemType.manual,
             ConnectionType.manual_webhook.value: SystemType.manual,
+            ConnectionType.manual_task.value: SystemType.manual,
             ConnectionType.manual.value: SystemType.manual,
             ConnectionType.mariadb.value: SystemType.database,
             ConnectionType.mongodb.value: SystemType.database,
@@ -229,6 +232,14 @@ class ConnectionConfig(Base):
         uselist=False,
     )
 
+    manual_task = relationship(  # type: ignore[misc]
+        "ManualTask",
+        primaryjoin="and_(ConnectionConfig.id == foreign(ManualTask.parent_entity_id), "
+        "ManualTask.parent_entity_type == 'connection_config')",
+        cascade="delete",
+        uselist=False,
+    )
+
     pre_approval_webhooks = relationship(  # type: ignore[misc]
         "PreApprovalWebhook",
         back_populates="connection_config",

fides/api/models/manual_tasks/__init__.py

@@ -3,12 +3,18 @@ from fides.api.models.manual_tasks.manual_task_config import (
     ManualTaskConfig,
     ManualTaskConfigField,
 )
+from fides.api.models.manual_tasks.manual_task_instance import (
+    ManualTaskInstance,
+    ManualTaskSubmission,
+)
 from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
 
 __all__ = [
     "ManualTask",
     "ManualTaskConfig",
     "ManualTaskConfigField",
-    "ManualTaskReference",
+    "ManualTaskInstance",
     "ManualTaskLog",
+    "ManualTaskReference",
+    "ManualTaskSubmission",
 ]

fides/api/models/manual_tasks/manual_task.py

@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING, Any
 
-from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy import Column, ForeignKey, String
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
@@ -18,6 +18,10 @@ if TYPE_CHECKING:  # pragma: no cover
     from fides.api.models.manual_tasks.manual_task_config import (  # pragma: no cover
         ManualTaskConfig,
     )
+    from fides.api.models.manual_tasks.manual_task_instance import (
+        ManualTaskInstance,
+        ManualTaskSubmission,
+    )
 
 
 class ManualTask(Base):
@@ -48,7 +52,6 @@ class ManualTask(Base):
         nullable=False,
         default=ManualTaskParentEntityType.connection_config,
     )
-    due_date = Column(DateTime, nullable=True)
 
     # Relationships
     references = relationship(
@@ -68,6 +71,19 @@ class ManualTask(Base):
         "ManualTaskConfig",
         back_populates="task",
         cascade="all, delete-orphan",
+        uselist=True,
+    )
+    instances = relationship(
+        "ManualTaskInstance",
+        back_populates="task",
+        viewonly=True,
+        uselist=True,
+    )
+    submissions = relationship(
+        "ManualTaskSubmission",
+        back_populates="task",
+        uselist=True,
+        viewonly=True,
     )
 
     # Properties
@@ -117,4 +133,4 @@ class ManualTaskReference(Base):
     reference_type = Column(EnumColumn(ManualTaskReferenceType), nullable=False)
 
     # Relationships
-    task = relationship("ManualTask", back_populates="references")
+    task = relationship("ManualTask", back_populates="references", viewonly=True)

fides/api/models/manual_tasks/manual_task_config.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Optional, cast
 
 from sqlalchemy import Boolean, Column, ForeignKey, Integer, String
 from sqlalchemy.dialects.postgresql import JSONB
@@ -13,10 +13,19 @@ from fides.api.schemas.manual_tasks.manual_task_config import (
     ManualTaskFieldMetadata,
     ManualTaskFieldType,
 )
-from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskLogStatus
+from fides.api.schemas.manual_tasks.manual_task_schemas import (
+    ManualTaskExecutionTiming,
+    ManualTaskLogStatus,
+)
 
 if TYPE_CHECKING:  # pragma: no cover
     from fides.api.models.manual_tasks.manual_task import ManualTask  # pragma: no cover
+    from fides.api.models.manual_tasks.manual_task_instance import (
+        ManualTaskInstance,  # pragma: no cover
+    )
+    from fides.api.models.manual_tasks.manual_task_instance import (
+        ManualTaskSubmission,  # pragma: no cover
+    )
 
 
 class ManualTaskConfig(Base):
@@ -32,9 +41,20 @@ class ManualTaskConfig(Base):
     config_type = Column(EnumColumn(ManualTaskConfigurationType), nullable=False)
     version = Column(Integer, nullable=False, default=1)
     is_current = Column(Boolean, nullable=False, default=True)
+    execution_timing = Column(
+        EnumColumn(ManualTaskExecutionTiming),
+        nullable=False,
+        default=ManualTaskExecutionTiming.pre_execution,
+    )
 
     # Relationships
-    task = relationship("ManualTask", back_populates="configs")
+    task = relationship("ManualTask", back_populates="configs", viewonly=True)
+    instances = relationship(
+        "ManualTaskInstance", back_populates="config", uselist=True, viewonly=True
+    )
+    submissions = relationship(
+        "ManualTaskSubmission", back_populates="config", uselist=True, viewonly=True
+    )
     field_definitions = relationship(
         "ManualTaskConfigField",
         back_populates="config",
@@ -45,7 +65,7 @@ class ManualTaskConfig(Base):
         "ManualTaskLog",
         back_populates="config",
         primaryjoin="ManualTaskConfig.id == ManualTaskLog.config_id",
-        viewonly=True,
+        cascade="all, delete-orphan",
     )
 
     @classmethod
@@ -95,14 +115,27 @@ class ManualTaskConfigField(Base):
     field_type = Column(
         EnumColumn(ManualTaskFieldType), nullable=False
     )  # Using ManualTaskFieldType
-    field_metadata = Column(JSONB, nullable=False, default={})
+    field_metadata: dict[str, Any] = cast(
+        dict[str, Any], Column(JSONB, nullable=False, default={})
+    )
 
     # Relationships
-    config = relationship("ManualTaskConfig", back_populates="field_definitions")
+    config = relationship(
+        "ManualTaskConfig", back_populates="field_definitions", viewonly=True
+    )
+    submissions = relationship(
+        "ManualTaskSubmission",
+        back_populates="field",
+        uselist=True,
+        cascade="all, delete-orphan",
+    )
 
     @property
     def field_metadata_model(self) -> ManualTaskFieldMetadata:
         """Get the field metadata as a Pydantic model."""
+        assert isinstance(
+            self.field_metadata, dict
+        ), "field_metadata must be a dictionary"
         return ManualTaskFieldMetadata.model_validate(self.field_metadata)
 
     @classmethod

fides/api/models/manual_tasks/manual_task_instance.py

@@ -0,0 +1,187 @@
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import relationship
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+from fides.api.models.manual_tasks.manual_task_config import ManualTaskConfigField
+from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskEntityType
+from fides.api.schemas.manual_tasks.manual_task_status import (
+    StatusTransitionMixin,
+    StatusType,
+)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from fides.api.models.attachment import Attachment  # pragma: no cover
+    from fides.api.models.fides_user import FidesUser  # pragma: no cover
+    from fides.api.models.manual_tasks.manual_task import ManualTask  # pragma: no cover
+    from fides.api.models.manual_tasks.manual_task_config import (
+        ManualTaskConfig,  # pragma: no cover
+    )
+    from fides.api.models.manual_tasks.manual_task_log import (
+        ManualTaskLog,  # pragma: no cover; pragma: no cover
+    )
+
+
+class ManualTaskInstance(Base, StatusTransitionMixin):
+    """Model for tracking task status per entity instance.
+
+    This model implements StatusTransitionProtocol through the StatusTransitionMixin.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_instance"
+
+    # Database columns
+    task_id: Column[str] = Column(String, ForeignKey("manual_task.id"), nullable=False)
+    config_id: Column[str] = Column(
+        String, ForeignKey("manual_task_config.id"), nullable=False
+    )
+    # entity id is the entity that the instance relates to
+    # (e.g. a privacy request is an entity that has its own manual task instance)
+    entity_id: Column[str] = Column(String, nullable=False)
+    entity_type: Column[ManualTaskEntityType] = Column(
+        EnumColumn(ManualTaskEntityType), nullable=False
+    )
+    # ingnore[assignment] because the mypy and sqlalchemy types mismatch
+    # upgrading to 2.0 allows mapping which provides better type safety visibility.
+    status: Column[StatusType] = Column(EnumColumn(StatusType), nullable=False, default=StatusType.pending)  # type: ignore[assignment]
+    completed_at: Column[Optional[datetime]] = Column(DateTime, nullable=True)  # type: ignore[assignment]
+    completed_by_id: Column[Optional[str]] = Column(String, nullable=True)  # type: ignore[assignment]
+    due_date: Column[Optional[datetime]] = Column(DateTime, nullable=True)
+
+    # Relationships
+    task = relationship("ManualTask", back_populates="instances")
+    config = relationship("ManualTaskConfig", back_populates="instances")
+    submissions = relationship(
+        "ManualTaskSubmission",
+        back_populates="instance",
+        cascade="all, delete-orphan",
+        uselist=True,
+    )
+    logs = relationship(
+        "ManualTaskLog",
+        back_populates="instance",
+        primaryjoin="ManualTaskInstance.id == ManualTaskLog.instance_id",
+        cascade="all, delete-orphan",
+        order_by="ManualTaskLog.created_at",
+        uselist=True,
+    )
+    attachments = relationship(
+        "Attachment",
+        secondary="attachment_reference",
+        primaryjoin="and_(ManualTaskInstance.id == ManualTaskSubmission.instance_id, "
+        "ManualTaskSubmission.id == AttachmentReference.reference_id, "
+        "AttachmentReference.reference_type == 'manual_task_submission')",
+        secondaryjoin="Attachment.id == AttachmentReference.attachment_id",
+        order_by="Attachment.created_at",
+        viewonly=True,
+        uselist=True,
+    )
+
+    @property
+    def required_fields(self) -> list["ManualTaskConfigField"]:
+        """Get all required fields."""
+        return [
+            field
+            for field in self.config.field_definitions
+            if field.field_metadata.get("required", False)
+        ]
+
+    @property
+    def incomplete_fields(self) -> list["ManualTaskConfigField"]:
+        """Get all fields that haven't been completed yet.
+        A field is considered incomplete if:
+        1. It's required and has no submission
+        Returns:
+            list[ManualTaskConfigField]: List of incomplete fields
+        """
+        return [
+            field
+            for field in self.required_fields
+            if not self.get_submission_for_field(field.id)
+        ]
+
+    @property
+    def completed_fields(self) -> list["ManualTaskConfigField"]:
+        """Get all fields that have been completed."""
+        return [
+            field
+            for field in self.config.field_definitions
+            if field.field_metadata.get("required", False)
+            and self.get_submission_for_field(field.id)
+        ]
+
+    def get_submission_for_field(
+        self, field_id: str
+    ) -> Optional["ManualTaskSubmission"]:
+        """Get the submission for a specific field.
+
+        Args:
+            field_id: The ID of the field to get the submission for
+
+        Returns:
+            Optional[ManualTaskSubmission]: The submission for the field, or None if no submission exists
+        """
+        return next(
+            (
+                submission
+                for submission in self.submissions
+                if submission.field_id == field_id
+            ),
+            None,
+        )
+
+
+class ManualTaskSubmission(Base):
+    """Model for storing user submissions.
+    Each submission represents data for a single field.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_submission"
+
+    # Database columns
+    task_id = Column(String, ForeignKey("manual_task.id"))
+    config_id = Column(String, ForeignKey("manual_task_config.id"))
+    field_id = Column(String, ForeignKey("manual_task_config_field.id"))
+    instance_id = Column(String, ForeignKey("manual_task_instance.id"), nullable=False)
+    submitted_by = Column(String, ForeignKey("fidesuser.id"), nullable=True)
+    submitted_at = Column(DateTime, default=datetime.now(timezone.utc), nullable=False)
+    data = Column(JSONB, nullable=False)
+
+    # Relationships
+    task = relationship("ManualTask", back_populates="submissions", viewonly=True)
+    config = relationship(
+        "ManualTaskConfig", back_populates="submissions", viewonly=True
+    )
+    field = relationship(
+        "ManualTaskConfigField", back_populates="submissions", viewonly=True
+    )
+    instance = relationship(
+        "ManualTaskInstance", back_populates="submissions", viewonly=True
+    )
+    attachments = relationship(
+        "Attachment",
+        secondary="attachment_reference",
+        primaryjoin="and_(ManualTaskSubmission.id == AttachmentReference.reference_id, "
+        "AttachmentReference.reference_type == 'manual_task_submission')",
+        secondaryjoin="Attachment.id == AttachmentReference.attachment_id",
+        order_by="Attachment.created_at",
+        viewonly=True,
+        uselist=True,
+    )
+
+    user = relationship(
+        "FidesUser",
+        primaryjoin="FidesUser.id == ManualTaskSubmission.submitted_by",
+        viewonly=True,
+    )

fides/api/models/manual_tasks/manual_task_log.py

@@ -13,6 +13,9 @@ if TYPE_CHECKING:  # pragma: no cover
     from fides.api.models.manual_tasks.manual_task_config import (
         ManualTaskConfig,  # pragma: no cover
     )
+    from fides.api.models.manual_tasks.manual_task_instance import (  # pragma: no cover
+        ManualTaskInstance,
+    )
 
 
 class ManualTaskLog(Base):
@@ -26,19 +29,29 @@ class ManualTaskLog(Base):
     task_id = Column(
         String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
     )
-    config_id = Column(String, ForeignKey("manual_task_config.id"), nullable=True)
-    instance_id = Column(String, nullable=True)
+    config_id = Column(
+        String, ForeignKey("manual_task_config.id", ondelete="CASCADE"), nullable=True
+    )
+    instance_id = Column(
+        String,
+        ForeignKey("manual_task_instance.id", ondelete="CASCADE"),
+        nullable=True,
+    )
     status = Column(String, nullable=False)
     message = Column(String, nullable=False)
     details = Column(JSONB, nullable=True)
 
-    # Relationships - using string references to avoid circular imports
-    task = relationship("ManualTask", back_populates="logs", foreign_keys=[task_id])
+    # Relationships
+    task = relationship(
+        "ManualTask", back_populates="logs", foreign_keys=[task_id], viewonly=True
+    )
     config = relationship(
-        "ManualTaskConfig", back_populates="logs", foreign_keys=[config_id]
+        "ManualTaskConfig",
+        back_populates="logs",
+        foreign_keys=[config_id],
+        viewonly=True,
     )
-    # TODO: Add instance relationship when it is implemented
-    # instance = relationship("ManualTaskInstance", back_populates="logs")
+    instance = relationship("ManualTaskInstance", back_populates="logs", viewonly=True)
 
     @classmethod
     def create_log(

fides/api/schemas/manual_tasks/manual_task_schemas.py

@@ -5,6 +5,15 @@ from typing import Annotated, Any, Optional
 from pydantic import ConfigDict, Field
 
 from fides.api.schemas.base_class import FidesSchema
+from fides.api.schemas.manual_tasks.manual_task_status import StatusType
+
+
+class ManualTaskExecutionTiming(str, Enum):
+    """Enum for when a manual task should be executed in the privacy request DAG."""
+
+    pre_execution = "pre_execution"  # Execute before the main DAG
+    post_execution = "post_execution"  # Execute after the main DAG
+    parallel = "parallel"  # Execute in parallel with the main DAG
 
 
 class ManualTaskType(str, Enum):
@@ -23,6 +32,13 @@ class ManualTaskParentEntityType(str, Enum):
     # Add more parent entity types as needed
 
 
+class ManualTaskEntityType(str, Enum):
+    """Enum for manual task entity types."""
+
+    privacy_request = "privacy_request"
+    # Add more entity types as needed
+
+
 class ManualTaskReferenceType(str, Enum):
     """Enum for manual task reference types."""
 
@@ -46,6 +62,32 @@ class ManualTaskLogStatus(str, Enum):
     awaiting_input = "awaiting_input"
 
 
+class ManualTaskResponse(FidesSchema):
+    """Schema for manual task response."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: Annotated[str, Field(..., description="Task ID")]
+    parent_entity_id: Annotated[str, Field(..., description="Parent entity ID")]
+    parent_entity_type: Annotated[
+        ManualTaskParentEntityType, Field(..., description="Parent entity type")
+    ]
+    status: Annotated[StatusType, Field(..., description="Task status")]
+    created_at: Annotated[datetime, Field(..., description="Creation timestamp")]
+    updated_at: Annotated[datetime, Field(..., description="Last update timestamp")]
+
+
+class ManualTaskCreate(FidesSchema):
+    """Schema for creating a manual task."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    parent_entity_id: Annotated[str, Field(..., description="Parent entity ID")]
+    parent_entity_type: Annotated[
+        ManualTaskParentEntityType, Field(..., description="Parent entity type")
+    ]
+
+
 class ManualTaskLogCreate(FidesSchema):
     """Schema for creating a manual task log entry."""