ethyca-fides 2.64.3b0__py2.py3-none-any.whl → 2.64.5rc3__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-30T18:45:24-0400",
+ "date": "2025-07-02T10:30:35-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "25f89cf50afcac815c38422d299fb39b35cd76a3",
- "version": "2.64.3b0"
+ "full-revisionid": "cd9aa2f5e7ed5c57bff3cff6af259ceae63728b3",
+ "version": "2.64.5rc3"
 }
 '''  # END VERSION_JSON
 

fides/api/api/v1/api.py

@@ -11,7 +11,6 @@ from fides.api.api.v1.endpoints import (
     masking_endpoints,
     messaging_endpoints,
     oauth_endpoints,
-    partitioning_endpoints,
     policy_endpoints,
     policy_webhook_endpoints,
     pre_approval_webhook_endpoints,
@@ -43,7 +42,6 @@ api_router.include_router(identity_verification_endpoints.router)
 api_router.include_router(storage_endpoints.router)
 api_router.include_router(messaging_endpoints.router)
 api_router.include_router(saas_config_endpoints.router)
-api_router.include_router(partitioning_endpoints.router)
 api_router.include_router(user_endpoints.router)
 api_router.include_router(user_permission_endpoints.router)
 api_router.include_router(manual_webhook_endpoints.router)

fides/api/db/base.py

@@ -31,15 +31,12 @@ from fides.api.models.fides_user_respondent_email_verification import (
 )
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
-from fides.api.models.manual_task import (
-    ManualTask,
+from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
+from fides.api.models.manual_tasks.manual_task_config import (
     ManualTaskConfig,
     ManualTaskConfigField,
-    ManualTaskInstance,
-    ManualTaskLog,
-    ManualTaskReference,
-    ManualTaskSubmission,
 )
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.messaging import MessagingConfig
 from fides.api.models.messaging_template import MessagingTemplate

fides/api/db/database.py

@@ -15,8 +15,8 @@ from sqlalchemy_utils.types.encrypted.encrypted_type import InvalidCiphertextErr
 
 from fides.api.db.base import Base  # type: ignore[attr-defined]
 from fides.api.db.seed import load_default_resources
-from fides.api.db.session import get_db_engine
 from fides.api.util.errors import get_full_exception_name
+from fides.core.utils import get_db_engine
 
 DatabaseHealth = Literal["healthy", "unhealthy", "needs migration"]
 
@@ -41,14 +41,6 @@ def upgrade_db(alembic_config: Config, revision: str = "head") -> None:
     command.upgrade(alembic_config, revision)
 
 
-def init_db(database_url: str) -> None:
-    """
-    Runs the migrations and creates all of the database objects.
-    """
-    alembic_config = get_alembic_config(database_url)
-    upgrade_db(alembic_config)
-
-
 def downgrade_db(alembic_config: Config, revision: str = "head") -> None:
     """Downgrade the database to the specified migration revision."""
     log.info(f"Running database downgrade to revision {revision}")
@@ -86,7 +78,7 @@ def reset_db(database_url: str) -> None:
     Drops all tables/metadata from the database.
     """
     log.info("Resetting database...")
-    engine = get_db_engine(database_uri=database_url)
+    engine = get_db_engine(database_url)
     with engine.connect() as connection:
         log.info("Dropping tables...")
         Base.metadata.drop_all(connection)
@@ -144,20 +136,3 @@ def configure_db(database_url: str, revision: Optional[str] = "head") -> None:
         log.error("Unable to configure database: {}: {}", error_type, error)
         log.opt(exception=True).error(error)
         raise
-
-
-def check_missing_migrations(database_url: str) -> None:
-    """
-    Tries to autogenerate migrations, returns True if a migration
-    was generated.
-    """
-
-    engine = get_db_engine(database_uri=database_url)
-    connection = engine.connect()
-
-    migration_context = migration.MigrationContext.configure(connection)
-    result = command.autogen.compare_metadata(migration_context, Base.metadata)  # type: ignore[attr-defined]
-
-    if result:
-        raise SystemExit("Migrations needs to be generated!")
-    print("No migrations need to be generated.")

fides/api/graph/config.py

@@ -95,10 +95,6 @@ from fides.api.graph.data_type import (
     DataTypeConverter,
     get_data_type_converter,
 )
-from fides.api.schemas.partitioning import TimeBasedPartitioning
-from fides.api.schemas.partitioning.time_based_partitioning import (
-    validate_partitioning_list,
-)
 from fides.api.util.collection_util import merge_dicts
 from fides.api.util.querytoken import QueryToken
 
@@ -463,7 +459,7 @@ class Collection(BaseModel):
     grouped_inputs: Set[str] = set()
     data_categories: Set[FidesKey] = set()
     masking_strategy_override: Optional[MaskingStrategyOverride] = None
-    partitioning: Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]] = None
+    partitioning: Optional[Dict] = None
 
     @property
     def field_dict(self) -> Dict[FieldPath, Field]:
@@ -659,12 +655,14 @@ class Collection(BaseModel):
     @field_validator("partitioning")
     @classmethod
     def validate_partitioning(
-        cls, partitioning: Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]
-    ) -> Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]:
+        cls, partitioning: Optional[Dict[str, Any]]
+    ) -> Optional[Dict[str, Any]]:
         """
-        Validates the `partitioning` field, which may be either a single dict
-        or a list of dicts.  Each dict must describe either legacy
-        `where_clauses` or the required keys for time-based partitioning.
+        Validates the `partitioning` dict field.
+
+        The `partitioning` dict field is untyped in Fideslang, but here we enforce
+        that it has the required and expected `where_clauses` key, whose value must be
+        a list of strings.
 
         The string values are validated to ensure they match the expected syntax, which
         is strictly prescribed. The string values MUST be a valid SQL clause that defines
@@ -685,12 +683,7 @@ class Collection(BaseModel):
         in its conditional
 
         """
-
-        if partitioning is None:
-            return partitioning
-
-        if isinstance(partitioning, list):
-            validate_partitioning_list(partitioning)
+        if not partitioning:
             return partitioning
 
         # NOTE: when we deprecate `where_clause` partitioning in favor of a more proper partitioning DSL,

fides/api/models/attachment.py

@@ -5,9 +5,9 @@ from typing import IO, TYPE_CHECKING, Any, Tuple
 
 from fideslang.validation import AnyHttpUrlString
 from loguru import logger as log
-from sqlalchemy import Column, DateTime
+from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import ForeignKey, Index, String, UniqueConstraint, func, orm
+from sqlalchemy import ForeignKey, String, UniqueConstraint, orm
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
@@ -64,14 +64,8 @@ class AttachmentReference(Base):
         """Overriding base class method to set the table name."""
         return "attachment_reference"
 
-    created_at = Column(
-        DateTime(timezone=True), server_default=func.now(), nullable=False
-    )
-
     attachment_id = Column(
-        String,
-        ForeignKey("attachment.id", name="attachment_reference_attachment_id_fkey"),
-        nullable=False,
+        String, ForeignKey("attachment.id", ondelete="CASCADE"), nullable=False
     )
     reference_id = Column(String, nullable=False)
     reference_type = Column(EnumColumn(AttachmentReferenceType), nullable=False)
@@ -80,8 +74,6 @@ class AttachmentReference(Base):
         UniqueConstraint(
             "attachment_id", "reference_id", name="_attachment_reference_uc"
         ),
-        Index("ix_attachment_reference_reference_id", "reference_id"),
-        Index("ix_attachment_reference_reference_type", "reference_type"),
     )
 
     # Relationships
@@ -100,10 +92,6 @@ class Attachment(Base):
     Stores information about an Attachment.
     """
 
-    created_at = Column(
-        DateTime(timezone=True), server_default=func.now(), nullable=False
-    )
-
     user_id = Column(
         String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
     )

fides/api/models/comment.py

@@ -1,13 +1,13 @@
 from enum import Enum as EnumType
 from typing import TYPE_CHECKING, Any
 
-from sqlalchemy import Column, DateTime
+from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import ForeignKey, Index, String, func, orm
+from sqlalchemy import ForeignKey, String, UniqueConstraint, orm
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
-from fides.api.db.base_class import Base, FidesBase
+from fides.api.db.base_class import Base
 from fides.api.models.attachment import Attachment, AttachmentReferenceType
 
 if TYPE_CHECKING:
@@ -36,7 +36,6 @@ class CommentReferenceType(str, EnumType):
     access_manual_webhook = "access_manual_webhook"
     erasure_manual_webhook = "erasure_manual_webhook"
     privacy_request = "privacy_request"
-    manual_task_submission = "manual_task_submission"
 
 
 class CommentReference(Base):
@@ -49,24 +48,12 @@ class CommentReference(Base):
         """Overriding base class method to set the table name."""
         return "comment_reference"
 
-    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
-    created_at = Column(
-        DateTime(timezone=True), server_default=func.now(), nullable=False
-    )
-
-    comment_id = Column(
-        String,
-        ForeignKey(
-            "comment.id", name="comment_reference_comment_id_fkey", ondelete="CASCADE"
-        ),
-        nullable=False,
-    )
+    comment_id = Column(String, ForeignKey("comment.id"), nullable=False)
     reference_id = Column(String, nullable=False)
     reference_type = Column(EnumColumn(CommentReferenceType), nullable=False)
 
     __table_args__ = (
-        Index("ix_comment_reference_reference_id", "reference_id"),
-        Index("ix_comment_reference_reference_type", "reference_type"),
+        UniqueConstraint("comment_id", "reference_id", name="comment_reference_uc"),
     )
 
     comment = relationship(
@@ -88,11 +75,6 @@ class Comment(Base):
     Stores information about a Comment.
     """
 
-    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
-    created_at = Column(
-        DateTime(timezone=True), server_default=func.now(), nullable=False
-    )
-
     user_id = Column(
         String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
     )

fides/api/models/connectionconfig.py

@@ -53,7 +53,6 @@ class ConnectionType(enum.Enum):
     https = "https"
     manual = "manual"  # Deprecated - use manual_webhook instead
     manual_webhook = "manual_webhook"  # Runs upfront before the traversal
-    manual_task = "manual_task"  # Manual task integration
     mariadb = "mariadb"
     mongodb = "mongodb"
     mssql = "mssql"
@@ -89,7 +88,6 @@ class ConnectionType(enum.Enum):
             ConnectionType.google_cloud_sql_postgres.value: "Google Cloud SQL for Postgres",
             ConnectionType.https.value: "Policy Webhook",
             ConnectionType.manual_webhook.value: "Manual Process",
-            ConnectionType.manual_task.value: "Manual Task",
             ConnectionType.manual.value: "Manual Connector",
             ConnectionType.mariadb.value: "MariaDB",
             ConnectionType.mongodb.value: "MongoDB",
@@ -134,7 +132,6 @@ class ConnectionType(enum.Enum):
             ConnectionType.google_cloud_sql_postgres.value: SystemType.database,
             ConnectionType.https.value: SystemType.manual,
             ConnectionType.manual_webhook.value: SystemType.manual,
-            ConnectionType.manual_task.value: SystemType.manual,
             ConnectionType.manual.value: SystemType.manual,
             ConnectionType.mariadb.value: SystemType.database,
             ConnectionType.mongodb.value: SystemType.database,
@@ -232,14 +229,6 @@ class ConnectionConfig(Base):
         uselist=False,
     )
 
-    manual_task = relationship(  # type: ignore[misc]
-        "ManualTask",
-        primaryjoin="and_(ConnectionConfig.id == foreign(ManualTask.parent_entity_id), "
-        "ManualTask.parent_entity_type == 'connection_config')",
-        cascade="delete",
-        uselist=False,
-    )
-
     pre_approval_webhooks = relationship(  # type: ignore[misc]
         "PreApprovalWebhook",
         back_populates="connection_config",

fides/api/models/db_cache.py

@@ -32,7 +32,7 @@ class DBCache(Base):
     cache_value = Column(BYTEA, nullable=False)
 
     __table_args__ = (
-        Index("ix_dbcache_namespace_cache_key", "namespace", "cache_key", unique=True),
+        Index("ix_dbcache_namespace_cache_key", "namespace", "cache_key"),
     )
 
     @classmethod

fides/api/models/detection_discovery/core.py

@@ -159,8 +159,6 @@ class MonitorConfig(Base):
         "MonitorExecution",
         cascade="all, delete-orphan",
         backref="monitor_config",
-        primaryjoin="MonitorExecution.monitor_config_key == foreign(MonitorConfig.key)",
-        single_parent=True,
     )
 
     shared_config_id = Column(
@@ -344,16 +342,16 @@ class StagedResourceAncestor(Base):
     Its entries should be deleted when the staged resource is deleted, via cascade.
     """
 
-    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
-
     ancestor_urn = Column(
         String,
         ForeignKey("stagedresource.urn", ondelete="CASCADE"),
+        primary_key=True,
         nullable=False,
     )
     descendant_urn = Column(
         String,
         ForeignKey("stagedresource.urn", ondelete="CASCADE"),
+        primary_key=True,
         nullable=False,
     )
 
@@ -374,7 +372,6 @@ class StagedResourceAncestor(Base):
         UniqueConstraint(
             "ancestor_urn", "descendant_urn", name="uq_staged_resource_ancestor"
         ),
-        Index("ix_staged_resource_ancestor_pkey", "id", unique=True),
         Index("ix_staged_resource_ancestor_ancestor", "ancestor_urn"),
         Index("ix_staged_resource_ancestor_descendant", "descendant_urn"),
     )
@@ -473,7 +470,12 @@ class StagedResource(Base):
         server_default="{}",
         default=dict,
     )
-    user_assigned_system_id = Column(String, nullable=True, index=True)
+    user_assigned_system_id = Column(
+        String,
+        ForeignKey(System.id_field_path),
+        nullable=True,
+        index=True,
+    )
 
     # pointers to child and parent URNs
     children = Column(
@@ -525,7 +527,7 @@ class StagedResource(Base):
 
     data_uses = Column(
         ARRAY(String),
-        nullable=True,
+        nullable=False,
         server_default="{}",
         default=dict,
     )
@@ -586,14 +588,12 @@ class MonitorExecution(Base):
     configuration details used in connecting to the external data store.
     """
 
-    # redefined here because there's a minor, unintended discrepancy between
-    # this `id` field and that of the `Base` class, which explicitly sets `index=True`.
-    # TODO: we likely should _not_ be setting `index=True` on the `id`
-    # attribute of the `Base` class, as `primary_key=True` already specifies a
-    # primary key constraint, which will implicitly create an index for the field.
-    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
-
-    monitor_config_key = Column(String, nullable=False, index=True)
+    monitor_config_key = Column(
+        String,
+        ForeignKey(MonitorConfig.key),
+        nullable=False,
+        index=True,
+    )
     status = Column(String, nullable=True)
     started = Column(
         DateTime(timezone=True),

fides/api/models/fides_user_respondent_email_verification.py

@@ -3,13 +3,12 @@ from __future__ import annotations
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, Optional
 
-from sqlalchemy import Column, DateTime, ForeignKey, Index, String
+from sqlalchemy import Column, DateTime, ForeignKey, String
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
-from sqlalchemy.sql import func
 
 from fides.api.cryptography.cryptographic_util import generate_secure_random_string
-from fides.api.db.base_class import Base, FidesBase
+from fides.api.db.base_class import Base
 from fides.api.util.identity_verification import IdentityVerificationMixin
 from fides.config import get_config
 
@@ -41,22 +40,6 @@ class FidesUserRespondentEmailVerification(Base, IdentityVerificationMixin):
     def __tablename__(self) -> str:
         return "fides_user_respondent_email_verification"
 
-    # redefined here because there's a minor, unintended discrepancy between
-    # this `id` field and that of the `Base` class, which explicitly sets `index=True`.
-    # TODO: we likely should _not_ be setting `index=True` on the `id`
-    # attribute of the `Base` class, as `primary_key=True` already specifies a
-    # primary key constraint, which will implicitly create an index for the field.
-    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
-    created_at = Column(
-        DateTime(timezone=True), server_default=func.now(), nullable=False
-    )
-    updated_at = Column(
-        DateTime(timezone=True),
-        server_default=func.now(),
-        onupdate=func.now(),
-        nullable=False,
-    )
-
     user_id = Column(
         String,
         ForeignKey("fidesuser.id", ondelete="CASCADE"),
@@ -76,14 +59,6 @@ class FidesUserRespondentEmailVerification(Base, IdentityVerificationMixin):
         foreign_keys=[user_id],
     )
 
-    __table_args__ = (
-        Index(
-            "ix_fides_user_respondent_email_verification_id",
-            "id",
-            unique=True,
-        ),
-    )
-
     @classmethod
     def create(
         cls, db: Session, *, data: dict[str, Any], check_name: bool = False

fides/api/models/manual_tasks/__init__.py

@@ -0,0 +1,14 @@
+from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
+from fides.api.models.manual_tasks.manual_task_config import (
+    ManualTaskConfig,
+    ManualTaskConfigField,
+)
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
+
+__all__ = [
+    "ManualTask",
+    "ManualTaskConfig",
+    "ManualTaskConfigField",
+    "ManualTaskReference",
+    "ManualTaskLog",
+]

fides/api/models/manual_tasks/manual_task.py

@@ -0,0 +1,120 @@
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
+from fides.api.schemas.manual_tasks.manual_task_schemas import (
+    ManualTaskLogStatus,
+    ManualTaskParentEntityType,
+    ManualTaskReferenceType,
+    ManualTaskType,
+)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from fides.api.models.manual_tasks.manual_task_config import (  # pragma: no cover
+        ManualTaskConfig,
+    )
+
+
+class ManualTask(Base):
+    """Model for storing manual tasks.
+
+    This model can be used for both privacy request tasks and general tasks.
+    For privacy requests, it replaces the functionality of manual webhooks.
+    For other use cases, it provides a flexible task management system.
+
+    There can only be one ManualTask per parent entity.
+    You can create multiple Configs for the same ManualTask.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task"
+
+    # Database columns
+    task_type = Column(
+        EnumColumn(ManualTaskType),
+        nullable=False,
+        default=ManualTaskType.privacy_request,
+    )
+    parent_entity_id = Column(String, nullable=False)
+    parent_entity_type = Column(
+        EnumColumn(ManualTaskParentEntityType),
+        nullable=False,
+        default=ManualTaskParentEntityType.connection_config,
+    )
+    due_date = Column(DateTime, nullable=True)
+
+    # Relationships
+    references = relationship(
+        "ManualTaskReference",
+        back_populates="task",
+        uselist=True,
+        cascade="all, delete-orphan",
+    )
+    logs = relationship(
+        "ManualTaskLog",
+        back_populates="task",
+        primaryjoin="and_(ManualTask.id == ManualTaskLog.task_id)",
+        viewonly=True,
+        order_by="ManualTaskLog.created_at",
+    )
+    configs = relationship(
+        "ManualTaskConfig",
+        back_populates="task",
+        cascade="all, delete-orphan",
+    )
+
+    # Properties
+    @property
+    def assigned_users(self) -> list[str]:
+        """Get all users assigned to this task."""
+        if not self.references:
+            return []
+        return [
+            ref.reference_id
+            for ref in self.references
+            if ref.reference_type == ManualTaskReferenceType.assigned_user
+        ]
+
+    # CRUD Operations
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = True
+    ) -> "ManualTask":
+        """Create a new manual task."""
+        task = super().create(db=db, data=data, check_name=check_name)
+        ManualTaskLog.create_log(
+            db=db,
+            task_id=task.id,
+            status=ManualTaskLogStatus.created,
+            message=f"Created manual task for {data['task_type']}",
+        )
+        return task
+
+
+class ManualTaskReference(Base):
+    """Join table to associate manual tasks with multiple references.
+
+    A single task may have many references including privacy requests, configurations, and assigned users.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_reference"
+
+    # Database columns
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(ManualTaskReferenceType), nullable=False)
+
+    # Relationships
+    task = relationship("ManualTask", back_populates="references")