ethyca-fides 2.64.5rc4__py2.py3-none-any.whl → 2.64.6b1__py2.py3-none-any.whl

fides/api/api/v1/endpoints/partitioning_endpoints.py

@@ -0,0 +1,108 @@
+from typing import List
+
+from fastapi import HTTPException, Security, status
+from loguru import logger
+from sqlalchemy_bigquery import BigQueryDialect
+
+from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.partitioning.time_based_partitioning import (
+    TimeBasedPartitioning,
+    combine_partitions,
+)
+from fides.api.util.api_router import APIRouter
+from fides.common.api.scope_registry import CTL_DATASET_READ
+from fides.common.api.v1.urn_registry import V1_URL_PREFIX
+
+router = APIRouter(tags=["Partitions"], prefix=V1_URL_PREFIX)
+
+
+@router.post(
+    "/dataset/partitions/verify",
+    dependencies=[Security(verify_oauth_client, scopes=[CTL_DATASET_READ])],
+    response_model=List[str],
+    status_code=status.HTTP_200_OK,
+    name="Verify dataset partitioning schemas",
+    summary="Verify dataset partitioning schemas",
+    description="Validate the provided partitioning specs and return the rendered SQL WHERE clauses in the BigQuery dialect.",
+    openapi_extra={
+        "requestBody": {
+            "content": {
+                "application/json": {
+                    "schema": {
+                        "type": "array",
+                        "items": {"$ref": "#/components/schemas/TimeBasedPartitioning"},
+                    },
+                    "examples": {
+                        "recent_data_weekly": {
+                            "summary": "Recent data partitioned weekly",
+                            "description": "Partition the last 30 days of data into weekly chunks",
+                            "value": [
+                                {
+                                    "field": "created_at",
+                                    "start": "NOW() - 30 DAYS",
+                                    "end": "NOW()",
+                                    "interval": "7 DAYS",
+                                }
+                            ],
+                        },
+                        "historical_data_monthly": {
+                            "summary": "Historical data partitioned monthly",
+                            "description": "Partition a specific year of data into monthly chunks",
+                            "value": [
+                                {
+                                    "field": "order_date",
+                                    "start": "2024-01-01",
+                                    "end": "2024-12-31",
+                                    "interval": "1 MONTH",
+                                }
+                            ],
+                        },
+                        "multiple_partitions": {
+                            "summary": "Multiple time ranges",
+                            "description": "Combine historical monthly data with recent daily data",
+                            "value": [
+                                {
+                                    "field": "event_timestamp",
+                                    "start": "2023-01-01",
+                                    "end": "2023-12-31",
+                                    "interval": "1 MONTH",
+                                },
+                                {
+                                    "field": "event_timestamp",
+                                    "start": "NOW() - 14 DAYS",
+                                    "end": "NOW()",
+                                    "interval": "1 DAY",
+                                },
+                            ],
+                        },
+                        "open_ended_partition": {
+                            "summary": "Open-ended partition",
+                            "description": "Partition all data from a specific date onwards",
+                            "value": [
+                                {"field": "user_created_at", "start": "2024-01-01"}
+                            ],
+                        },
+                    },
+                }
+            }
+        }
+    },
+)
+async def verify_dataset_partitioning(
+    partitions: List[TimeBasedPartitioning],
+) -> List[str]:
+    """Validate the provided partitioning specs and return the rendered SQL WHERE clauses in the BigQuery dialect."""
+    try:
+        expressions = combine_partitions(partitions)
+        dialect = BigQueryDialect()
+        compiled_clauses: List[str] = [
+            str(expr.compile(dialect=dialect, compile_kwargs={"literal_binds": True}))
+            for expr in expressions
+        ]
+        return compiled_clauses
+    except Exception as exc:
+        logger.error("Partitioning verification failed", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(exc),
+        )

fides/api/db/base.py

@@ -31,12 +31,15 @@ from fides.api.models.fides_user_respondent_email_verification import (
 )
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
-from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
-from fides.api.models.manual_tasks.manual_task_config import (
+from fides.api.models.manual_task import (
+    ManualTask,
     ManualTaskConfig,
     ManualTaskConfigField,
+    ManualTaskInstance,
+    ManualTaskLog,
+    ManualTaskReference,
+    ManualTaskSubmission,
 )
-from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.messaging import MessagingConfig
 from fides.api.models.messaging_template import MessagingTemplate

fides/api/db/database.py

@@ -15,8 +15,8 @@ from sqlalchemy_utils.types.encrypted.encrypted_type import InvalidCiphertextErr
 
 from fides.api.db.base import Base  # type: ignore[attr-defined]
 from fides.api.db.seed import load_default_resources
+from fides.api.db.session import get_db_engine
 from fides.api.util.errors import get_full_exception_name
-from fides.core.utils import get_db_engine
 
 DatabaseHealth = Literal["healthy", "unhealthy", "needs migration"]
 
@@ -41,6 +41,14 @@ def upgrade_db(alembic_config: Config, revision: str = "head") -> None:
     command.upgrade(alembic_config, revision)
 
 
+def init_db(database_url: str) -> None:
+    """
+    Runs the migrations and creates all of the database objects.
+    """
+    alembic_config = get_alembic_config(database_url)
+    upgrade_db(alembic_config)
+
+
 def downgrade_db(alembic_config: Config, revision: str = "head") -> None:
     """Downgrade the database to the specified migration revision."""
     log.info(f"Running database downgrade to revision {revision}")
@@ -78,7 +86,7 @@ def reset_db(database_url: str) -> None:
     Drops all tables/metadata from the database.
     """
     log.info("Resetting database...")
-    engine = get_db_engine(database_url)
+    engine = get_db_engine(database_uri=database_url)
     with engine.connect() as connection:
         log.info("Dropping tables...")
         Base.metadata.drop_all(connection)
@@ -136,3 +144,20 @@ def configure_db(database_url: str, revision: Optional[str] = "head") -> None:
         log.error("Unable to configure database: {}: {}", error_type, error)
         log.opt(exception=True).error(error)
         raise
+
+
+def check_missing_migrations(database_url: str) -> None:
+    """
+    Tries to autogenerate migrations, returns True if a migration
+    was generated.
+    """
+
+    engine = get_db_engine(database_uri=database_url)
+    connection = engine.connect()
+
+    migration_context = migration.MigrationContext.configure(connection)
+    result = command.autogen.compare_metadata(migration_context, Base.metadata)  # type: ignore[attr-defined]
+
+    if result:
+        raise SystemExit("Migrations needs to be generated!")
+    print("No migrations need to be generated.")

fides/api/graph/config.py

@@ -95,6 +95,10 @@ from fides.api.graph.data_type import (
     DataTypeConverter,
     get_data_type_converter,
 )
+from fides.api.schemas.partitioning import TimeBasedPartitioning
+from fides.api.schemas.partitioning.time_based_partitioning import (
+    validate_partitioning_list,
+)
 from fides.api.util.collection_util import merge_dicts
 from fides.api.util.querytoken import QueryToken
 
@@ -459,7 +463,7 @@ class Collection(BaseModel):
     grouped_inputs: Set[str] = set()
     data_categories: Set[FidesKey] = set()
     masking_strategy_override: Optional[MaskingStrategyOverride] = None
-    partitioning: Optional[Dict] = None
+    partitioning: Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]] = None
 
     @property
     def field_dict(self) -> Dict[FieldPath, Field]:
@@ -655,14 +659,12 @@ class Collection(BaseModel):
     @field_validator("partitioning")
     @classmethod
     def validate_partitioning(
-        cls, partitioning: Optional[Dict[str, Any]]
-    ) -> Optional[Dict[str, Any]]:
+        cls, partitioning: Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]
+    ) -> Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]:
         """
-        Validates the `partitioning` dict field.
-
-        The `partitioning` dict field is untyped in Fideslang, but here we enforce
-        that it has the required and expected `where_clauses` key, whose value must be
-        a list of strings.
+        Validates the `partitioning` field, which may be either a single dict
+        or a list of dicts.  Each dict must describe either legacy
+        `where_clauses` or the required keys for time-based partitioning.
 
         The string values are validated to ensure they match the expected syntax, which
         is strictly prescribed. The string values MUST be a valid SQL clause that defines
@@ -683,7 +685,12 @@ class Collection(BaseModel):
         in its conditional
 
         """
-        if not partitioning:
+
+        if partitioning is None:
+            return partitioning
+
+        if isinstance(partitioning, list):
+            validate_partitioning_list(partitioning)
             return partitioning
 
         # NOTE: when we deprecate `where_clause` partitioning in favor of a more proper partitioning DSL,

fides/api/graph/traversal.py

@@ -114,6 +114,23 @@ class BaseTraversal:
                 )
             )
 
+        # Ensure manual_task collections execute right after ROOT
+        from fides.api.task.manual.manual_task_utils import ManualTaskAddress
+
+        for addr in self.traversal_node_dict.keys():
+            if ManualTaskAddress.is_manual_task_address(addr):
+                # Add a simple synthetic edge ROOT.id -> manual_data.id
+                self.edges.add(
+                    Edge(
+                        FieldAddress(
+                            ROOT_COLLECTION_ADDRESS.dataset,
+                            ROOT_COLLECTION_ADDRESS.collection,
+                            "id",
+                        ),
+                        addr.field_address(FieldPath("id")),
+                    )
+                )
+
         self._verify_traversal()
 
     def _verify_traversal(self) -> None:
@@ -192,6 +209,7 @@ class BaseTraversal:
         )
         finished_nodes: dict[CollectionAddress, TraversalNode] = {}
         running_node_queue: MatchingQueue[TraversalNode] = MatchingQueue(self.root_node)
+
         remaining_edges: Set[Edge] = self.edges.copy()
         while not running_node_queue.is_empty():
             # this is to support the "run traversal_node A AFTER traversal_node B functionality:"

fides/api/models/attachment.py

@@ -5,9 +5,9 @@ from typing import IO, TYPE_CHECKING, Any, Tuple
 
 from fideslang.validation import AnyHttpUrlString
 from loguru import logger as log
-from sqlalchemy import Column
+from sqlalchemy import Column, DateTime
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import ForeignKey, String, UniqueConstraint, orm
+from sqlalchemy import ForeignKey, Index, String, UniqueConstraint, func, orm
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
@@ -64,8 +64,14 @@ class AttachmentReference(Base):
         """Overriding base class method to set the table name."""
         return "attachment_reference"
 
+    created_at = Column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
     attachment_id = Column(
-        String, ForeignKey("attachment.id", ondelete="CASCADE"), nullable=False
+        String,
+        ForeignKey("attachment.id", name="attachment_reference_attachment_id_fkey"),
+        nullable=False,
     )
     reference_id = Column(String, nullable=False)
     reference_type = Column(EnumColumn(AttachmentReferenceType), nullable=False)
@@ -74,6 +80,8 @@ class AttachmentReference(Base):
         UniqueConstraint(
             "attachment_id", "reference_id", name="_attachment_reference_uc"
         ),
+        Index("ix_attachment_reference_reference_id", "reference_id"),
+        Index("ix_attachment_reference_reference_type", "reference_type"),
     )
 
     # Relationships
@@ -92,6 +100,10 @@ class Attachment(Base):
     Stores information about an Attachment.
     """
 
+    created_at = Column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
     user_id = Column(
         String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
     )

fides/api/models/comment.py

@@ -1,13 +1,13 @@
 from enum import Enum as EnumType
 from typing import TYPE_CHECKING, Any
 
-from sqlalchemy import Column
+from sqlalchemy import Column, DateTime
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import ForeignKey, String, UniqueConstraint, orm
+from sqlalchemy import ForeignKey, Index, String, func, orm
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
-from fides.api.db.base_class import Base
+from fides.api.db.base_class import Base, FidesBase
 from fides.api.models.attachment import Attachment, AttachmentReferenceType
 
 if TYPE_CHECKING:
@@ -36,6 +36,7 @@ class CommentReferenceType(str, EnumType):
     access_manual_webhook = "access_manual_webhook"
     erasure_manual_webhook = "erasure_manual_webhook"
     privacy_request = "privacy_request"
+    manual_task_submission = "manual_task_submission"
 
 
 class CommentReference(Base):
@@ -48,12 +49,24 @@ class CommentReference(Base):
         """Overriding base class method to set the table name."""
         return "comment_reference"
 
-    comment_id = Column(String, ForeignKey("comment.id"), nullable=False)
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+    created_at = Column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
+    comment_id = Column(
+        String,
+        ForeignKey(
+            "comment.id", name="comment_reference_comment_id_fkey", ondelete="CASCADE"
+        ),
+        nullable=False,
+    )
     reference_id = Column(String, nullable=False)
     reference_type = Column(EnumColumn(CommentReferenceType), nullable=False)
 
     __table_args__ = (
-        UniqueConstraint("comment_id", "reference_id", name="comment_reference_uc"),
+        Index("ix_comment_reference_reference_id", "reference_id"),
+        Index("ix_comment_reference_reference_type", "reference_type"),
     )
 
     comment = relationship(
@@ -75,6 +88,11 @@ class Comment(Base):
     Stores information about a Comment.
     """
 
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+    created_at = Column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
     user_id = Column(
         String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
     )

fides/api/models/connectionconfig.py

@@ -53,6 +53,7 @@ class ConnectionType(enum.Enum):
     https = "https"
     manual = "manual"  # Deprecated - use manual_webhook instead
     manual_webhook = "manual_webhook"  # Runs upfront before the traversal
+    manual_task = "manual_task"  # Manual task integration
     mariadb = "mariadb"
     mongodb = "mongodb"
     mssql = "mssql"
@@ -88,6 +89,7 @@ class ConnectionType(enum.Enum):
             ConnectionType.google_cloud_sql_postgres.value: "Google Cloud SQL for Postgres",
             ConnectionType.https.value: "Policy Webhook",
             ConnectionType.manual_webhook.value: "Manual Process",
+            ConnectionType.manual_task.value: "Manual Task",
             ConnectionType.manual.value: "Manual Connector",
             ConnectionType.mariadb.value: "MariaDB",
             ConnectionType.mongodb.value: "MongoDB",
@@ -132,6 +134,7 @@ class ConnectionType(enum.Enum):
             ConnectionType.google_cloud_sql_postgres.value: SystemType.database,
             ConnectionType.https.value: SystemType.manual,
             ConnectionType.manual_webhook.value: SystemType.manual,
+            ConnectionType.manual_task.value: SystemType.manual,
             ConnectionType.manual.value: SystemType.manual,
             ConnectionType.mariadb.value: SystemType.database,
             ConnectionType.mongodb.value: SystemType.database,
@@ -229,6 +232,14 @@ class ConnectionConfig(Base):
         uselist=False,
     )
 
+    manual_task = relationship(  # type: ignore[misc]
+        "ManualTask",
+        primaryjoin="and_(ConnectionConfig.id == foreign(ManualTask.parent_entity_id), "
+        "ManualTask.parent_entity_type == 'connection_config')",
+        cascade="delete",
+        uselist=False,
+    )
+
     pre_approval_webhooks = relationship(  # type: ignore[misc]
         "PreApprovalWebhook",
         back_populates="connection_config",

fides/api/models/db_cache.py

@@ -32,7 +32,7 @@ class DBCache(Base):
     cache_value = Column(BYTEA, nullable=False)
 
     __table_args__ = (
-        Index("ix_dbcache_namespace_cache_key", "namespace", "cache_key"),
+        Index("ix_dbcache_namespace_cache_key", "namespace", "cache_key", unique=True),
     )
 
     @classmethod

fides/api/models/detection_discovery/core.py

@@ -159,6 +159,8 @@ class MonitorConfig(Base):
         "MonitorExecution",
         cascade="all, delete-orphan",
         backref="monitor_config",
+        primaryjoin="MonitorExecution.monitor_config_key == foreign(MonitorConfig.key)",
+        single_parent=True,
     )
 
     shared_config_id = Column(
@@ -342,16 +344,16 @@ class StagedResourceAncestor(Base):
     Its entries should be deleted when the staged resource is deleted, via cascade.
     """
 
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+
     ancestor_urn = Column(
         String,
         ForeignKey("stagedresource.urn", ondelete="CASCADE"),
-        primary_key=True,
         nullable=False,
     )
     descendant_urn = Column(
         String,
         ForeignKey("stagedresource.urn", ondelete="CASCADE"),
-        primary_key=True,
         nullable=False,
     )
 
@@ -372,6 +374,7 @@ class StagedResourceAncestor(Base):
         UniqueConstraint(
             "ancestor_urn", "descendant_urn", name="uq_staged_resource_ancestor"
         ),
+        Index("ix_staged_resource_ancestor_pkey", "id", unique=True),
         Index("ix_staged_resource_ancestor_ancestor", "ancestor_urn"),
         Index("ix_staged_resource_ancestor_descendant", "descendant_urn"),
     )
@@ -470,12 +473,7 @@ class StagedResource(Base):
         server_default="{}",
         default=dict,
     )
-    user_assigned_system_id = Column(
-        String,
-        ForeignKey(System.id_field_path),
-        nullable=True,
-        index=True,
-    )
+    user_assigned_system_id = Column(String, nullable=True, index=True)
 
     # pointers to child and parent URNs
     children = Column(
@@ -527,7 +525,7 @@ class StagedResource(Base):
 
     data_uses = Column(
         ARRAY(String),
-        nullable=False,
+        nullable=True,
         server_default="{}",
         default=dict,
     )
@@ -588,12 +586,14 @@ class MonitorExecution(Base):
     configuration details used in connecting to the external data store.
     """
 
-    monitor_config_key = Column(
-        String,
-        ForeignKey(MonitorConfig.key),
-        nullable=False,
-        index=True,
-    )
+    # redefined here because there's a minor, unintended discrepancy between
+    # this `id` field and that of the `Base` class, which explicitly sets `index=True`.
+    # TODO: we likely should _not_ be setting `index=True` on the `id`
+    # attribute of the `Base` class, as `primary_key=True` already specifies a
+    # primary key constraint, which will implicitly create an index for the field.
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+
+    monitor_config_key = Column(String, nullable=False, index=True)
     status = Column(String, nullable=True)
     started = Column(
         DateTime(timezone=True),

fides/api/models/fides_user_respondent_email_verification.py

@@ -3,12 +3,13 @@ from __future__ import annotations
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, Optional
 
-from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy import Column, DateTime, ForeignKey, Index, String
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
+from sqlalchemy.sql import func
 
 from fides.api.cryptography.cryptographic_util import generate_secure_random_string
-from fides.api.db.base_class import Base
+from fides.api.db.base_class import Base, FidesBase
 from fides.api.util.identity_verification import IdentityVerificationMixin
 from fides.config import get_config
 
@@ -40,6 +41,22 @@ class FidesUserRespondentEmailVerification(Base, IdentityVerificationMixin):
     def __tablename__(self) -> str:
         return "fides_user_respondent_email_verification"
 
+    # redefined here because there's a minor, unintended discrepancy between
+    # this `id` field and that of the `Base` class, which explicitly sets `index=True`.
+    # TODO: we likely should _not_ be setting `index=True` on the `id`
+    # attribute of the `Base` class, as `primary_key=True` already specifies a
+    # primary key constraint, which will implicitly create an index for the field.
+    id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
+    created_at = Column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        DateTime(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+
     user_id = Column(
         String,
         ForeignKey("fidesuser.id", ondelete="CASCADE"),
@@ -59,6 +76,14 @@ class FidesUserRespondentEmailVerification(Base, IdentityVerificationMixin):
         foreign_keys=[user_id],
     )
 
+    __table_args__ = (
+        Index(
+            "ix_fides_user_respondent_email_verification_id",
+            "id",
+            unique=True,
+        ),
+    )
+
     @classmethod
     def create(
         cls, db: Session, *, data: dict[str, Any], check_name: bool = False