ethyca-fides 2.70.1rc3__py2.py3-none-any.whl → 2.70.3b0__py2.py3-none-any.whl

fides/api/db/safe_crud.py

@@ -0,0 +1,377 @@
+"""
+This module contains "safe" versions of CRUD operations that do NOT
+manually manage database transactions. Transaction management is left
+to the calling code (endpoints, services) to handle at the appropriate
+boundary.
+
+Use functions from this module instead of fides.api.db.crud
+"""
+
+from collections import defaultdict
+from typing import Any, Dict, List, Tuple, Type, TypeVar
+
+from fastapi import HTTPException
+from loguru import logger as log
+from sqlalchemy import and_, column
+from sqlalchemy import delete as _delete
+from sqlalchemy import or_
+from sqlalchemy import update as _update
+from sqlalchemy.dialects.postgresql import Insert as _insert
+from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.future import select
+from sqlalchemy.sql import Select
+from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
+
+from fides.api.models.sql_models import (  # type: ignore[attr-defined]
+    CustomField,
+    CustomFieldDefinition,
+    FidesBase,
+    ResourceTypes,
+)
+from fides.api.util import errors
+
+# Helps return type be linked to the type of the parameter
+T = TypeVar("T", bound="FidesBase")
+
+
+# CRUD Functions
+async def create_resource(
+    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
+) -> T:
+    """
+    Create a resource in the database.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    with log.contextualize(
+        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
+    ):
+
+        existing_resource = await get_resource(
+            sql_model, resource_dict["fides_key"], async_session, raise_not_found=False
+        )
+
+        if existing_resource is not None:
+            already_exists_error = errors.AlreadyExistsError(
+                sql_model.__name__, resource_dict["fides_key"]
+            )
+            log.bind(error=already_exists_error.detail["error"]).info(  # type: ignore[index]
+                "Failed to insert resource"
+            )
+            raise already_exists_error
+
+        try:
+            log.debug("Creating resource")
+            query = _insert(sql_model.__table__).values(resource_dict)
+            await async_session.execute(query)
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to create resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+
+async def get_custom_fields_filtered(
+    async_session: AsyncSession,
+    resource_types_to_ids: Dict[ResourceTypes, List[str]] = defaultdict(list),
+) -> FidesBase:
+    """
+    Utility function to construct a filtered query for custom field values based on provided mapping of
+    resource types to resource IDs.
+
+    Only custom fields with an "active" CustomFieldDefinition are returned.
+
+    This is for use in bulk querying of custom fields, to avoid multiple round trips to the db.
+    """
+    with log.contextualize(model=CustomField):
+        try:
+            log.debug("Fetching resource")
+            query = select(
+                CustomField.resource_id,
+                CustomField.value,
+                CustomFieldDefinition.resource_type,
+                CustomFieldDefinition.name,
+                CustomFieldDefinition.field_type,
+            ).join(
+                CustomFieldDefinition,
+                CustomFieldDefinition.id == CustomField.custom_field_definition_id,
+            )
+
+            criteria = [
+                and_(
+                    CustomFieldDefinition.resource_type == resource_type.value,
+                    CustomField.resource_id.in_(resource_ids),
+                    # pylint: disable=singleton-comparison
+                    CustomFieldDefinition.active == True,
+                )
+                for resource_type, resource_ids in resource_types_to_ids.items()
+            ]
+            query = query.where(or_(False, *criteria))
+            result = await async_session.execute(query)
+            return result.mappings().all()
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch custom fields with error: '{e}'")
+            raise sa_error
+
+
+async def get_resource(
+    sql_model: Type[T],
+    fides_key: str,
+    async_session: AsyncSession,
+    raise_not_found: bool = True,
+) -> T:
+    """
+    Get a resource from the database by its FidesKey.
+
+    Returns a SQLAlchemy model of that resource.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        try:
+            log.debug("Fetching resource")
+            query = select(sql_model).where(sql_model.fides_key == fides_key)
+            result = await async_session.execute(query)
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch resource with error: '{e}'")
+            raise sa_error
+
+        sql_resource = result.scalars().first()
+        if sql_resource is None and raise_not_found:
+            not_found_error = errors.NotFoundError(sql_model.__name__, fides_key)
+            log.bind(error=not_found_error.detail["error"]).info("Resource not found")  # type: ignore[index]
+            raise not_found_error
+
+        return sql_resource
+
+
+async def get_resource_with_custom_fields(
+    sql_model: Type[T], fides_key: str, async_session: AsyncSession
+) -> Dict[str, Any]:
+    """Get a resource from the database by its FidesKey including it's custom fields.
+
+    Returns a dictionary of that resource.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    resource: T = await get_resource(sql_model, fides_key, async_session)
+    resource_dict = resource.__dict__
+    resource_dict.pop("_sa_instance_state", None)
+
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        try:
+            log.debug("Fetching custom fields for resource")
+            query = (
+                select(CustomFieldDefinition.name, CustomField.value)
+                .join(
+                    CustomField,
+                    CustomField.custom_field_definition_id == CustomFieldDefinition.id,
+                )
+                .where(
+                    (CustomField.resource_id == resource.fides_key)
+                    & (  # pylint: disable=singleton-comparison
+                        CustomFieldDefinition.active == True
+                    )
+                )
+            )
+            result = await async_session.execute(query)
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch custom fields with error: '{e}'")
+            raise sa_error
+
+        custom_fields = result.mappings().all()
+
+    if not custom_fields:
+        return resource_dict
+
+    for field in custom_fields:
+        if field["name"] in resource_dict:
+            resource_dict[field["name"]] = (
+                f"{resource_dict[field['name']]}, {', '.join(field['value'])}"
+            )
+        else:
+            resource_dict[field["name"]] = ", ".join(field["value"])
+
+    return resource_dict
+
+
+async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List[T]:
+    """
+    Get a list of all of the resources of this type from the database.
+
+    Returns a list of SQLAlchemy models of that resource type.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    query = select(sql_model)
+    return await list_resource_query(async_session, query, sql_model)
+
+
+async def list_resource_query(
+    async_session: AsyncSession, query: Select, sql_model: Type[T]
+) -> List[T]:
+    """
+    Utility function to wrap a select query in generic "list_resource" execution handling.
+    Wrapping includes execution against the DB session, logging and error handling.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(sql_model=sql_model.__name__):
+        try:
+            log.debug("Fetching resources")
+            result = await async_session.execute(query)
+            sql_resources = result.scalars().all()
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to fetch resources with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return sql_resources
+
+
+async def update_resource(
+    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
+) -> Dict:
+    """
+    Update a resource in the database by its fides_key.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(
+        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
+    ):
+        await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+        try:
+            log.debug("Updating resource")
+            await async_session.execute(
+                _update(sql_model.__table__)
+                .where(sql_model.fides_key == resource_dict["fides_key"])
+                .values(resource_dict)
+            )
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to update resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+
+async def upsert_resources(
+    sql_model: Type[T], resource_dicts: List[Dict], async_session: AsyncSession
+) -> Tuple[int, int]:
+    """
+    Insert new resources into the database. If a resource already exists,
+    update it by it's fides_key.
+
+    Returns a Tuple containing the counts of inserted and updated rows.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(
+        sql_model=sql_model.__name__,
+        fides_keys=[resource["fides_key"] for resource in resource_dicts],
+    ):
+        try:
+            log.debug("Upserting resources")
+            insert_stmt = (
+                _insert(sql_model.__table__)
+                .values(resource_dicts)
+                .returning(
+                    (column("xmax") == 0),  # Row was inserted
+                    (column("xmax") != 0),  # Row was updated
+                )
+            )
+
+            excluded = dict(insert_stmt.excluded.items())  # type: ignore[attr-defined]
+            excluded.pop("id", None)  # If updating, don't update the "id"
+
+            result = await async_session.execute(
+                insert_stmt.on_conflict_do_update(
+                    index_elements=["fides_key"],
+                    set_=excluded,
+                )
+            )
+
+            inserts, updates = 0, 0
+            for xmax in result:
+                inserts += 1 if xmax[0] else 0
+                updates += 1 if xmax[1] else 0
+
+            return (inserts, updates)
+
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to upsert resources with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+
+async def delete_resource(
+    sql_model: Type[T], fides_key: str, async_session: AsyncSession
+) -> T:
+    """
+    Delete a resource by its fides_key.
+
+    If the resource has child keys referring to it, also delete those.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        sql_resource = await get_resource(sql_model, fides_key, async_session)
+
+        try:
+            # Automatically delete related resources if they are CTL objects
+            if hasattr(sql_model, "parent_key"):
+                log.debug("Deleting resource and its children")
+                query = (
+                    _delete(sql_model.__table__)
+                    .where(
+                        or_(
+                            sql_model.fides_key == fides_key,
+                            sql_model.fides_key.like(f"{fides_key}.%"),
+                        )
+                    )
+                    .execution_options(synchronize_session=False)
+                )
+            else:
+                log.debug("Deleting resource")
+                query = _delete(sql_model.__table__).where(
+                    sql_model.fides_key == fides_key
+                )
+            await async_session.execute(query)
+        except IntegrityError as err:
+            raw_error_text: str = err.orig.args[0]
+
+            if "violates foreign key constraint" in raw_error_text:
+                error_message = "Failed to delete resource! Foreign key constraint found, try deleting related resources first."
+            else:
+                error_message = "Failed to delete resource!"
+
+            log.bind(error="SQL Query integrity error!").error(raw_error_text)
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=error_message,
+            )
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to delete resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return sql_resource

fides/api/migrations/post_upgrade_index_creation.py

@@ -153,6 +153,16 @@ TABLE_OBJECT_MAP: Dict[str, List[Dict[str, str]]] = {
             "statement": "CREATE INDEX CONCURRENTLY ix_stagedresource_system_vendor_consent ON stagedresource (system_id, vendor_id, (meta->>'consent_aggregated'))",
             "type": "index",
         },
+        {
+            "name": "idx_stagedresource_user_categories_gin",
+            "statement": "CREATE INDEX CONCURRENTLY idx_stagedresource_user_categories_gin ON stagedresource USING GIN (user_assigned_data_categories)",
+            "type": "index",
+        },
+        {
+            "name": "idx_stagedresource_classifications_gin",
+            "statement": "CREATE INDEX CONCURRENTLY idx_stagedresource_classifications_gin ON stagedresource USING GIN (classifications)",
+            "type": "index",
+        },
     ],
 }
 

fides/api/models/conditional_dependency/conditional_dependency_base.py

@@ -0,0 +1,82 @@
+from enum import Enum
+from typing import Any, Optional
+
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Session
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+)
+
+
+class ConditionalDependencyType(str, Enum):
+    """Shared enum for conditional dependency node types."""
+
+    leaf = "leaf"
+    group = "group"
+
+
+class ConditionalDependencyBase(Base):
+    """Abstract base class for all conditional dependency models."""
+
+    __abstract__ = True
+
+    # Tree structure - parent_id defined in concrete classes for proper foreign keys
+    condition_type = Column(EnumColumn(ConditionalDependencyType), nullable=False)
+
+    # Condition details (for leaf nodes)
+    field_address = Column(String(255), nullable=True)  # For leaf conditions
+    operator = Column(String, nullable=True)  # For leaf conditions
+    value = Column(JSONB, nullable=True)  # For leaf conditions
+    logical_operator = Column(String, nullable=True)  # 'and' or 'or' for groups
+
+    # Ordering
+    sort_order = Column(Integer, nullable=False, default=0)
+
+    def to_condition_leaf(self) -> ConditionLeaf:
+        """Convert to ConditionLeaf if this is a leaf condition"""
+        if self.condition_type != "leaf":
+            raise ValueError("Cannot convert group condition to leaf")
+
+        return ConditionLeaf(
+            field_address=self.field_address, operator=self.operator, value=self.value
+        )
+
+    def to_condition_group(self) -> ConditionGroup:
+        """Convert to ConditionGroup if this is a group condition"""
+        if self.condition_type != "group":
+            raise ValueError("Cannot convert leaf condition to group")
+
+        # Recursively build children
+        child_conditions = []
+        children_list = [child for child in self.children]  # type: ignore[attr-defined]
+        for child in sorted(children_list, key=lambda x: x.sort_order):
+            if child.condition_type == "leaf":
+                child_conditions.append(child.to_condition_leaf())
+            else:
+                child_conditions.append(child.to_condition_group())
+
+        return ConditionGroup(
+            logical_operator=self.logical_operator, conditions=child_conditions
+        )
+
+    @classmethod
+    def get_root_condition(
+        cls, db: Session, *args: Any, **kwargs: Any
+    ) -> Optional[Condition]:
+        """Get the root condition for a parent entity - implemented by subclasses
+
+        Args:
+            db: Database session
+            *args: Additional positional arguments specific to each implementation
+            **kwargs: Additional keyword arguments specific to each implementation
+
+        Returns:
+            Optional[Condition]: Root condition or None if not found
+        """
+        raise NotImplementedError("Subclasses must implement get_root_condition")

fides/api/models/detection_discovery/__init__.py

@@ -1,3 +1,4 @@
+from .classification_benchmark import ClassificationBenchmark
 from .core import (
     DiffStatus,
     MonitorConfig,
@@ -18,6 +19,7 @@ from .monitor_task import (
 )
 
 __all__ = [
+    "ClassificationBenchmark",
     "DiffStatus",
     "MonitorConfig",
     "MonitorExecution",

fides/api/models/detection_discovery/classification_benchmark.py

@@ -0,0 +1,140 @@
+"""
+Database model for classification benchmark results.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import ARRAY, Column, ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.ext.mutable import MutableDict
+from sqlalchemy.orm import Query, RelationshipProperty, Session, relationship
+
+from fides.api.db.base_class import Base, FidesBase
+from fides.api.models.detection_discovery.core import MonitorConfig
+from fides.api.models.sql_models import Dataset  # type: ignore[attr-defined]
+
+
+class ClassificationBenchmark(Base, FidesBase):
+    """
+    Database model for storing classification benchmark results.
+
+    This model stores the results of classification accuracy benchmarks that compare
+    system-assigned classifications on staged resources with ground truth data from datasets.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "classification_benchmark"
+
+    # Foreign key relationships
+    monitor_config_key = Column(
+        String,
+        ForeignKey(MonitorConfig.key, ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    dataset_fides_key = Column(
+        String,
+        ForeignKey(Dataset.fides_key, ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Resource URNs (array of strings)
+    resource_urns = Column(
+        ARRAY(String),
+        nullable=False,
+        server_default="{}",
+        default=list,
+    )
+
+    # Overall accuracy metrics stored as JSONB
+    overall_metrics = Column(
+        MutableDict.as_mutable(JSONB),
+        nullable=True,
+        server_default=None,
+        default=None,
+    )
+
+    # Field-level accuracy details stored as JSONB array
+    field_accuracy_details = Column(
+        ARRAY(JSONB),
+        nullable=False,
+        server_default="{}",
+        default=list,
+    )
+
+    # Status and messages for tracking benchmark execution
+    status = Column(
+        String,
+        nullable=False,
+        server_default="'pending'",
+        default="pending",
+        index=True,
+    )
+    messages = Column(
+        ARRAY(String),
+        nullable=False,
+        server_default="{}",
+        default=list,
+    )
+
+    # Relationships
+    monitor_config: RelationshipProperty[MonitorConfig] = relationship(
+        MonitorConfig,
+        foreign_keys=[monitor_config_key],
+        primaryjoin="ClassificationBenchmark.monitor_config_key == MonitorConfig.key",
+    )
+
+    dataset: RelationshipProperty[Dataset] = relationship(
+        Dataset,
+        foreign_keys=[dataset_fides_key],
+        primaryjoin="ClassificationBenchmark.dataset_fides_key == Dataset.fides_key",
+    )
+
+    @classmethod
+    def list_benchmarks(
+        cls,
+        db: Session,
+        monitor_config_key: Optional[str] = None,
+        dataset_fides_key: Optional[str] = None,
+        created_after: Optional[datetime] = None,
+        created_before: Optional[datetime] = None,
+        status: Optional[str] = None,
+    ) -> Query:
+        """
+        Get a filtered query for benchmarks.
+
+        Args:
+            db: Database session
+            monitor_config_key: Filter by monitor config key
+            dataset_fides_key: Filter by dataset fides key
+            created_after: Filter by creation date (inclusive)
+            created_before: Filter by creation date (exclusive)
+            status: Filter by status
+        Returns:
+            Filtered query (not executed)
+        """
+        # Build the query with filters
+        query = db.query(cls)
+
+        # Apply filters
+        if monitor_config_key:
+            query = query.filter(cls.monitor_config_key == monitor_config_key)
+        if dataset_fides_key:
+            query = query.filter(cls.dataset_fides_key == dataset_fides_key)
+        if created_after:
+            query = query.filter(cls.created_at >= created_after)
+        if created_before:
+            query = query.filter(cls.created_at < created_before)
+        if status:
+            query = query.filter(cls.status == status)
+
+        # Apply sorting
+        query = query.order_by(cls.created_at.desc())
+
+        return query

fides/api/models/detection_discovery/core.py

@@ -599,6 +599,17 @@ class StagedResource(Base):
             "vendor_id",
             text("(meta->>'consent_aggregated')"),
         ),
+        # GIN indices for array operations (&&, @>, <@)
+        Index(
+            "idx_stagedresource_classifications_gin",
+            "classifications",
+            postgresql_using="gin",
+        ),
+        Index(
+            "idx_stagedresource_user_categories_gin",
+            "user_assigned_data_categories",
+            postgresql_using="gin",
+        ),
     )
 
     @classmethod