ethyca-fides 2.70.4b0__py2.py3-none-any.whl → 2.70.4rc0__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/a8e0c016afd_add_classification_benchmark_table.py

@@ -1,126 +0,0 @@
-"""Add classification benchmark table
-
-Revision ID: a8e0c016afd
-Revises: 4d8c0fcc5771
-Create Date: 2024-12-19 12:00:00.000000
-
-"""
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "a8e0c016afd"
-down_revision = "4d8c0fcc5771"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "classification_benchmark",
-        sa.Column("id", sa.String(length=255), nullable=False),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=True,
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=True,
-        ),
-        sa.Column(
-            "monitor_config_key",
-            sa.String(),
-            sa.ForeignKey("monitorconfig.key", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column(
-            "dataset_fides_key",
-            sa.String(),
-            sa.ForeignKey("ctl_datasets.fides_key", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column(
-            "resource_urns",
-            postgresql.ARRAY(sa.String()),
-            nullable=False,
-            server_default="{}",
-        ),
-        sa.Column(
-            "overall_metrics",
-            postgresql.JSONB(astext_type=sa.Text()),
-            nullable=True,
-            server_default=None,
-        ),
-        sa.Column(
-            "field_accuracy_details",
-            postgresql.ARRAY(postgresql.JSONB(astext_type=sa.Text())),
-            nullable=False,
-            server_default="{}",
-        ),
-        sa.Column(
-            "status",
-            sa.String(length=50),
-            nullable=False,
-            server_default="pending",
-        ),
-        sa.Column(
-            "messages",
-            postgresql.ARRAY(sa.String()),
-            nullable=False,
-            server_default="{}",
-        ),
-        sa.PrimaryKeyConstraint("id"),
-    )
-    op.create_index(
-        op.f("ix_classification_benchmark_id"),
-        "classification_benchmark",
-        ["id"],
-        unique=False,
-    )
-    op.create_index(
-        op.f("ix_classification_benchmark_monitor_config_key"),
-        "classification_benchmark",
-        ["monitor_config_key"],
-        unique=False,
-    )
-    op.create_index(
-        op.f("ix_classification_benchmark_dataset_fides_key"),
-        "classification_benchmark",
-        ["dataset_fides_key"],
-        unique=False,
-    )
-    op.create_index(
-        op.f("ix_classification_benchmark_status"),
-        "classification_benchmark",
-        ["status"],
-        unique=False,
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(
-        op.f("ix_classification_benchmark_status"),
-        table_name="classification_benchmark",
-    )
-    op.drop_index(
-        op.f("ix_classification_benchmark_dataset_fides_key"),
-        table_name="classification_benchmark",
-    )
-    op.drop_index(
-        op.f("ix_classification_benchmark_monitor_config_key"),
-        table_name="classification_benchmark",
-    )
-    op.drop_index(
-        op.f("ix_classification_benchmark_id"), table_name="classification_benchmark"
-    )
-    op.drop_table("classification_benchmark")
-    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/cd8649be3a2b_add_classifications_and_user_assigned_.py

@@ -1,74 +0,0 @@
-"""add classifications and user_assigned_data_categories indices
-
-Revision ID: cd8649be3a2b
-Revises: a8e0c016afd
-Create Date: 2025-09-19 13:49:02.762672
-
-"""
-
-import sqlalchemy as sa
-from alembic import op
-from loguru import logger
-
-# revision identifiers, used by Alembic.
-revision = "cd8649be3a2b"
-down_revision = "a8e0c016afd"
-branch_labels = None
-depends_on = None
-
-"""
-WARNING - Conditional migration based on table size
-"""
-
-
-def upgrade():
-    # Check stagedresource table size
-    connection = op.get_bind()
-
-    # Check stagedresource table size to decide if we should create indexes immediately
-    table_size = connection.execute(
-        sa.text("SELECT COUNT(*) FROM stagedresource")
-    ).scalar()
-
-    if table_size < 1000000:
-        logger.info(
-            f"stagedresource table has {table_size} rows, creating GIN indices directly"
-        )
-
-        # Create GIN index for user_assigned_data_categories array operations (&&, @>, <@)
-        op.execute(
-            "CREATE INDEX idx_stagedresource_user_categories_gin "
-            "ON stagedresource USING GIN (user_assigned_data_categories)"
-        )
-
-        # Create GIN index for classifications array operations (&&, @>, <@)
-        op.execute(
-            "CREATE INDEX idx_stagedresource_classifications_gin "
-            "ON stagedresource USING GIN (classifications)"
-        )
-
-        logger.info("GIN indices created successfully")
-
-    else:
-        logger.warning(
-            "The stagedresource table has more than 1 million rows, "
-            "skipping index creation. Indexes will be created during application startup "
-            "via post_upgrade_index_creation.py"
-        )
-
-
-def downgrade():
-    # Drop indices in reverse order
-    logger.info("Dropping GIN indices for stagedresource arrays")
-
-    try:
-        op.drop_index("idx_stagedresource_classifications_gin", "stagedresource")
-        logger.info("Dropped classifications GIN index")
-    except Exception as e:
-        logger.warning(f"Could not drop classifications index: {e}")
-
-    try:
-        op.drop_index("idx_stagedresource_user_categories_gin", "stagedresource")
-        logger.info("Dropped user_assigned_data_categories GIN index")
-    except Exception as e:
-        logger.warning(f"Could not drop user_categories index: {e}")

fides/api/db/safe_crud.py

@@ -1,377 +0,0 @@
-"""
-This module contains "safe" versions of CRUD operations that do NOT
-manually manage database transactions. Transaction management is left
-to the calling code (endpoints, services) to handle at the appropriate
-boundary.
-
-Use functions from this module instead of fides.api.db.crud
-"""
-
-from collections import defaultdict
-from typing import Any, Dict, List, Tuple, Type, TypeVar
-
-from fastapi import HTTPException
-from loguru import logger as log
-from sqlalchemy import and_, column
-from sqlalchemy import delete as _delete
-from sqlalchemy import or_
-from sqlalchemy import update as _update
-from sqlalchemy.dialects.postgresql import Insert as _insert
-from sqlalchemy.exc import IntegrityError, SQLAlchemyError
-from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.future import select
-from sqlalchemy.sql import Select
-from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
-
-from fides.api.models.sql_models import (  # type: ignore[attr-defined]
-    CustomField,
-    CustomFieldDefinition,
-    FidesBase,
-    ResourceTypes,
-)
-from fides.api.util import errors
-
-# Helps return type be linked to the type of the parameter
-T = TypeVar("T", bound="FidesBase")
-
-
-# CRUD Functions
-async def create_resource(
-    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
-) -> T:
-    """
-    Create a resource in the database.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-    with log.contextualize(
-        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
-    ):
-
-        existing_resource = await get_resource(
-            sql_model, resource_dict["fides_key"], async_session, raise_not_found=False
-        )
-
-        if existing_resource is not None:
-            already_exists_error = errors.AlreadyExistsError(
-                sql_model.__name__, resource_dict["fides_key"]
-            )
-            log.bind(error=already_exists_error.detail["error"]).info(  # type: ignore[index]
-                "Failed to insert resource"
-            )
-            raise already_exists_error
-
-        try:
-            log.debug("Creating resource")
-            query = _insert(sql_model.__table__).values(resource_dict)
-            await async_session.execute(query)
-        except SQLAlchemyError as e:
-            log.exception(f"Failed to create resource with error: '{e}'")
-            sa_error = errors.QueryError()
-            raise sa_error
-
-        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
-
-
-async def get_custom_fields_filtered(
-    async_session: AsyncSession,
-    resource_types_to_ids: Dict[ResourceTypes, List[str]] = defaultdict(list),
-) -> FidesBase:
-    """
-    Utility function to construct a filtered query for custom field values based on provided mapping of
-    resource types to resource IDs.
-
-    Only custom fields with an "active" CustomFieldDefinition are returned.
-
-    This is for use in bulk querying of custom fields, to avoid multiple round trips to the db.
-    """
-    with log.contextualize(model=CustomField):
-        try:
-            log.debug("Fetching resource")
-            query = select(
-                CustomField.resource_id,
-                CustomField.value,
-                CustomFieldDefinition.resource_type,
-                CustomFieldDefinition.name,
-                CustomFieldDefinition.field_type,
-            ).join(
-                CustomFieldDefinition,
-                CustomFieldDefinition.id == CustomField.custom_field_definition_id,
-            )
-
-            criteria = [
-                and_(
-                    CustomFieldDefinition.resource_type == resource_type.value,
-                    CustomField.resource_id.in_(resource_ids),
-                    # pylint: disable=singleton-comparison
-                    CustomFieldDefinition.active == True,
-                )
-                for resource_type, resource_ids in resource_types_to_ids.items()
-            ]
-            query = query.where(or_(False, *criteria))
-            result = await async_session.execute(query)
-            return result.mappings().all()
-        except SQLAlchemyError as e:
-            sa_error = errors.QueryError()
-            log.exception(f"Failed to fetch custom fields with error: '{e}'")
-            raise sa_error
-
-
-async def get_resource(
-    sql_model: Type[T],
-    fides_key: str,
-    async_session: AsyncSession,
-    raise_not_found: bool = True,
-) -> T:
-    """
-    Get a resource from the database by its FidesKey.
-
-    Returns a SQLAlchemy model of that resource.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
-        try:
-            log.debug("Fetching resource")
-            query = select(sql_model).where(sql_model.fides_key == fides_key)
-            result = await async_session.execute(query)
-        except SQLAlchemyError as e:
-            sa_error = errors.QueryError()
-            log.exception(f"Failed to fetch resource with error: '{e}'")
-            raise sa_error
-
-        sql_resource = result.scalars().first()
-        if sql_resource is None and raise_not_found:
-            not_found_error = errors.NotFoundError(sql_model.__name__, fides_key)
-            log.bind(error=not_found_error.detail["error"]).info("Resource not found")  # type: ignore[index]
-            raise not_found_error
-
-        return sql_resource
-
-
-async def get_resource_with_custom_fields(
-    sql_model: Type[T], fides_key: str, async_session: AsyncSession
-) -> Dict[str, Any]:
-    """Get a resource from the database by its FidesKey including it's custom fields.
-
-    Returns a dictionary of that resource.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-    resource: T = await get_resource(sql_model, fides_key, async_session)
-    resource_dict = resource.__dict__
-    resource_dict.pop("_sa_instance_state", None)
-
-    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
-        try:
-            log.debug("Fetching custom fields for resource")
-            query = (
-                select(CustomFieldDefinition.name, CustomField.value)
-                .join(
-                    CustomField,
-                    CustomField.custom_field_definition_id == CustomFieldDefinition.id,
-                )
-                .where(
-                    (CustomField.resource_id == resource.fides_key)
-                    & (  # pylint: disable=singleton-comparison
-                        CustomFieldDefinition.active == True
-                    )
-                )
-            )
-            result = await async_session.execute(query)
-        except SQLAlchemyError as e:
-            sa_error = errors.QueryError()
-            log.exception(f"Failed to fetch custom fields with error: '{e}'")
-            raise sa_error
-
-        custom_fields = result.mappings().all()
-
-    if not custom_fields:
-        return resource_dict
-
-    for field in custom_fields:
-        if field["name"] in resource_dict:
-            resource_dict[field["name"]] = (
-                f"{resource_dict[field['name']]}, {', '.join(field['value'])}"
-            )
-        else:
-            resource_dict[field["name"]] = ", ".join(field["value"])
-
-    return resource_dict
-
-
-async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List[T]:
-    """
-    Get a list of all of the resources of this type from the database.
-
-    Returns a list of SQLAlchemy models of that resource type.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-    query = select(sql_model)
-    return await list_resource_query(async_session, query, sql_model)
-
-
-async def list_resource_query(
-    async_session: AsyncSession, query: Select, sql_model: Type[T]
-) -> List[T]:
-    """
-    Utility function to wrap a select query in generic "list_resource" execution handling.
-    Wrapping includes execution against the DB session, logging and error handling.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-
-    with log.contextualize(sql_model=sql_model.__name__):
-        try:
-            log.debug("Fetching resources")
-            result = await async_session.execute(query)
-            sql_resources = result.scalars().all()
-        except SQLAlchemyError as e:
-            log.exception(f"Failed to fetch resources with error: '{e}'")
-            sa_error = errors.QueryError()
-            raise sa_error
-
-        return sql_resources
-
-
-async def update_resource(
-    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
-) -> Dict:
-    """
-    Update a resource in the database by its fides_key.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-
-    with log.contextualize(
-        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
-    ):
-        await get_resource(sql_model, resource_dict["fides_key"], async_session)
-
-        try:
-            log.debug("Updating resource")
-            await async_session.execute(
-                _update(sql_model.__table__)
-                .where(sql_model.fides_key == resource_dict["fides_key"])
-                .values(resource_dict)
-            )
-        except SQLAlchemyError as e:
-            log.exception(f"Failed to update resource with error: '{e}'")
-            sa_error = errors.QueryError()
-            raise sa_error
-
-        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
-
-
-async def upsert_resources(
-    sql_model: Type[T], resource_dicts: List[Dict], async_session: AsyncSession
-) -> Tuple[int, int]:
-    """
-    Insert new resources into the database. If a resource already exists,
-    update it by it's fides_key.
-
-    Returns a Tuple containing the counts of inserted and updated rows.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-
-    with log.contextualize(
-        sql_model=sql_model.__name__,
-        fides_keys=[resource["fides_key"] for resource in resource_dicts],
-    ):
-        try:
-            log.debug("Upserting resources")
-            insert_stmt = (
-                _insert(sql_model.__table__)
-                .values(resource_dicts)
-                .returning(
-                    (column("xmax") == 0),  # Row was inserted
-                    (column("xmax") != 0),  # Row was updated
-                )
-            )
-
-            excluded = dict(insert_stmt.excluded.items())  # type: ignore[attr-defined]
-            excluded.pop("id", None)  # If updating, don't update the "id"
-
-            result = await async_session.execute(
-                insert_stmt.on_conflict_do_update(
-                    index_elements=["fides_key"],
-                    set_=excluded,
-                )
-            )
-
-            inserts, updates = 0, 0
-            for xmax in result:
-                inserts += 1 if xmax[0] else 0
-                updates += 1 if xmax[1] else 0
-
-            return (inserts, updates)
-
-        except SQLAlchemyError as e:
-            log.exception(f"Failed to upsert resources with error: '{e}'")
-            sa_error = errors.QueryError()
-            raise sa_error
-
-
-async def delete_resource(
-    sql_model: Type[T], fides_key: str, async_session: AsyncSession
-) -> T:
-    """
-    Delete a resource by its fides_key.
-
-    If the resource has child keys referring to it, also delete those.
-
-    This version does NOT manually manage transactions - transaction management
-    is left to the calling code.
-    """
-
-    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
-        sql_resource = await get_resource(sql_model, fides_key, async_session)
-
-        try:
-            # Automatically delete related resources if they are CTL objects
-            if hasattr(sql_model, "parent_key"):
-                log.debug("Deleting resource and its children")
-                query = (
-                    _delete(sql_model.__table__)
-                    .where(
-                        or_(
-                            sql_model.fides_key == fides_key,
-                            sql_model.fides_key.like(f"{fides_key}.%"),
-                        )
-                    )
-                    .execution_options(synchronize_session=False)
-                )
-            else:
-                log.debug("Deleting resource")
-                query = _delete(sql_model.__table__).where(
-                    sql_model.fides_key == fides_key
-                )
-            await async_session.execute(query)
-        except IntegrityError as err:
-            raw_error_text: str = err.orig.args[0]
-
-            if "violates foreign key constraint" in raw_error_text:
-                error_message = "Failed to delete resource! Foreign key constraint found, try deleting related resources first."
-            else:
-                error_message = "Failed to delete resource!"
-
-            log.bind(error="SQL Query integrity error!").error(raw_error_text)
-            raise HTTPException(
-                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=error_message,
-            )
-        except SQLAlchemyError as e:
-            log.exception(f"Failed to delete resource with error: '{e}'")
-            sa_error = errors.QueryError()
-            raise sa_error
-
-        return sql_resource

fides/api/models/conditional_dependency/conditional_dependency_base.py

@@ -1,82 +0,0 @@
-from enum import Enum
-from typing import Any, Optional
-
-from sqlalchemy import Column, Integer, String
-from sqlalchemy.dialects.postgresql import JSONB
-from sqlalchemy.orm import Session
-
-from fides.api.db.base_class import Base
-from fides.api.db.util import EnumColumn
-from fides.api.task.conditional_dependencies.schemas import (
-    Condition,
-    ConditionGroup,
-    ConditionLeaf,
-)
-
-
-class ConditionalDependencyType(str, Enum):
-    """Shared enum for conditional dependency node types."""
-
-    leaf = "leaf"
-    group = "group"
-
-
-class ConditionalDependencyBase(Base):
-    """Abstract base class for all conditional dependency models."""
-
-    __abstract__ = True
-
-    # Tree structure - parent_id defined in concrete classes for proper foreign keys
-    condition_type = Column(EnumColumn(ConditionalDependencyType), nullable=False)
-
-    # Condition details (for leaf nodes)
-    field_address = Column(String(255), nullable=True)  # For leaf conditions
-    operator = Column(String, nullable=True)  # For leaf conditions
-    value = Column(JSONB, nullable=True)  # For leaf conditions
-    logical_operator = Column(String, nullable=True)  # 'and' or 'or' for groups
-
-    # Ordering
-    sort_order = Column(Integer, nullable=False, default=0)
-
-    def to_condition_leaf(self) -> ConditionLeaf:
-        """Convert to ConditionLeaf if this is a leaf condition"""
-        if self.condition_type != "leaf":
-            raise ValueError("Cannot convert group condition to leaf")
-
-        return ConditionLeaf(
-            field_address=self.field_address, operator=self.operator, value=self.value
-        )
-
-    def to_condition_group(self) -> ConditionGroup:
-        """Convert to ConditionGroup if this is a group condition"""
-        if self.condition_type != "group":
-            raise ValueError("Cannot convert leaf condition to group")
-
-        # Recursively build children
-        child_conditions = []
-        children_list = [child for child in self.children]  # type: ignore[attr-defined]
-        for child in sorted(children_list, key=lambda x: x.sort_order):
-            if child.condition_type == "leaf":
-                child_conditions.append(child.to_condition_leaf())
-            else:
-                child_conditions.append(child.to_condition_group())
-
-        return ConditionGroup(
-            logical_operator=self.logical_operator, conditions=child_conditions
-        )
-
-    @classmethod
-    def get_root_condition(
-        cls, db: Session, *args: Any, **kwargs: Any
-    ) -> Optional[Condition]:
-        """Get the root condition for a parent entity - implemented by subclasses
-
-        Args:
-            db: Database session
-            *args: Additional positional arguments specific to each implementation
-            **kwargs: Additional keyword arguments specific to each implementation
-
-        Returns:
-            Optional[Condition]: Root condition or None if not found
-        """
-        raise NotImplementedError("Subclasses must implement get_root_condition")