ethyca-fides 2.70.1rc3__py2.py3-none-any.whl → 2.70.3b0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-19T12:20:48-0700",
+ "date": "2025-09-19T17:26:35-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "8aaac09cdc6c1b430cb6b26467c533c72436c9aa",
- "version": "2.70.1rc3"
+ "full-revisionid": "fe651d7445cf98722c20b976cf2ebf8b4807527e",
+ "version": "2.70.3b0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/a8e0c016afd_add_classification_benchmark_table.py

@@ -0,0 +1,126 @@
+"""Add classification benchmark table
+
+Revision ID: a8e0c016afd
+Revises: 4d8c0fcc5771
+Create Date: 2024-12-19 12:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "a8e0c016afd"
+down_revision = "4d8c0fcc5771"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "classification_benchmark",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "monitor_config_key",
+            sa.String(),
+            sa.ForeignKey("monitorconfig.key", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "dataset_fides_key",
+            sa.String(),
+            sa.ForeignKey("ctl_datasets.fides_key", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "resource_urns",
+            postgresql.ARRAY(sa.String()),
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column(
+            "overall_metrics",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+            server_default=None,
+        ),
+        sa.Column(
+            "field_accuracy_details",
+            postgresql.ARRAY(postgresql.JSONB(astext_type=sa.Text())),
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column(
+            "status",
+            sa.String(length=50),
+            nullable=False,
+            server_default="pending",
+        ),
+        sa.Column(
+            "messages",
+            postgresql.ARRAY(sa.String()),
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_id"),
+        "classification_benchmark",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_monitor_config_key"),
+        "classification_benchmark",
+        ["monitor_config_key"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_dataset_fides_key"),
+        "classification_benchmark",
+        ["dataset_fides_key"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_status"),
+        "classification_benchmark",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_classification_benchmark_status"),
+        table_name="classification_benchmark",
+    )
+    op.drop_index(
+        op.f("ix_classification_benchmark_dataset_fides_key"),
+        table_name="classification_benchmark",
+    )
+    op.drop_index(
+        op.f("ix_classification_benchmark_monitor_config_key"),
+        table_name="classification_benchmark",
+    )
+    op.drop_index(
+        op.f("ix_classification_benchmark_id"), table_name="classification_benchmark"
+    )
+    op.drop_table("classification_benchmark")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/cd8649be3a2b_add_classifications_and_user_assigned_.py

@@ -0,0 +1,74 @@
+"""add classifications and user_assigned_data_categories indices
+
+Revision ID: cd8649be3a2b
+Revises: a8e0c016afd
+Create Date: 2025-09-19 13:49:02.762672
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "cd8649be3a2b"
+down_revision = "a8e0c016afd"
+branch_labels = None
+depends_on = None
+
+"""
+WARNING - Conditional migration based on table size
+"""
+
+
+def upgrade():
+    # Check stagedresource table size
+    connection = op.get_bind()
+
+    # Check stagedresource table size to decide if we should create indexes immediately
+    table_size = connection.execute(
+        sa.text("SELECT COUNT(*) FROM stagedresource")
+    ).scalar()
+
+    if table_size < 1000000:
+        logger.info(
+            f"stagedresource table has {table_size} rows, creating GIN indices directly"
+        )
+
+        # Create GIN index for user_assigned_data_categories array operations (&&, @>, <@)
+        op.execute(
+            "CREATE INDEX idx_stagedresource_user_categories_gin "
+            "ON stagedresource USING GIN (user_assigned_data_categories)"
+        )
+
+        # Create GIN index for classifications array operations (&&, @>, <@)
+        op.execute(
+            "CREATE INDEX idx_stagedresource_classifications_gin "
+            "ON stagedresource USING GIN (classifications)"
+        )
+
+        logger.info("GIN indices created successfully")
+
+    else:
+        logger.warning(
+            "The stagedresource table has more than 1 million rows, "
+            "skipping index creation. Indexes will be created during application startup "
+            "via post_upgrade_index_creation.py"
+        )
+
+
+def downgrade():
+    # Drop indices in reverse order
+    logger.info("Dropping GIN indices for stagedresource arrays")
+
+    try:
+        op.drop_index("idx_stagedresource_classifications_gin", "stagedresource")
+        logger.info("Dropped classifications GIN index")
+    except Exception as e:
+        logger.warning(f"Could not drop classifications index: {e}")
+
+    try:
+        op.drop_index("idx_stagedresource_user_categories_gin", "stagedresource")
+        logger.info("Dropped user_assigned_data_categories GIN index")
+    except Exception as e:
+        logger.warning(f"Could not drop user_categories index: {e}")

fides/api/db/crud.py

@@ -1,6 +1,6 @@
 """
-Contains all of the generic CRUD endpoints that can be
-generated programmatically for each resource.
+DEPRECATED: This module uses manual transaction handling which can
+lead to unexpected behavior. Use fides.api.db.safe_crud instead.
 """
 
 from collections import defaultdict
@@ -18,6 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.sql import Select
 from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
+from typing_extensions import deprecated
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined]
     CustomField,
@@ -32,6 +33,9 @@ T = TypeVar("T", bound="FidesBase")
 
 
 # CRUD Functions
+@deprecated(
+    "This function uses a manual session.begin() which can lead to unexpected transaction handling. Use create_resource from safe_crud instead."
+)
 async def create_resource(
     sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
 ) -> T:
@@ -66,6 +70,9 @@ async def create_resource(
         return await get_resource(sql_model, resource_dict["fides_key"], async_session)
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use get_custom_fields_filtered from safe_crud instead."
+)
 async def get_custom_fields_filtered(
     async_session: AsyncSession,
     resource_types_to_ids: Dict[ResourceTypes, List[str]] = defaultdict(list),
@@ -111,6 +118,9 @@ async def get_custom_fields_filtered(
                 raise sa_error
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use get_resource from safe_crud instead."
+)
 async def get_resource(
     sql_model: Type[T],
     fides_key: str,
@@ -142,6 +152,9 @@ async def get_resource(
             return sql_resource
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use get_resource_with_custom_fields from safe_crud instead."
+)
 async def get_resource_with_custom_fields(
     sql_model: Type[T], fides_key: str, async_session: AsyncSession
 ) -> Dict[str, Any]:
@@ -193,6 +206,9 @@ async def get_resource_with_custom_fields(
     return resource_dict
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use list_resource from safe_crud instead."
+)
 async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List[T]:
     """
     Get a list of all of the resources of this type from the database.
@@ -203,6 +219,9 @@ async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List
     return await list_resource_query(async_session, query, sql_model)
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use list_resource_query from safe_crud instead."
+)
 async def list_resource_query(
     async_session: AsyncSession, query: Select, sql_model: Type[T]
 ) -> List[T]:
@@ -225,6 +244,9 @@ async def list_resource_query(
             return sql_resources
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use update_resource from safe_crud instead."
+)
 async def update_resource(
     sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
 ) -> Dict:
@@ -251,6 +273,9 @@ async def update_resource(
         return await get_resource(sql_model, resource_dict["fides_key"], async_session)
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use upsert_resources from safe_crud instead."
+)
 async def upsert_resources(
     sql_model: Type[T], resource_dicts: List[Dict], async_session: AsyncSession
 ) -> Tuple[int, int]:
@@ -300,6 +325,9 @@ async def upsert_resources(
                 raise sa_error
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use delete_resource from safe_crud instead."
+)
 async def delete_resource(
     sql_model: Type[T], fides_key: str, async_session: AsyncSession
 ) -> T:

fides/api/db/database.py

@@ -159,5 +159,5 @@ def check_missing_migrations(database_url: str) -> None:
     result = command.autogen.compare_metadata(migration_context, Base.metadata)  # type: ignore[attr-defined]
 
     if result:
-        raise SystemExit("Migrations needs to be generated!")
+        raise SystemExit("Migration needs to be generated!")
     print("No migrations need to be generated.")