ethyca-fides 2.68.1b1__py2.py3-none-any.whl → 2.68.1b2__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/3baf42d251a6_add_generic_taxonomy_models.py

@@ -0,0 +1,239 @@
+"""add generic taxonomy models
+
+Revision ID: 3baf42d251a6
+Revises: 2f3c1a2d6b10
+Create Date: 2025-08-05 21:49:39.619569
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "3baf42d251a6"
+down_revision = "2f3c1a2d6b10"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Create taxonomy table
+    op.create_table(
+        "taxonomy",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("fides_key", sa.String(), nullable=False),
+        sa.Column("organization_fides_key", sa.Text(), nullable=True),
+        sa.Column("tags", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.Column("name", sa.Text(), nullable=True),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.PrimaryKeyConstraint("fides_key"),
+        sa.UniqueConstraint("id", name="uq_taxonomy_id"),
+    )
+    op.create_index(
+        op.f("ix_taxonomy_fides_key"), "taxonomy", ["fides_key"], unique=True
+    )
+
+    # Create taxonomy_allowed_usage table
+    op.create_table(
+        "taxonomy_allowed_usage",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("source_taxonomy_key", sa.String(), nullable=False),
+        sa.Column(
+            "target_type", sa.String(), nullable=False
+        ),  # Can be "system" OR "data_categories"
+        sa.ForeignKeyConstraint(
+            ["source_taxonomy_key"], ["taxonomy.fides_key"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("source_taxonomy_key", "target_type"),
+        sa.UniqueConstraint("id", name="uq_taxonomy_allowed_usage_id"),
+    )
+
+    # Create taxonomy_element table
+    op.create_table(
+        "taxonomy_element",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("fides_key", sa.String(), nullable=False),
+        sa.Column("organization_fides_key", sa.Text(), nullable=True),
+        sa.Column("tags", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.Column("name", sa.Text(), nullable=True),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("taxonomy_type", sa.String(), nullable=False),
+        sa.Column("parent_key", sa.Text(), nullable=True),
+        sa.Column("active", sa.BOOLEAN(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["parent_key"], ["taxonomy_element.fides_key"], ondelete="RESTRICT"
+        ),
+        sa.ForeignKeyConstraint(
+            ["taxonomy_type"], ["taxonomy.fides_key"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("fides_key"),
+        sa.UniqueConstraint("id", name="uq_taxonomy_element_id"),
+    )
+    op.create_index(
+        op.f("ix_taxonomy_element_fides_key"),
+        "taxonomy_element",
+        ["fides_key"],
+        unique=True,
+    )
+    op.create_index(
+        op.f("ix_taxonomy_element_active"), "taxonomy_element", ["active"], unique=False
+    )
+    op.create_index(
+        op.f("ix_taxonomy_element_parent_key"),
+        "taxonomy_element",
+        ["parent_key"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_taxonomy_element_taxonomy_type"),
+        "taxonomy_element",
+        ["taxonomy_type"],
+        unique=False,
+    )
+
+    # Create taxonomy_usage table
+    op.create_table(
+        "taxonomy_usage",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("source_element_key", sa.String(), nullable=False),
+        sa.Column("target_element_key", sa.String(), nullable=False),
+        sa.Column("source_taxonomy", sa.String(), nullable=False),
+        sa.Column("target_taxonomy", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["source_taxonomy", "target_taxonomy"],
+            [
+                "taxonomy_allowed_usage.source_taxonomy_key",
+                "taxonomy_allowed_usage.target_type",
+            ],
+            name="fk_taxonomy_usage_allowed",
+            ondelete="RESTRICT",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "source_element_key",
+            "target_element_key",
+            name="uq_taxonomy_usage",
+        ),
+    )
+    op.create_index(
+        op.f("ix_taxonomy_usage_source_element_key"),
+        "taxonomy_usage",
+        ["source_element_key"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_taxonomy_usage_id"), "taxonomy_usage", ["id"], unique=False
+    )
+    op.create_index(
+        op.f("ix_taxonomy_usage_source_taxonomy"),
+        "taxonomy_usage",
+        ["source_taxonomy"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_taxonomy_usage_target_element_key"),
+        "taxonomy_usage",
+        ["target_element_key"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_taxonomy_usage_target_taxonomy"),
+        "taxonomy_usage",
+        ["target_taxonomy"],
+        unique=False,
+    )
+
+    # Seed legacy taxonomies that are managed by the system
+    op.execute(
+        """
+        INSERT INTO taxonomy (id, created_at, updated_at, fides_key, organization_fides_key, tags, name, description)
+        VALUES
+          ('data_category', now(), now(), 'data_category', NULL, NULL, 'Data categories', 'Taxonomy for data categories'),
+          ('data_use', now(), now(), 'data_use', NULL, NULL, 'Data uses', 'Taxonomy for data uses'),
+          ('data_subject', now(), now(), 'data_subject', NULL, NULL, 'Data subjects', 'Taxonomy for data subjects'),
+          ('system_group', now(), now(), 'system_group', NULL, NULL, 'System groups', 'Taxonomy for system groups')
+        """
+    )
+
+
+def downgrade():
+    # Drop taxonomy_usage
+    op.drop_index(
+        op.f("ix_taxonomy_usage_source_element_key"), table_name="taxonomy_usage"
+    )
+    op.drop_index(
+        op.f("ix_taxonomy_usage_target_taxonomy"), table_name="taxonomy_usage"
+    )
+    op.drop_index(
+        op.f("ix_taxonomy_usage_target_element_key"), table_name="taxonomy_usage"
+    )
+    op.drop_index(
+        op.f("ix_taxonomy_usage_source_taxonomy"), table_name="taxonomy_usage"
+    )
+    op.drop_index(op.f("ix_taxonomy_usage_id"), table_name="taxonomy_usage")
+    op.drop_table("taxonomy_usage")
+
+    # Drop taxonomy_element
+    op.drop_index(
+        op.f("ix_taxonomy_element_taxonomy_type"), table_name="taxonomy_element"
+    )
+    op.drop_index(op.f("ix_taxonomy_element_parent_key"), table_name="taxonomy_element")
+    op.drop_index(op.f("ix_taxonomy_element_active"), table_name="taxonomy_element")
+    op.drop_index(op.f("ix_taxonomy_element_fides_key"), table_name="taxonomy_element")
+    op.drop_table("taxonomy_element")
+
+    # Drop taxonomy_allowed_usage
+    op.drop_table("taxonomy_allowed_usage")
+
+    # Drop taxonomy
+    op.drop_index(op.f("ix_taxonomy_fides_key"), table_name="taxonomy")
+    op.drop_table("taxonomy")

fides/api/api/v1/endpoints/generic_overrides.py

@@ -1,5 +1,5 @@
 from functools import partial
-from typing import Dict, List, Optional, Type, Union
+from typing import Dict, List, Optional, Union
 
 from fastapi import Depends, HTTPException, Query, Response, Security
 from fastapi.concurrency import run_in_threadpool
@@ -11,17 +11,16 @@ from fideslang.models import Dataset as FideslangDataset
 from pydantic import ValidationError as PydanticValidationError
 from sqlalchemy import not_, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import Session, load_only
+from sqlalchemy.orm import load_only
 from starlette import status
 from starlette.status import (
     HTTP_200_OK,
+    HTTP_400_BAD_REQUEST,
     HTTP_404_NOT_FOUND,
     HTTP_422_UNPROCESSABLE_ENTITY,
 )
 
-from fides.api.api.deps import get_db
 from fides.api.common_exceptions import KeyOrNameAlreadyExists, ValidationError
-from fides.api.db.base_class import get_key_from_data
 from fides.api.db.crud import list_resource_query
 from fides.api.db.ctl_session import get_async_db
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
@@ -37,9 +36,8 @@ from fides.api.schemas.taxonomy_extensions import (
     DataUse,
     DataUseCreateOrUpdate,
 )
-from fides.api.service.deps import get_dataset_service
+from fides.api.service.deps import get_dataset_service, get_taxonomy_service
 from fides.api.util.api_router import APIRouter
-from fides.api.util.errors import FidesError, ForbiddenIsDefaultTaxonomyError
 from fides.api.util.filter_utils import apply_filters_to_query
 from fides.common.api.scope_registry import (
     CTL_DATASET_CREATE,
@@ -57,18 +55,12 @@ from fides.service.dataset.dataset_service import (
     DatasetNotFoundException,
     DatasetService,
 )
+from fides.service.taxonomy.taxonomy_service import TaxonomyService
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     Dataset as CtlDataset,
 )
 
-from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
-    DataCategory as DataCategoryDbModel,
-    DataSubject as DataSubjectDbModel,
-    DataUse as DataUseDbModel,
-    ModelWithDefaultField,
-)
-
 # We create routers to override specific methods in those defined in generic.py
 # when we need more custom implementations for only some of the methods in a router.
 
@@ -354,135 +346,6 @@ def clean_datasets(
     )
 
 
-def activate_taxonomy_parents(
-    resource: Union[DataCategoryDbModel, DataUseDbModel, DataSubjectDbModel],
-    db: Session,
-) -> None:
-    """
-    Activates parents to match newly-active taxonomy node.
-    """
-    parent = resource.parent
-    if parent:
-        parent.active = True
-        db.commit()
-
-        activate_taxonomy_parents(parent, db)
-
-
-def deactivate_taxonomy_node_and_descendants(
-    resource: Union[DataCategoryDbModel, DataUseDbModel, DataSubjectDbModel],
-    db: Session,
-) -> None:
-    """
-    Recursively de-activates all descendants of a given taxonomy node.
-    """
-    resource.active = False
-    db.commit()
-    children = resource.children
-
-    for child in children:
-        # Deactivate current child
-        child.active = False
-        db.commit()
-
-        # Recursively deactivate all descendants of this child
-        deactivate_taxonomy_node_and_descendants(child, db)
-
-
-def validate_and_create_taxonomy(
-    db: Session,
-    model: Union[
-        Type[DataCategoryDbModel], Type[DataUseDbModel], Type[DataSubjectDbModel]
-    ],
-    validation_schema: type,
-    data: Union[
-        DataCategoryCreateOrUpdate, DataUseCreateOrUpdate, DataSubjectCreateOrUpdate
-    ],
-) -> Dict:
-    """
-    Validate and create a taxonomy element.
-    """
-    if not data.fides_key:
-        raise FidesError(f"Fides key is required to create a {model.__name__} resource")
-    if isinstance(data, ModelWithDefaultField) and data.is_default:
-        raise ForbiddenIsDefaultTaxonomyError(
-            model.__name__, data.fides_key, action="create"
-        )
-    validated_taxonomy = validation_schema(**data.model_dump(mode="json"))
-    return model.create(db=db, data=validated_taxonomy.model_dump(mode="json"))
-
-
-def validate_and_update_taxonomy(
-    db: Session,
-    resource: Union[DataCategoryDbModel, DataUseDbModel, DataSubjectDbModel],
-    validation_schema: type,
-    data: Union[
-        DataCategoryCreateOrUpdate, DataUseCreateOrUpdate, DataSubjectCreateOrUpdate
-    ],
-) -> Dict:
-    """
-    Validate and update a taxonomy element.
-    """
-    if (
-        isinstance(data, ModelWithDefaultField)
-        and data.is_default != resource.is_default
-    ):
-        raise ForbiddenIsDefaultTaxonomyError(
-            "resource", data.fides_key, action="modify"
-        )
-
-    # If active field is being updated, cascade change either up or down
-    if hasattr(data, "active"):
-        if data.active:
-            activate_taxonomy_parents(resource, db)
-        else:
-            # Cascade down - deactivate current node and children to match newly-deactivated taxonomy item
-            deactivate_taxonomy_node_and_descendants(resource, db)
-
-    validated_taxonomy = validation_schema(**data.model_dump(mode="json"))
-    return resource.update(db=db, data=validated_taxonomy.model_dump(mode="json"))
-
-
-def create_or_update_taxonomy(
-    db: Session,
-    data: Union[
-        DataCategoryCreateOrUpdate, DataUseCreateOrUpdate, DataSubjectCreateOrUpdate
-    ],
-    model: Union[
-        Type[DataCategoryDbModel], Type[DataUseDbModel], Type[DataSubjectDbModel]
-    ],
-    validation_schema: type,
-) -> Dict:
-    """
-    Create or update a taxonomy element.
-    If the element is deactivated, it will be updated and re-activated, along with its parents.
-    """
-    if data.fides_key is None:
-        disabled_resource_with_name = (
-            db.query(model)
-            .filter(
-                model.active.is_(False),
-                model.name == data.name,
-            )
-            .first()
-        )
-        data.fides_key = get_key_from_data(
-            {"key": data.fides_key, "name": data.name}, validation_schema.__name__
-        )
-        if data.parent_key if hasattr(data, "parent_key") else None:
-            # Updates fides_key if it is not the root level taxonomy node
-            data.fides_key = f"{data.parent_key}.{data.fides_key}"  # type: ignore[union-attr]
-        if disabled_resource_with_name:
-            data.active = True
-            activate_taxonomy_parents(disabled_resource_with_name, db)
-            return validate_and_update_taxonomy(
-                db, disabled_resource_with_name, validation_schema, data
-            )
-        return validate_and_create_taxonomy(db, model, validation_schema, data)
-
-    return validate_and_create_taxonomy(db, model, validation_schema, data)
-
-
 @data_use_router.post(
     "/data_use",
     dependencies=[Security(verify_oauth_client, scopes=[DATA_USE_CREATE])],
@@ -492,19 +355,21 @@ def create_or_update_taxonomy(
 )
 async def create_data_use(
     data_use: DataUseCreateOrUpdate,
-    db: Session = Depends(get_db),
+    taxonomy_service: TaxonomyService = Depends(get_taxonomy_service),
 ) -> Dict:
     """
     Create a data use. Updates existing data use if data use with name already exists and is disabled.
     """
     try:
-        return create_or_update_taxonomy(db, data_use, DataUseDbModel, DataUse)
+        return taxonomy_service.create_or_update_element(
+            "data_uses", data_use.model_dump()
+        )
     except KeyOrNameAlreadyExists:
         raise HTTPException(
             status_code=HTTP_422_UNPROCESSABLE_ENTITY,
             detail=f"Data use with key {data_use.fides_key} or name {data_use.name} already exists.",
         )
-    except PydanticValidationError as e:
+    except (ValidationError, PydanticValidationError) as e:
         raise HTTPException(
             status_code=HTTP_422_UNPROCESSABLE_ENTITY,
             detail=str(e),
@@ -520,21 +385,26 @@ async def create_data_use(
 )
 async def create_data_category(
     data_category: DataCategoryCreateOrUpdate,
-    db: Session = Depends(get_db),
+    taxonomy_service: TaxonomyService = Depends(get_taxonomy_service),
 ) -> Dict:
     """
     Create a data category
     """
 
     try:
-        return create_or_update_taxonomy(
-            db, data_category, DataCategoryDbModel, DataCategory
+        return taxonomy_service.create_or_update_element(
+            "data_categories", data_category.model_dump()
         )
     except KeyOrNameAlreadyExists:
         raise HTTPException(
             status_code=HTTP_422_UNPROCESSABLE_ENTITY,
             detail=f"Data category with key {data_category.fides_key} or name {data_category.name} already exists.",
         )
+    except (ValidationError, PydanticValidationError) as e:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
 
 
 @data_subject_router.post(
@@ -546,21 +416,26 @@ async def create_data_category(
 )
 async def create_data_subject(
     data_subject: DataSubjectCreateOrUpdate,
-    db: Session = Depends(get_db),
+    taxonomy_service: TaxonomyService = Depends(get_taxonomy_service),
 ) -> Dict:
     """
     Create a data subject
     """
 
     try:
-        return create_or_update_taxonomy(
-            db, data_subject, DataSubjectDbModel, DataSubject
+        return taxonomy_service.create_or_update_element(
+            "data_subjects", data_subject.model_dump()
         )
     except KeyOrNameAlreadyExists:
         raise HTTPException(
             status_code=HTTP_422_UNPROCESSABLE_ENTITY,
             detail=f"Data subject with key {data_subject.fides_key} or name {data_subject.name} already exists.",
         )
+    except (ValidationError, PydanticValidationError) as e:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
 
 
 @data_use_router.put(
@@ -572,19 +447,31 @@ async def create_data_subject(
 )
 async def update_data_use(
     data_use: DataUseCreateOrUpdate,
-    db: Session = Depends(get_db),
+    taxonomy_service: TaxonomyService = Depends(get_taxonomy_service),
 ) -> Dict:
     """
     Update a data use. Ensures updates to "active" are appropriately cascaded.
     """
-
-    resource = DataUseDbModel.get_by(db, field="fides_key", value=data_use.fides_key)
-    if not resource:
+    if not data_use.fides_key:
         raise HTTPException(
-            status_code=HTTP_404_NOT_FOUND,
-            detail=f"Data use not found with key: {data_use.fides_key}",
+            status_code=HTTP_400_BAD_REQUEST,
+            detail="fides_key is required for update operations.",
+        )
+    try:
+        result = taxonomy_service.update_element(
+            "data_uses", data_use.fides_key, data_use.model_dump()
+        )
+        if not result:
+            raise HTTPException(
+                status_code=HTTP_404_NOT_FOUND,
+                detail=f"Data use not found with key: {data_use.fides_key}",
+            )
+        return result
+    except (ValidationError, PydanticValidationError) as e:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
         )
-    return validate_and_update_taxonomy(db, resource, DataUse, data_use)
 
 
 @data_category_router.put(
@@ -596,21 +483,31 @@ async def update_data_use(
 )
 async def update_data_category(
     data_category: DataCategoryCreateOrUpdate,
-    db: Session = Depends(get_db),
+    taxonomy_service: TaxonomyService = Depends(get_taxonomy_service),
 ) -> Dict:
     """
     Update a data category. Ensures updates to "active" are appropriately cascaded.
     """
-
-    resource = DataCategoryDbModel.get_by(
-        db, field="fides_key", value=data_category.fides_key
-    )
-    if not resource:
+    if not data_category.fides_key:
         raise HTTPException(
-            status_code=HTTP_404_NOT_FOUND,
-            detail=f"Data category not found with key: {data_category.fides_key}",
+            status_code=HTTP_400_BAD_REQUEST,
+            detail="fides_key is required for update operations.",
+        )
+    try:
+        result = taxonomy_service.update_element(
+            "data_categories", data_category.fides_key, data_category.model_dump()
+        )
+        if not result:
+            raise HTTPException(
+                status_code=HTTP_404_NOT_FOUND,
+                detail=f"Data category not found with key: {data_category.fides_key}",
+            )
+        return result
+    except (ValidationError, PydanticValidationError) as e:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
         )
-    return validate_and_update_taxonomy(db, resource, DataCategory, data_category)
 
 
 GENERIC_OVERRIDES_ROUTER = APIRouter()

fides/api/db/base.py

@@ -76,6 +76,12 @@ from fides.api.models.storage import StorageConfig
 from fides.api.models.system_compass_sync import SystemCompassSync
 from fides.api.models.system_history import SystemHistory
 from fides.api.models.system_manager import SystemManager
+from fides.api.models.taxonomy import (
+    Taxonomy,
+    TaxonomyAllowedUsage,
+    TaxonomyElement,
+    TaxonomyUsage,
+)
 from fides.api.models.tcf_publisher_restrictions import (
     TCFConfiguration,
     TCFPublisherRestriction,