ethyca-fides 2.68.1b1__py2.py3-none-any.whl → 2.68.1b2__py2.py3-none-any.whl

fides/api/models/taxonomy.py

@@ -0,0 +1,275 @@
+# pylint: disable=protected-access
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, List, Optional, Type
+
+from sqlalchemy import (
+    BOOLEAN,
+    Column,
+    ForeignKey,
+    ForeignKeyConstraint,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import RelationshipProperty, Session, relationship
+
+from fides.api.common_exceptions import ValidationError
+from fides.api.db.base_class import Base
+from fides.api.models.sql_models import FidesBase  # type: ignore[attr-defined]
+
+LEGACY_TAXONOMIES = {"data_categories", "data_uses", "data_subjects"}
+
+
+class TargetType(str, Enum):
+    """Enumeration of target types that taxonomies can apply to."""
+
+    SYSTEM = "system"
+    PRIVACY_DECLARATION = "privacy_declaration"
+    TAXONOMY = "taxonomy"  # For taxonomy-to-taxonomy relationships
+
+
+class Taxonomy(Base, FidesBase):
+    """The SQL model for taxonomy resources.
+
+    This is a generic taxonomy model that can be used to create any taxonomy.
+    For now we seed the database with the legacy taxonomies (data_category, data use, data subject)
+    so that these legacy taxonomy types can be used for allowed usage relationships.
+    """
+
+    # Overriding the id definition from Base so we don't treat this as the primary key
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    # The fides_key is inherited from FidesBase and acts as the primary key
+
+    # This is private to encourage the use of applies_to (see comment below)
+    _allowed_usages: RelationshipProperty[List[TaxonomyAllowedUsage]] = relationship(
+        "TaxonomyAllowedUsage",
+        back_populates="source_taxonomy",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+
+    # Association proxy to simplify access to target_type values
+    # This allows getting a list of strings while the actual storage
+    # is handled through the TaxonomyAllowedUsage model
+    # Updates should be done through the create/update methods
+    applies_to: List[str] = association_proxy(
+        "_allowed_usages",
+        "target_type",
+        # Allow setting via strings, the relationship backref will set FK
+        creator=lambda target_type: TaxonomyAllowedUsage(target_type=target_type),
+    )
+
+    @classmethod
+    def create(
+        cls: Type["Taxonomy"],
+        db: Session,
+        *,
+        data: Dict[str, Any],
+        check_name: bool = True,
+    ) -> "Taxonomy":
+        """Create a new Taxonomy with proper handling of applies_to."""
+        # Disallow creating taxonomies that represent legacy types
+        fides_key = data.get("fides_key")
+        if fides_key in LEGACY_TAXONOMIES:
+            raise ValidationError(
+                f"Cannot create taxonomy '{fides_key}'. This is a taxonomy managed by the system."
+            )
+        applies_to = data.pop("applies_to", [])
+
+        # Create the taxonomy
+        taxonomy: Taxonomy = super().create(db=db, data=data, check_name=check_name)
+
+        # Reconcile allowed usages if applies_to was provided
+        if applies_to:
+            taxonomy._reconcile_allowed_usages(db, applies_to)
+
+        return cls.persist_obj(db, taxonomy)
+
+    def update(self, db: Session, *, data: Dict[str, Any]) -> "Taxonomy":
+        """Update a Taxonomy with proper handling of applies_to."""
+        applies_to = data.pop("applies_to", None)
+
+        # Update the base fields
+        super().update(db=db, data=data)
+
+        # If applies_to was provided, reconcile allowed usages
+        if applies_to is not None:
+            self._reconcile_allowed_usages(db, applies_to)
+
+        return self
+
+    def save(self, db: Session) -> "Taxonomy":
+        """Override save to reconcile any direct `applies_to` edits before persisting.
+
+        This allows callers to mutate `applies_to` via the association proxy and then call save.
+        """
+        # Ensure no duplicate target types and reconcile relationship objects to the current values
+        self._reconcile_allowed_usages(db, list(self.applies_to))
+        return super().save(db)  # type: ignore[return-value]
+
+    def _reconcile_allowed_usages(self, db: Session, applies_to: List[str]) -> None:
+        """Ensure `_allowed_usages` matches the provided `applies_to` list.
+
+        - Deletes usages not in the provided list
+        - Creates usages missing from the relationship
+        - Deduplicates by target_type
+        """
+        existing_usages = {usage.target_type: usage for usage in self._allowed_usages}
+        desired_types = set(applies_to)
+
+        # Delete usages that should no longer exist
+        for target_type in set(existing_usages.keys()) - desired_types:
+            # Remove from relationship; delete-orphan cascade will handle DB delete
+            self._allowed_usages.remove(existing_usages[target_type])
+
+        # Add missing usages
+        for target_type in desired_types - set(existing_usages.keys()):
+            self._allowed_usages.append(TaxonomyAllowedUsage(target_type=target_type))
+
+        # Deduplicate any accidental duplicates in-memory
+        seen: set[str] = set()
+        for usage in list(self._allowed_usages):
+            if usage.target_type in seen:
+                # Remove duplicate from relationship; delete-orphan will handle DB
+                self._allowed_usages.remove(usage)
+            else:
+                seen.add(usage.target_type)
+
+
+class TaxonomyAllowedUsage(Base):
+    """
+    The SQL model for taxonomy allowed usage.
+    Defines what types of targets a taxonomy can be applied to.
+
+    target_type can be either:
+    - A generic type: "system", "privacy_declaration", "taxonomy"
+    - A taxonomy key: "data_categories", "data_uses", etc.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "taxonomy_allowed_usage"
+
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    source_taxonomy: RelationshipProperty[Taxonomy] = relationship(
+        "Taxonomy",
+        back_populates="_allowed_usages",
+    )
+
+    source_taxonomy_key: Column[str] = Column(
+        String,
+        ForeignKey("taxonomy.fides_key", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    target_type: Column[str] = Column(
+        String, primary_key=True
+    )  # Can be "system", "dataset", OR a taxonomy key like "data_categories"
+
+
+class TaxonomyElement(Base, FidesBase):
+    """
+    The SQL model for taxonomy elements.
+
+    This is a generic taxonomy element model that can be used to create any taxonomy element.
+
+    As of now the legacy taxonomy elements still exist in their own tables (ctl_data_categories, ctl_data_uses, ctl_data_subjects),
+    but we can migrate them to this model in the future if needed.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "taxonomy_element"
+
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    # Which taxonomy this element belongs to
+    taxonomy_type = Column(
+        String,
+        ForeignKey("taxonomy.fides_key", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    parent_key = Column(
+        Text, ForeignKey("taxonomy_element.fides_key", ondelete="RESTRICT"), index=True
+    )
+    active = Column(BOOLEAN, default=True, nullable=False, index=True)
+
+    children: RelationshipProperty[List[TaxonomyElement]] = relationship(
+        "TaxonomyElement",
+        back_populates="parent",
+        cascade="save-update, merge, refresh-expire",  # intentionally do not cascade deletes
+        passive_deletes="all",
+    )
+
+    parent: RelationshipProperty[Optional[TaxonomyElement]] = relationship(
+        "TaxonomyElement",
+        back_populates="children",
+        remote_side="TaxonomyElement.fides_key",
+    )
+
+
+class TaxonomyUsage(Base):
+    """
+    The SQL model for taxonomy usage.
+    Tracks the application of taxonomy elements to other taxonomy elements.
+
+    Example: Applying a "high" tag (from sensitivity taxonomy) to "user.contact.email" (from data_categories taxonomy).
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "taxonomy_usage"
+
+    # The taxonomy element being applied (e.g., risk)
+    source_element_key = Column(String, nullable=False, index=True)
+
+    # The taxonomy element it's being applied to (e.g., a data category)
+    target_element_key = Column(String, nullable=False, index=True)
+
+    # Denormalized taxonomy types for validation and performance
+    source_taxonomy = Column(String, nullable=False, index=True)
+    target_taxonomy = Column(String, nullable=False, index=True)
+
+    __table_args__ = (
+        # Validate that this type of usage is allowed
+        ForeignKeyConstraint(
+            ["source_taxonomy", "target_taxonomy"],
+            [
+                "taxonomy_allowed_usage.source_taxonomy_key",
+                "taxonomy_allowed_usage.target_type",
+            ],
+            ondelete="RESTRICT",
+            name="fk_taxonomy_usage_allowed",
+        ),
+        # Prevent duplicate applications
+        UniqueConstraint(
+            "source_element_key",
+            "target_element_key",
+            name="uq_taxonomy_usage",
+        ),
+    )

fides/api/service/deps.py

@@ -8,6 +8,7 @@ from fides.service.dataset.dataset_config_service import DatasetConfigService
 from fides.service.dataset.dataset_service import DatasetService
 from fides.service.messaging.messaging_service import MessagingService
 from fides.service.privacy_request.privacy_request_service import PrivacyRequestService
+from fides.service.taxonomy.taxonomy_service import TaxonomyService
 from fides.service.user.user_service import UserService
 
 
@@ -41,3 +42,7 @@ def get_user_service(
     config_proxy: ConfigProxy = Depends(get_config_proxy),
 ) -> UserService:
     return UserService(db, config, config_proxy)
+
+
+def get_taxonomy_service(db: Session = Depends(get_db)) -> TaxonomyService:
+    return TaxonomyService(db)

fides/api/service/privacy_request/request_service.py

@@ -191,7 +191,11 @@ def poll_for_exited_privacy_request_tasks(self: DatabaseTask) -> Set[str]:
             db.query(PrivacyRequest)
             .filter(
                 PrivacyRequest.status.in_(
-                    [PrivacyRequestStatus.in_processing, PrivacyRequestStatus.approved]
+                    [
+                        PrivacyRequestStatus.in_processing,
+                        PrivacyRequestStatus.approved,
+                        PrivacyRequestStatus.requires_input,
+                    ]
                 )
             )
             # Only look at Privacy Requests that haven't been deleted
@@ -546,6 +550,7 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                         [
                             PrivacyRequestStatus.in_processing,
                             PrivacyRequestStatus.approved,
+                            PrivacyRequestStatus.requires_input,
                         ]
                     )
                 )

fides/api/task/manual/manual_task_graph_task.py

@@ -63,6 +63,17 @@ class ManualTaskGraphTask(GraphTask):
         Calls _run_request with ACCESS configs.
         Returns data if submitted, raise AwaitingAsyncTaskCallback if not
         """
+        if self.resources.request.policy.get_action_type() == ActionType.erasure:
+            # We're in an erasure privacy request's access phase - complete access task immediately
+            # since access is just for data collection to support erasure, not for user data access
+            self.update_status(
+                "Access task completed immediately for erasure privacy request (data collection only)",
+                [],
+                ActionType.access,
+                ExecutionLogStatus.complete,
+            )
+            return []
+
         result = self._run_request(
             ManualTaskConfigurationType.access_privacy_request,
             ActionType.access,

fides/api/util/connection_type.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Dict, Set
+from typing import Any, Dict, Optional, Set
 
 from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError
 from fides.api.models.connectionconfig import ConnectionType
@@ -141,7 +141,9 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]
     Note that this does not return actual secrets, instead we return the *types* of
     secret fields needed to authenticate.
     """
-    connection_system_types: list[ConnectionSystemTypeMap] = get_connection_types()
+    connection_system_types: list[ConnectionSystemTypeMap] = get_connection_types(
+        include_test_connections=True
+    )
     if not any(item.identifier == connection_type for item in connection_system_types):
         raise NoSuchConnectionTypeSecretSchemaError(
             f"No connection type found with name '{connection_type}'."
@@ -189,15 +191,15 @@ def get_connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]
     return schema
 
 
-def get_connection_types(
-    search: str | None = None,
-    system_type: SystemType | None = None,
+def _is_match(elem: str, search: Optional[str] = None) -> bool:
+    """If a search query param was included, is it a substring of an available connector type?"""
+    return search.lower() in elem.lower() if search else True
+
+
+def get_saas_connection_types(
     action_types: Set[ActionType] = SUPPORTED_ACTION_TYPES,
+    search: Optional[str] = None,
 ) -> list[ConnectionSystemTypeMap]:
-    def is_match(elem: str) -> bool:
-        """If a search query param was included, is it a substring of an available connector type?"""
-        return search.lower() in elem.lower() if search else True
-
     def saas_request_type_filter(connection_type: str) -> bool:
         """
         If any of the request type filters are set to true,
@@ -216,6 +218,45 @@ def get_connection_types(
             action_type in template.supported_actions for action_type in action_types
         )
 
+    saas_connection_types: list[ConnectionSystemTypeMap] = []
+    saas_types: list[str] = sorted(
+        [
+            saas_type
+            for saas_type in ConnectorRegistry.connector_types()
+            if _is_match(saas_type, search) and saas_request_type_filter(saas_type)
+        ]
+    )
+
+    for item in saas_types:
+        connector_template = ConnectorRegistry.get_connector_template(item)
+        if connector_template is not None:
+            saas_connection_types.append(
+                ConnectionSystemTypeMap(
+                    identifier=item,
+                    type=SystemType.saas,
+                    human_readable=connector_template.human_readable,
+                    encoded_icon=connector_template.icon,
+                    authorization_required=connector_template.authorization_required,
+                    user_guide=connector_template.user_guide,
+                    supported_actions=connector_template.supported_actions,
+                )
+            )
+
+    return saas_connection_types
+
+
+# FIXME: this function needs a refactor
+def get_connection_types(
+    search: str | None = None,
+    system_type: SystemType | None = None,
+    action_types: Set[ActionType] = SUPPORTED_ACTION_TYPES,
+    include_test_connections: bool = False,
+) -> list[ConnectionSystemTypeMap]:
+    """
+    Returns a list of ConnectionSystemTypeMap objects that match the given search and system type.
+
+    If include_test_connections is True, test connections like test_website will be included in the response.
+    """
     connection_system_types: list[ConnectionSystemTypeMap] = []
     if (system_type == SystemType.database or system_type is None) and (
         ActionType.access in action_types or ActionType.erasure in action_types
@@ -238,7 +279,7 @@ def get_connection_types(
                     ConnectionType.sovrn,
                     ConnectionType.test_website,
                 ]
-                and is_match(conn_type.value)
+                and _is_match(conn_type.value, search)
             ],
             key=lambda x: x.value,
         )
@@ -254,28 +295,10 @@ def get_connection_types(
             ]
         )
     if system_type == SystemType.saas or system_type is None:
-        saas_types: list[str] = sorted(
-            [
-                saas_type
-                for saas_type in ConnectorRegistry.connector_types()
-                if is_match(saas_type) and saas_request_type_filter(saas_type)
-            ]
+        saas_connection_types = get_saas_connection_types(
+            action_types=action_types, search=search
         )
-
-        for item in saas_types:
-            connector_template = ConnectorRegistry.get_connector_template(item)
-            if connector_template is not None:
-                connection_system_types.append(
-                    ConnectionSystemTypeMap(
-                        identifier=item,
-                        type=SystemType.saas,
-                        human_readable=connector_template.human_readable,
-                        encoded_icon=connector_template.icon,
-                        authorization_required=connector_template.authorization_required,
-                        user_guide=connector_template.user_guide,
-                        supported_actions=connector_template.supported_actions,
-                    )
-                )
+        connection_system_types.extend(saas_connection_types)
 
     if (system_type == SystemType.manual or system_type is None) and (
         ActionType.access in action_types or ActionType.erasure in action_types
@@ -285,7 +308,7 @@ def get_connection_types(
                 manual_type.value
                 for manual_type in ConnectionType
                 if manual_type == ConnectionType.manual_webhook
-                and is_match(manual_type.value)
+                and _is_match(manual_type.value, search)
             ]
         )
         connection_system_types.extend(
@@ -307,7 +330,7 @@ def get_connection_types(
                 for email_type in ConnectionType
                 if email_type
                 in ERASURE_EMAIL_CONNECTOR_TYPES + CONSENT_EMAIL_CONNECTOR_TYPES
-                and is_match(email_type.value)
+                and _is_match(email_type.value, search)
                 and (  # include consent or erasure connectors if requested, respectively
                     (
                         ActionType.consent in action_types
@@ -339,4 +362,16 @@ def get_connection_types(
             ]
         )
 
+    if include_test_connections and (
+        system_type == SystemType.website or system_type is None
+    ):
+        connection_system_types.append(
+            ConnectionSystemTypeMap(
+                identifier=ConnectionType.test_website.value,
+                type=SystemType.website,
+                human_readable=ConnectionType.test_website.human_readable,
+                supported_actions=[],
+            )
+        )
+
     return connection_system_types

fides/data/sample_project/docker-compose.yml

@@ -87,7 +87,7 @@ services:
 
   fides-db:
     container_name: fides-db
-    image: postgres:12
+    image: postgres:16
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 15s
@@ -104,14 +104,14 @@ services:
 
   redis:
     container_name: fides-redis
-    image: redis:6.2-alpine
+    image: redis:8.0-alpine
     command: redis-server --requirepass redispassword
     ports:
       - "7379:6379"
 
   postgres-test:
     container_name: fides-postgres-example-db
-    image: postgres:12
+    image: postgres:16
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 15s

fides/service/taxonomy/handlers/__init__.py

@@ -0,0 +1,11 @@
+"""
+Taxonomy handlers package.
+"""
+
+from .base import TaxonomyHandler
+from .legacy_handler import LegacyTaxonomyHandler
+
+__all__ = [
+    "TaxonomyHandler",
+    "LegacyTaxonomyHandler",
+]

fides/service/taxonomy/handlers/base.py

@@ -0,0 +1,42 @@
+"""
+Base taxonomy handler and core validation functions.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Type
+
+from sqlalchemy.orm import Session
+
+from fides.api.db.base_class import FidesBase
+
+
+class TaxonomyHandler(ABC):
+    """Abstract handler for taxonomy CRUD operations."""
+
+    def __init__(self, db: Session, taxonomy_type: str):
+        self.db = db
+        self.taxonomy_type = taxonomy_type
+
+    @abstractmethod
+    def get_model(self) -> Type[FidesBase]:
+        pass
+
+    @abstractmethod
+    def get_elements(self, active_only: bool, parent_key: Optional[str]) -> List[Any]:
+        pass
+
+    @abstractmethod
+    def get_element(self, fides_key: str) -> Optional[Any]:
+        pass
+
+    @abstractmethod
+    def create_element(self, element_data: Dict) -> Any:
+        pass
+
+    @abstractmethod
+    def update_element(self, fides_key: str, element_data: Dict) -> Optional[Any]:
+        pass
+
+    @abstractmethod
+    def delete_element(self, fides_key: str) -> None:
+        pass

fides/service/taxonomy/handlers/legacy_handler.py

@@ -0,0 +1,95 @@
+"""
+Legacy taxonomy handler for DataCategory, DataUse, and DataSubject models.
+"""
+
+from typing import Dict, List, Optional, Type, Union
+
+from sqlalchemy.orm import Query, Session
+
+from fides.api.models.sql_models import (  # type:ignore[attr-defined]
+    DataCategory,
+    DataSubject,
+    DataUse,
+)
+
+from .base import TaxonomyHandler
+
+
+class LegacyTaxonomyHandler(TaxonomyHandler):
+    """Handler for legacy taxonomy models (DataCategory, DataUse, DataSubject)."""
+
+    def __init__(self, db: Session, taxonomy_type: str) -> None:
+        super().__init__(db, taxonomy_type)
+        self.models = {
+            "data_categories": DataCategory,
+            "data_uses": DataUse,
+            "data_subjects": DataSubject,
+        }
+        if taxonomy_type not in self.models:
+            raise ValueError(
+                f"Taxonomy type '{taxonomy_type}' not supported. Supported types: {list(self.models.keys())}"
+            )
+
+    def get_elements(
+        self, active_only: bool, parent_key: Optional[str]
+    ) -> List[Union[DataCategory, DataUse, DataSubject]]:
+        model_class = self.get_model()
+        query = self._build_query(model_class, active_only, parent_key)
+        return query.all()
+
+    def get_element(
+        self, fides_key: str
+    ) -> Optional[Union[DataCategory, DataUse, DataSubject]]:
+        model_class = self.get_model()
+        element = (
+            self.db.query(model_class)
+            .filter(model_class.fides_key == fides_key)
+            .first()
+        )
+        return element
+
+    def create_element(
+        self, element_data: Dict
+    ) -> Union[DataCategory, DataUse, DataSubject]:
+        model_class = self.get_model()
+        element = model_class.create(self.db, data=element_data)
+        return element
+
+    def update_element(
+        self, fides_key: str, element_data: Dict
+    ) -> Optional[Union[DataCategory, DataUse, DataSubject]]:
+        model_class = self.get_model()
+        element = (
+            self.db.query(model_class)
+            .filter(model_class.fides_key == fides_key)
+            .first()
+        )
+
+        if not element:
+            return None
+
+        return element.update(db=self.db, data=element_data)
+
+    def delete_element(self, fides_key: str) -> None:
+        model_class = self.get_model()
+        element = (
+            self.db.query(model_class)
+            .filter(model_class.fides_key == fides_key)
+            .first()
+        )
+
+        if element:
+            self.db.delete(element)
+
+    def get_model(self) -> Type[Union[DataCategory, DataUse, DataSubject]]:
+        return self.models[self.taxonomy_type]
+
+    def _build_query(
+        self, model_class: Type, active_only: bool, parent_key: Optional[str]
+    ) -> Query:
+        query = self.db.query(model_class)
+        if active_only:
+            query = query.filter(model_class.active.is_(True))
+        if parent_key and hasattr(model_class, "parent_key"):
+            query = query.filter(model_class.parent_key == parent_key)
+        return query