ethyca-fides 2.68.1b0__py2.py3-none-any.whl → 2.68.1b2__py2.py3-none-any.whl

fides/api/models/taxonomy.py

@@ -0,0 +1,275 @@
+# pylint: disable=protected-access
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, List, Optional, Type
+
+from sqlalchemy import (
+    BOOLEAN,
+    Column,
+    ForeignKey,
+    ForeignKeyConstraint,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import RelationshipProperty, Session, relationship
+
+from fides.api.common_exceptions import ValidationError
+from fides.api.db.base_class import Base
+from fides.api.models.sql_models import FidesBase  # type: ignore[attr-defined]
+
+LEGACY_TAXONOMIES = {"data_categories", "data_uses", "data_subjects"}
+
+
+class TargetType(str, Enum):
+    """Enumeration of target types that taxonomies can apply to."""
+
+    SYSTEM = "system"
+    PRIVACY_DECLARATION = "privacy_declaration"
+    TAXONOMY = "taxonomy"  # For taxonomy-to-taxonomy relationships
+
+
+class Taxonomy(Base, FidesBase):
+    """The SQL model for taxonomy resources.
+
+    This is a generic taxonomy model that can be used to create any taxonomy.
+    For now we seed the database with the legacy taxonomies (data_category, data use, data subject)
+    so that these legacy taxonomy types can be used for allowed usage relationships.
+    """
+
+    # Overriding the id definition from Base so we don't treat this as the primary key
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    # The fides_key is inherited from FidesBase and acts as the primary key
+
+    # This is private to encourage the use of applies_to (see comment below)
+    _allowed_usages: RelationshipProperty[List[TaxonomyAllowedUsage]] = relationship(
+        "TaxonomyAllowedUsage",
+        back_populates="source_taxonomy",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+
+    # Association proxy to simplify access to target_type values
+    # This allows getting a list of strings while the actual storage
+    # is handled through the TaxonomyAllowedUsage model
+    # Updates should be done through the create/update methods
+    applies_to: List[str] = association_proxy(
+        "_allowed_usages",
+        "target_type",
+        # Allow setting via strings, the relationship backref will set FK
+        creator=lambda target_type: TaxonomyAllowedUsage(target_type=target_type),
+    )
+
+    @classmethod
+    def create(
+        cls: Type["Taxonomy"],
+        db: Session,
+        *,
+        data: Dict[str, Any],
+        check_name: bool = True,
+    ) -> "Taxonomy":
+        """Create a new Taxonomy with proper handling of applies_to."""
+        # Disallow creating taxonomies that represent legacy types
+        fides_key = data.get("fides_key")
+        if fides_key in LEGACY_TAXONOMIES:
+            raise ValidationError(
+                f"Cannot create taxonomy '{fides_key}'. This is a taxonomy managed by the system."
+            )
+        applies_to = data.pop("applies_to", [])
+
+        # Create the taxonomy
+        taxonomy: Taxonomy = super().create(db=db, data=data, check_name=check_name)
+
+        # Reconcile allowed usages if applies_to was provided
+        if applies_to:
+            taxonomy._reconcile_allowed_usages(db, applies_to)
+
+        return cls.persist_obj(db, taxonomy)
+
+    def update(self, db: Session, *, data: Dict[str, Any]) -> "Taxonomy":
+        """Update a Taxonomy with proper handling of applies_to."""
+        applies_to = data.pop("applies_to", None)
+
+        # Update the base fields
+        super().update(db=db, data=data)
+
+        # If applies_to was provided, reconcile allowed usages
+        if applies_to is not None:
+            self._reconcile_allowed_usages(db, applies_to)
+
+        return self
+
+    def save(self, db: Session) -> "Taxonomy":
+        """Override save to reconcile any direct `applies_to` edits before persisting.
+
+        This allows callers to mutate `applies_to` via the association proxy and then call save.
+        """
+        # Ensure no duplicate target types and reconcile relationship objects to the current values
+        self._reconcile_allowed_usages(db, list(self.applies_to))
+        return super().save(db)  # type: ignore[return-value]
+
+    def _reconcile_allowed_usages(self, db: Session, applies_to: List[str]) -> None:
+        """Ensure `_allowed_usages` matches the provided `applies_to` list.
+
+        - Deletes usages not in the provided list
+        - Creates usages missing from the relationship
+        - Deduplicates by target_type
+        """
+        existing_usages = {usage.target_type: usage for usage in self._allowed_usages}
+        desired_types = set(applies_to)
+
+        # Delete usages that should no longer exist
+        for target_type in set(existing_usages.keys()) - desired_types:
+            # Remove from relationship; delete-orphan cascade will handle DB delete
+            self._allowed_usages.remove(existing_usages[target_type])
+
+        # Add missing usages
+        for target_type in desired_types - set(existing_usages.keys()):
+            self._allowed_usages.append(TaxonomyAllowedUsage(target_type=target_type))
+
+        # Deduplicate any accidental duplicates in-memory
+        seen: set[str] = set()
+        for usage in list(self._allowed_usages):
+            if usage.target_type in seen:
+                # Remove duplicate from relationship; delete-orphan will handle DB
+                self._allowed_usages.remove(usage)
+            else:
+                seen.add(usage.target_type)
+
+
+class TaxonomyAllowedUsage(Base):
+    """
+    The SQL model for taxonomy allowed usage.
+    Defines what types of targets a taxonomy can be applied to.
+
+    target_type can be either:
+    - A generic type: "system", "privacy_declaration", "taxonomy"
+    - A taxonomy key: "data_categories", "data_uses", etc.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "taxonomy_allowed_usage"
+
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    source_taxonomy: RelationshipProperty[Taxonomy] = relationship(
+        "Taxonomy",
+        back_populates="_allowed_usages",
+    )
+
+    source_taxonomy_key: Column[str] = Column(
+        String,
+        ForeignKey("taxonomy.fides_key", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    target_type: Column[str] = Column(
+        String, primary_key=True
+    )  # Can be "system", "dataset", OR a taxonomy key like "data_categories"
+
+
+class TaxonomyElement(Base, FidesBase):
+    """
+    The SQL model for taxonomy elements.
+
+    This is a generic taxonomy element model that can be used to create any taxonomy element.
+
+    As of now the legacy taxonomy elements still exist in their own tables (ctl_data_categories, ctl_data_uses, ctl_data_subjects),
+    but we can migrate them to this model in the future if needed.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "taxonomy_element"
+
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    # Which taxonomy this element belongs to
+    taxonomy_type = Column(
+        String,
+        ForeignKey("taxonomy.fides_key", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    parent_key = Column(
+        Text, ForeignKey("taxonomy_element.fides_key", ondelete="RESTRICT"), index=True
+    )
+    active = Column(BOOLEAN, default=True, nullable=False, index=True)
+
+    children: RelationshipProperty[List[TaxonomyElement]] = relationship(
+        "TaxonomyElement",
+        back_populates="parent",
+        cascade="save-update, merge, refresh-expire",  # intentionally do not cascade deletes
+        passive_deletes="all",
+    )
+
+    parent: RelationshipProperty[Optional[TaxonomyElement]] = relationship(
+        "TaxonomyElement",
+        back_populates="children",
+        remote_side="TaxonomyElement.fides_key",
+    )
+
+
+class TaxonomyUsage(Base):
+    """
+    The SQL model for taxonomy usage.
+    Tracks the application of taxonomy elements to other taxonomy elements.
+
+    Example: Applying a "high" tag (from sensitivity taxonomy) to "user.contact.email" (from data_categories taxonomy).
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "taxonomy_usage"
+
+    # The taxonomy element being applied (e.g., risk)
+    source_element_key = Column(String, nullable=False, index=True)
+
+    # The taxonomy element it's being applied to (e.g., a data category)
+    target_element_key = Column(String, nullable=False, index=True)
+
+    # Denormalized taxonomy types for validation and performance
+    source_taxonomy = Column(String, nullable=False, index=True)
+    target_taxonomy = Column(String, nullable=False, index=True)
+
+    __table_args__ = (
+        # Validate that this type of usage is allowed
+        ForeignKeyConstraint(
+            ["source_taxonomy", "target_taxonomy"],
+            [
+                "taxonomy_allowed_usage.source_taxonomy_key",
+                "taxonomy_allowed_usage.target_type",
+            ],
+            ondelete="RESTRICT",
+            name="fk_taxonomy_usage_allowed",
+        ),
+        # Prevent duplicate applications
+        UniqueConstraint(
+            "source_element_key",
+            "target_element_key",
+            name="uq_taxonomy_usage",
+        ),
+    )

fides/api/schemas/application_config.py

@@ -73,7 +73,8 @@ class ExecutionApplicationConfig(FidesSchema):
     memory_watchdog_enabled: Optional[bool] = None
     sql_dry_run: Optional[SqlDryRunMode] = None
 
-    model_config = ConfigDict(use_enum_values=True, extra="forbid")
+    # Allow deprecated / unknown fields (e.g. “safe_mode”) to pass through
+    model_config = ConfigDict(use_enum_values=True, extra="ignore")
 
 
 class AdminUIConfig(FidesSchema):

fides/api/schemas/privacy_center_config.py

@@ -12,10 +12,22 @@ class CustomIdentity(FidesSchema):
     label: str
 
 
+class LocationIdentityField(FidesSchema):
+    """Location field configuration that extends the useful parts of CustomPrivacyRequestField"""
+
+    label: str
+    required: Optional[bool] = True
+    default_value: Optional[str] = None
+    query_param_key: Optional[str] = None
+    ip_geolocation_hint: Optional[bool] = False
+    # Note: We intentionally omit 'hidden' field as it doesn't make sense for location identity input
+
+
 class IdentityInputs(FidesSchema):
     name: Optional[RequiredType] = None
     email: Optional[RequiredType] = None
     phone: Optional[RequiredType] = None
+    location: Optional[Union[RequiredType, LocationIdentityField]] = None
     model_config = ConfigDict(extra="allow")
 
     def __init__(self, **data: Any):
@@ -30,6 +42,9 @@ class IdentityInputs(FidesSchema):
                         f'Custom identity "{field}" must be an instance of CustomIdentity '
                         '(e.g. {"label": "Field label"})'
                     )
+            elif field == "location" and isinstance(value, dict):
+                # Handle location field as LocationIdentityField
+                data[field] = LocationIdentityField(**value)
         super().__init__(**data)
 
 

fides/api/service/deps.py

@@ -8,6 +8,7 @@ from fides.service.dataset.dataset_config_service import DatasetConfigService
 from fides.service.dataset.dataset_service import DatasetService
 from fides.service.messaging.messaging_service import MessagingService
 from fides.service.privacy_request.privacy_request_service import PrivacyRequestService
+from fides.service.taxonomy.taxonomy_service import TaxonomyService
 from fides.service.user.user_service import UserService
 
 
@@ -41,3 +42,7 @@ def get_user_service(
     config_proxy: ConfigProxy = Depends(get_config_proxy),
 ) -> UserService:
     return UserService(db, config, config_proxy)
+
+
+def get_taxonomy_service(db: Session = Depends(get_db)) -> TaxonomyService:
+    return TaxonomyService(db)

fides/api/service/privacy_request/request_service.py

@@ -191,7 +191,11 @@ def poll_for_exited_privacy_request_tasks(self: DatabaseTask) -> Set[str]:
             db.query(PrivacyRequest)
             .filter(
                 PrivacyRequest.status.in_(
-                    [PrivacyRequestStatus.in_processing, PrivacyRequestStatus.approved]
+                    [
+                        PrivacyRequestStatus.in_processing,
+                        PrivacyRequestStatus.approved,
+                        PrivacyRequestStatus.requires_input,
+                    ]
                 )
             )
             # Only look at Privacy Requests that haven't been deleted
@@ -546,6 +550,7 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                         [
                             PrivacyRequestStatus.in_processing,
                             PrivacyRequestStatus.approved,
+                            PrivacyRequestStatus.requires_input,
                         ]
                     )
                 )

fides/api/task/conditional_dependencies/evaluator.py

@@ -1,15 +1,20 @@
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.graph.config import FieldPath
-from fides.api.task.conditional_dependencies.operators import operator_methods
+from fides.api.task.conditional_dependencies.operators import (
+    LOGICAL_OPERATORS,
+    OPERATOR_METHODS,
+)
 from fides.api.task.conditional_dependencies.schemas import (
     Condition,
+    ConditionEvaluationResult,
     ConditionGroup,
     ConditionLeaf,
-    GroupOperator,
+    EvaluationResult,
+    GroupEvaluationResult,
     Operator,
 )
 
@@ -19,84 +24,226 @@ class ConditionEvaluationError(Exception):
 
 
 class ConditionEvaluator:
-    """Evaluates nested conditions for manual task creation"""
+    """Evaluates nested conditions and returns a boolean result and a detailed evaluation report"""
 
     def __init__(self, db: Session):
         self.db = db
 
-    def evaluate_rule(self, rule: Condition, data: Union[dict, Any]) -> bool:
-        """Evaluate a nested condition rule against input data"""
+    def evaluate_rule(
+        self, rule: Condition, data: Union[dict, Any]
+    ) -> EvaluationResult:
+        """Evaluate a nested condition rule against input data and return detailed results
+
+        Args:
+            rule: The condition rule to evaluate
+            data: The data to evaluate the condition against
+
+        Returns:
+            evaluation report: A detailed report of the evaluation
+            - The field address of the condition
+            - The operator used in the condition
+            - The expected value of the condition
+            - The actual value of the condition
+            - The result of the condition evaluation
+            - A message describing the condition evaluation
+        """
         if isinstance(rule, ConditionLeaf):
-            return self._evaluate_leaf_condition(rule, data)
+            leaf_result = self._evaluate_leaf_condition(rule, data)
+            return leaf_result
         # ConditionGroup
-        return self._evaluate_group_condition(rule, data)
+        group_result = self._evaluate_group_condition(rule, data)
+        return group_result
 
     def _evaluate_leaf_condition(
         self, condition: ConditionLeaf, data: Union[dict, Any]
-    ) -> bool:
-        """Evaluate a leaf condition against input data"""
-        data_value = self._get_nested_value(data, condition.field_address.split("."))
-        # Apply operator and return result
-        return self._apply_operator(data_value, condition.operator, condition.value)
+    ) -> ConditionEvaluationResult:
+        """Evaluate a leaf condition against input data
+
+        Args:
+            condition: The leaf condition to evaluate
+            data: The data to evaluate the condition against
+
+        Returns:
+            A detailed evaluation report for the leaf condition
+
+        Raises:
+            ConditionEvaluationError: If there is an issue applying the operator or if an unexpected error occurs.
+        """
+        # Handle both colon-separated and dot-separated field addresses
+        if ":" in condition.field_address:
+            # Full field address like "dataset:collection:field" - split on colons
+            keys = condition.field_address.split(":")
+        else:
+            # Relative field path like "field.subfield" - split on dots
+            keys = condition.field_address.split(".")
+
+        data_value = self._get_nested_value(data, keys)
+
+        # Apply operator and get result
+        try:
+            result = self._apply_operator(
+                data_value, condition.operator, condition.value
+            )
+            message = f"Condition '{condition.field_address} {condition.operator} {condition.value}' evaluated to {result}"
+        except ConditionEvaluationError as e:
+            logger.error(
+                f"Unexpected error evaluating condition '{condition.field_address} {condition.operator} {condition.value}': {str(e)}"
+            )
+            raise
+
+        return ConditionEvaluationResult(
+            field_address=condition.field_address,
+            operator=condition.operator,
+            expected_value=condition.value,
+            actual_value=data_value,
+            result=result,
+            message=message,
+        )
 
     def _evaluate_group_condition(
         self, group: ConditionGroup, data: Union[dict, Any]
-    ) -> bool:
-        """Evaluate a group condition against input data"""
+    ) -> GroupEvaluationResult:
+        """Evaluate a group condition against input data
+
+        Args:
+            group: The group condition to evaluate
+            data: The data to evaluate the condition against
+
+        Returns:
+            A detailed evaluation report for the group condition
+
+        Raises:
+            ConditionEvaluationError: If there is an issue evaluating the group condition (e.g., from evaluate_rule calls)
+        """
+        try:
+            operator_func = LOGICAL_OPERATORS[group.logical_operator]
+        except KeyError as e:
+            raise ConditionEvaluationError(
+                f"Unknown logical operator: {group.logical_operator}"
+            ) from e
+
         results = [
             self.evaluate_rule(condition, data) for condition in group.conditions
         ]
+        group_result = operator_func([r.result for r in results])
+
+        return GroupEvaluationResult(
+            logical_operator=group.logical_operator,
+            condition_results=results,
+            result=group_result,
+        )
 
-        logical_operators = {GroupOperator.and_: all, GroupOperator.or_: any}
-        operator_func = logical_operators.get(group.logical_operator)
+    def _get_nested_value_from_fides_reference_structure(
+        self, data: Any, keys: list[str]
+    ) -> Optional[Any]:
+        """Get nested value from Fides reference structure
 
-        if operator_func is None:
-            logger.warning(f"Unknown logical operator: {group.logical_operator}")
-            return False
+        Args:
+            data: The Fides reference structure to get the nested value from
+            keys: The keys to for the specific nested value in the data
 
-        return operator_func(results)
+        Returns:
+            The nested value from the data or None if not a Fides reference structure
+
+        Raises:
+            AttributeError: If the data does not have a get_field_value method
+            ValueError: If the keys are not valid for the Fides reference structure
+        """
+        if hasattr(data, "get_field_value"):
+            try:
+                field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
+                return data.get_field_value(field_path)
+            except (AttributeError, ValueError):
+                logger.debug(
+                    f"Fides reference structure does not have a get_field_value method: {data}"
+                )
+                raise
+        raise ConditionEvaluationError(
+            f"Data does not have a get_field_value method: {data}"
+        )
+
+    def _get_nested_value_from_dict(self, data: dict, keys: list[str]) -> Optional[Any]:
+        """Get nested value from dictionary. This is the fallback and will return None if the key is not found.
+        When the data is missing the None value will work with exists/not_exists operations and correctly evaluate to False
+        for other operations like eq, not_eq, etc.
+
+        Args:
+            data: The dictionary to get the nested value from
+            keys: The keys to for the specific nested value in the data
+
+        Returns:
+            The nested value from the data
+
+        Raises:
+            KeyError: If the keys are not valid for the dictionary
+        """
+        current: Any = data
+        for key in keys:
+            if not isinstance(current, dict):
+                return None
+            current = current.get(key)
+            if current is None:
+                return None
+        return current
 
     def _get_nested_value(self, data: Union[dict, Any], keys: list[str]) -> Any:
-        """Get nested value from data using dot notation
+        """Get nested value from data using dot notation or colon notation
 
         Supports both simple dictionary access and Fides reference structures:
         - Simple dict: data["user"]["name"]
         - Fides FieldAddress: data.get_field_value(FieldAddress("dataset", "collection", "field_address"))
         - Fides Collection: data.get_field_value(FieldPath("field_address", "subfield"))
+
+        Also supports full field addresses with dataset:collection:field format
+
+        Args:
+            data: The data to get the nested value from
+            keys: The keys to for the specific nested value in the data
+
+        Returns:
+            The nested value from the data
+
+        Raises:
+            KeyError: If the keys are not valid for the dictionary
         """
         if not keys:
             return data
 
-        current = data
-
         # Try Fides reference structures first
-        if hasattr(current, "get_field_value"):
-            try:
-                field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
-                return current.get_field_value(field_path)
-            except (AttributeError, ValueError):
-                pass
-
-        # Fall back to dictionary access
-        for key in keys:
-            if not isinstance(current, dict):
-                current = current.get(key, {}) if hasattr(current, "get") else None
-            else:
-                current = current.get(key, {})
+        try:
+            return self._get_nested_value_from_fides_reference_structure(data, keys)
+        except (AttributeError, ValueError, ConditionEvaluationError):
+            pass
 
-        return current if current != {} else None
+        # Fall back to dictionary access for all path types
+        return self._get_nested_value_from_dict(data, keys)
 
     def _apply_operator(
         self, data_value: Any, operator: Operator, user_input_value: Any
     ) -> bool:
-        """Apply operator to actual and expected values"""
+        """Apply operator to actual and expected values
+        The operator is validated in the ConditionLeaf and ConditionGroup schemas,
+        so we don't need to validate it here.
 
+        Args:
+            data_value: The actual value to evaluate
+            operator: The operator to apply
+            user_input_value: The expected value to evaluate against
+
+        Returns:
+            The result of the operator applied to the actual and expected values
+        """
         # Get the method for the operator and execute it
-        operator_method = operator_methods.get(operator)
-        if operator_method is None:
-            logger.warning(f"Unknown operator: {operator}")
-            raise ConditionEvaluationError(f"Unknown operator: {operator}")
         try:
+            operator_method = OPERATOR_METHODS[operator]
             return operator_method(data_value, user_input_value)
-        except (TypeError, ValueError) as e:
-            raise ConditionEvaluationError(f"Error evaluating condition: {e}") from e
+        except KeyError as e:
+            # Unknown operator
+            logger.error(f"Unknown operator: {operator}")
+            raise ConditionEvaluationError(f"Unknown operator: {operator}") from e
+        except Exception as e:
+            # Log unexpected errors but still raise them
+            logger.error(f"Unexpected error in operator {operator}: {e}")
+            raise ConditionEvaluationError(
+                f"Unexpected error evaluating condition: {e}"
+            ) from e