ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/models/event_audit.py

@@ -0,0 +1,85 @@
+"""EventAudit model and related enums for comprehensive audit logging."""
+
+from enum import Enum as EnumType
+
+from sqlalchemy import Column, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+
+
+class EventAuditType(str, EnumType):
+    """Hierarchical event types for audit logging - variable depth as needed."""
+
+    # System
+    system_updated = "system.updated"
+
+    # System group
+    system_group_created = "system_group.created"
+    system_group_updated = "system_group.updated"
+    system_group_deleted = "system_group.deleted"
+
+    # Taxonomy
+    taxonomy_created = "taxonomy.created"
+    taxonomy_updated = "taxonomy.updated"
+    taxonomy_deleted = "taxonomy.deleted"
+    taxonomy_usage_updated = "taxonomy.usage.updated"
+    taxonomy_usage_deleted = "taxonomy.usage.deleted"
+    taxonomy_element_created = "taxonomy.element.created"
+    taxonomy_element_updated = "taxonomy.element.updated"
+    taxonomy_element_deleted = "taxonomy.element.deleted"
+
+    # Connection operations
+    connection_created = "connection.created"
+    connection_updated = "connection.updated"
+    connection_deleted = "connection.deleted"
+
+    # Connection secrets operations
+    connection_secrets_created = "connection.secrets.created"
+    connection_secrets_updated = "connection.secrets.updated"
+
+    # Digest
+    digest_execution_started = "digest.execution.started"
+    digest_execution_completed = "digest.execution.completed"
+    digest_execution_interrupted = "digest.execution.interrupted"
+    digest_execution_resumed = "digest.execution.resumed"
+    digest_communications_sent = "digest.communications.sent"
+    digest_checkpoint_created = "digest.checkpoint.created"
+
+
+class EventAuditStatus(str, EnumType):
+    """Status enum for event audit logging."""
+
+    succeeded = "succeeded"
+    failed = "failed"
+
+
+class EventAudit(Base):
+    """Audit log for significant business events across the Fides platform."""
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "event_audit"
+
+    # Uses EventAuditType values but left as String to avoid future migrations
+    event_type = Column(String, index=True, nullable=False)
+    user_id = Column(String, nullable=True, index=True)
+
+    # Resource information
+    resource_type = Column(String, nullable=True, index=True)
+    resource_identifier = Column(String, nullable=True, index=True)
+
+    # User-friendly description of event
+    description = Column(Text, nullable=True)
+
+    # Structured data about event
+    event_details = Column(JSONB, nullable=True)
+
+    # Status of the event
+    status = Column(
+        EnumColumn(EventAuditStatus),
+        index=True,
+        nullable=False,
+    )

fides/api/models/experience_notices.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from uuid import uuid4
 
-from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy import Column, ForeignKey, Integer, String
 
 from fides.api.db.base_class import Base
 
@@ -42,3 +42,8 @@ class ExperienceNotices(Base):
         nullable=False,
         primary_key=True,
     )
+    display_order = Column(
+        Integer,
+        nullable=True,
+        doc="Optional ordering for notices within an experience. Lower values appear first.",
+    )

fides/api/models/fides_user.py

@@ -31,6 +31,7 @@ if TYPE_CHECKING:
     from fides.api.models.fides_user_respondent_email_verification import (
         FidesUserRespondentEmailVerification,
     )
+    from fides.api.models.manual_task.manual_task import ManualTaskReference
     from fides.api.models.sql_models import System  # type: ignore[attr-defined]
     from fides.api.models.system_manager import SystemManager
 
@@ -90,6 +91,14 @@ class FidesUser(Base):
         uselist=False,
     )
 
+    # Manual task assignments relationship
+    manual_task_references = relationship(
+        "ManualTaskReference",
+        primaryjoin="and_(FidesUser.id == foreign(ManualTaskReference.reference_id), "
+        "ManualTaskReference.reference_type == 'assigned_user')",
+        viewonly=True,
+    )
+
     @property
     def system_ids(self) -> List[str]:
         return [system.id for system in self.systems]

fides/api/models/identity_definition.py

@@ -0,0 +1,66 @@
+from enum import Enum
+
+from sqlalchemy import Boolean, Column, String, Text
+from sqlalchemy.ext.declarative import declared_attr
+
+from fides.api.db.base_class import Base, FidesBase
+from fides.api.db.util import EnumColumn
+
+
+class IdentityDefinitionType(str, Enum):
+    """Enum for the type of identity"""
+
+    EMAIL = "email"
+    PHONE_NUMBER = "phone_number"
+    UUID = "uuid"
+    STRING = "string"
+    INTEGER = "integer"
+
+
+class IdentityDefinition(Base):
+    """
+    Model for identity definitions in Fides. This isn't for specific identity values,
+    but for the types of identities that can be used in Fides.
+
+    For example:
+    ```json
+    {
+        "identity_key": "customer_id",
+        "name": "Customer ID",
+        "description": "The unique identifier for the customer",
+        "type": "string"
+    }
+    ```
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "identity_definition"
+
+    # Overriding the id definition from Base so we don't treat this as the primary key
+    id = Column(
+        String(255),
+        nullable=False,
+        index=False,
+        unique=True,
+        default=FidesBase.generate_uuid,
+    )
+
+    # Primary key
+    identity_key = Column(String(255), primary_key=True)
+
+    # Schema definition
+    name = Column(String(255), nullable=False)
+    description = Column(Text, nullable=True)
+    type = Column(
+        EnumColumn(
+            IdentityDefinitionType,
+            native_enum=False,
+            values_callable=lambda x: [
+                i.value for i in x
+            ],  # allows enum _values_ to be stored rather than name
+        ),
+        nullable=False,
+    )
+    created_by = Column(String(255), nullable=True)
+    is_default = Column(Boolean, nullable=False, server_default="false")

fides/api/models/location_regulation_selections.py

@@ -408,6 +408,28 @@ privacy_notice_regions_by_id: Dict[str, Union[Location, LocationGroup]] = (
 )  # should only be accessed for read-only access
 
 
+def get_location_by_id(location: str) -> Optional[Location]:
+    """
+    Get a Location object by ID, with automatic format normalization.
+
+    Handles various location formats (e.g., "US-CA", "us-ca", "us_ca") by
+    normalizing to the internal format (lowercase with underscores) used in locations.yml.
+
+    Args:
+        location: Location string in any format
+
+    Returns:
+        Location object if found, None otherwise
+
+    Examples:
+        >>> get_location_by_id("US-CA")  # ISO 3166 format
+        >>> get_location_by_id("us_ca")  # Internal format
+        >>> get_location_by_id("us-ca")  # Mixed format
+    """
+    normalized = location.lower().replace("-", "_")
+    return locations_by_id.get(normalized)
+
+
 # dynamically create an enum based on definitions loaded from YAML
 # This is a combination of "locations" and "location groups" for use on Privacy Experiences
 PrivacyNoticeRegion = Enum(  # type: ignore[misc]

fides/api/models/manual_task/conditional_dependency.py

@@ -1,13 +1,13 @@
-from enum import Enum
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
 
-from sqlalchemy import Column, ForeignKey, Index, Integer, String
-from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy import Column, ForeignKey, String
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
-from fides.api.db.base_class import Base, FidesBase
-from fides.api.db.util import EnumColumn
+from fides.api.db.base_class import FidesBase
+from fides.api.models.conditional_dependency.conditional_dependency_base import (
+    ConditionalDependencyBase,
+)
 from fides.api.task.conditional_dependencies.schemas import (
     ConditionGroup,
     ConditionLeaf,
@@ -17,69 +17,28 @@ if TYPE_CHECKING:
     from fides.api.models.manual_task.manual_task import ManualTask
 
 
-class ManualTaskConditionalDependencyType(str, Enum):
-    """Enum for manual task conditional dependency types.
-
-    This enum defines the two types of nodes in a conditional dependency tree:
-
-    - leaf: A terminal node that represents a single condition (e.g., "user.age >= 18")
-    - group: A non-terminal node that groups multiple conditions with logical operators (AND/OR)
-
-    Examples:
-        leaf: Used for simple field comparisons like:
-            - "user.name exists"
-            - "user.age >= 18"
-            - "billing.subscription.status == 'active'"
-
-        group: Used to combine multiple conditions with logical operators:
-            - AND group: "user.age >= 18 AND user.active == true"
-            - OR group: "user.role == 'admin' OR user.verified == true"
-            - Nested groups: "(user.age >= 18 AND (user.role == 'admin' OR user.verified == true))"
-    """
-
-    leaf = "leaf"
-    group = "group"
-
-
-class ManualTaskConditionalDependency(Base):
-    """Model for storing conditional dependencies."""
+class ManualTaskConditionalDependency(ConditionalDependencyBase):
+    """Manual task conditional dependencies - single type hierarchy."""
 
     @declared_attr
     def __tablename__(cls) -> str:
-        """Overriding base class method to set the table name."""
         return "manual_task_conditional_dependency"
 
     # We need to redefine it here so that self-referential relationships
     # can properly reference the `id` column instead of the built-in Python function.
     id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
-
     # Foreign key relationships
     manual_task_id = Column(
-        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+        String,
+        ForeignKey("manual_task.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
     )
     parent_id = Column(
         String,
         ForeignKey("manual_task_conditional_dependency.id", ondelete="CASCADE"),
         nullable=True,
-    )
-
-    # Condition metadata
-    condition_type = Column(
-        EnumColumn(ManualTaskConditionalDependencyType), nullable=False
-    )  # leaf or group
-    field_address = Column(String, nullable=True)  # For leaf conditions
-    operator = Column(String, nullable=True)  # For leaf conditions
-    value = Column(JSONB, nullable=True)  # For leaf conditions
-    logical_operator = Column(String, nullable=True)  # 'and' or 'or' for groups
-
-    # Ordering
-    sort_order = Column(Integer, nullable=False, default=0)
-
-    __table_args__ = (
-        Index("ix_manual_task_conditional_dependency_manual_task_id", "manual_task_id"),
-        Index("ix_manual_task_conditional_dependency_parent_id", "parent_id"),
-        Index("ix_manual_task_conditional_dependency_condition_type", "condition_type"),
-        Index("ix_manual_task_conditional_dependency_sort_order", "sort_order"),
+        index=True,
     )
 
     # Relationships
@@ -97,38 +56,24 @@ class ManualTaskConditionalDependency(Base):
         foreign_keys=[parent_id],
     )
 
-    def to_condition_leaf(self) -> ConditionLeaf:
-        """Convert to ConditionLeaf if this is a leaf condition"""
-        if self.condition_type != "leaf":
-            raise ValueError("Cannot convert group condition to leaf")
-
-        return ConditionLeaf(
-            field_address=self.field_address, operator=self.operator, value=self.value
-        )
-
-    def to_condition_group(self) -> ConditionGroup:
-        """Convert to ConditionGroup if this is a group condition"""
-        if self.condition_type != "group":
-            raise ValueError("Cannot convert leaf condition to group")
-
-        # Recursively build children
-        child_conditions = []
-        children_list = [child for child in self.children]  # type: ignore[attr-defined]
-        for child in sorted(children_list, key=lambda x: x.sort_order):
-            if child.condition_type == "leaf":
-                child_conditions.append(child.to_condition_leaf())
-            else:
-                child_conditions.append(child.to_condition_group())
-
-        return ConditionGroup(
-            logical_operator=self.logical_operator, conditions=child_conditions
-        )
-
     @classmethod
     def get_root_condition(
-        cls, db: Session, manual_task_id: str
+        cls, db: Session, **kwargs: Any
     ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
-        """Get the root condition for a config"""
+        """Get the root condition for a manual task
+
+        Args:
+            db: Database session
+            **kwargs: Keyword arguments containing:
+                manual_task_id: ID of the manual task
+
+        Raises:
+            ValueError: If manual_task_id is not provided
+        """
+        manual_task_id = kwargs.get("manual_task_id")
+
+        if not manual_task_id:
+            raise ValueError("manual_task_id is required as a keyword argument")
         root = (
             db.query(cls)
             .filter(cls.manual_task_id == manual_task_id, cls.parent_id.is_(None))
@@ -138,7 +83,4 @@ class ManualTaskConditionalDependency(Base):
         if not root:
             return None
 
-        if root.condition_type == "leaf":
-            return root.to_condition_leaf()
-
-        return root.to_condition_group()
+        return root.to_correct_condition_type()

fides/api/models/manual_webhook.py

@@ -5,7 +5,7 @@ from pydantic import ConfigDict, create_model
 from sqlalchemy import Column, ForeignKey, String, or_, text
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.mutable import MutableList
-from sqlalchemy.orm import Session, relationship
+from sqlalchemy.orm import Session, relationship, selectinload
 
 from fides.api.db.base_class import Base
 from fides.api.models.attachment import Attachment, AttachmentReference
@@ -137,11 +137,22 @@ class AccessManualWebhook(Base):
     ) -> list["AccessManualWebhook"]:
         """Get all enabled access manual webhooks with fields"""
 
-        query = db.query(cls).filter(
-            AccessManualWebhook.connection_config_id == ConnectionConfig.id,
-            ConnectionConfig.disabled.is_(False),
-            AccessManualWebhook.fields != text("'null'"),
-            AccessManualWebhook.fields != "[]",
+        query = (
+            db.query(cls)
+            .join(
+                ConnectionConfig,
+                AccessManualWebhook.connection_config_id == ConnectionConfig.id,
+            )
+            .options(
+                selectinload(AccessManualWebhook.connection_config).selectinload(
+                    ConnectionConfig.system
+                ),  # type: ignore[attr-defined]
+            )
+            .filter(
+                ConnectionConfig.disabled.is_(False),
+                AccessManualWebhook.fields != text("'null'"),
+                AccessManualWebhook.fields != "[]",
+            )
         )
 
         # Add action_type filter only if action_type is provided

fides/api/models/messaging.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
+from datetime import datetime
 from typing import Dict, Optional, Type, Union
 
 from loguru import logger
 from pydantic import ValidationError
-from sqlalchemy import Column, Enum, String
+from sqlalchemy import Boolean, Column, DateTime, Enum, String
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.orm import Session
@@ -18,17 +19,18 @@ from fides.api.db.base_class import Base, JSONTypeOverride
 from fides.api.schemas.messaging.messaging import (
     EMAIL_MESSAGING_SERVICES,
     SMS_MESSAGING_SERVICES,
-    SUPPORTED_MESSAGING_SERVICE_SECRETS,
+    MessagingConnectionTestStatus,
     MessagingMethod,
+    MessagingServiceType,
+)
+from fides.api.schemas.messaging.shared_schemas import (
+    SUPPORTED_MESSAGING_SERVICE_SECRETS,
     MessagingServiceSecretsAWS_SES,
     MessagingServiceSecretsMailchimpTransactional,
     MessagingServiceSecretsMailgun,
     MessagingServiceSecretsTwilioEmail,
     MessagingServiceSecretsTwilioSMS,
-    MessagingServiceType,
-)
-from fides.api.schemas.messaging.messaging_secrets_docs_only import (
-    possible_messaging_secrets,
+    PossibleMessagingSecrets,
 )
 from fides.api.util.logger import Pii
 from fides.config import CONFIG
@@ -48,7 +50,7 @@ def get_messaging_method(
 
 def get_schema_for_secrets(
     service_type: MessagingServiceType,
-    secrets: possible_messaging_secrets,
+    secrets: PossibleMessagingSecrets,
 ) -> SUPPORTED_MESSAGING_SERVICE_SECRETS:
     """
     Returns the secrets that pertain to `service_type` represented as a Pydantic schema
@@ -87,6 +89,8 @@ class MessagingConfig(Base):
         Enum(MessagingServiceType), index=True, unique=True, nullable=False
     )
     details = Column(MutableDict.as_mutable(JSONB), nullable=True)
+    last_test_timestamp = Column(DateTime(timezone=True))
+    last_test_succeeded = Column(Boolean)
     secrets = Column(
         MutableDict.as_mutable(
             StringEncryptedType(
@@ -136,7 +140,7 @@ class MessagingConfig(Base):
         self,
         *,
         db: Session,
-        messaging_secrets: possible_messaging_secrets,
+        messaging_secrets: PossibleMessagingSecrets,
     ) -> None:
         """Creates or updates secrets associated with a config id"""
 
@@ -185,6 +189,18 @@ class MessagingConfig(Base):
                 f"Unknown notification_service_type {active_default_messaging_type} configured"
             )
 
+    def update_test_status(
+        self, test_status: MessagingConnectionTestStatus, db: Session
+    ) -> None:
+        """
+        Updates last_test_timestamp and last_test_succeeded after an attempt to send a test message.
+        """
+        self.last_test_timestamp = datetime.now()
+        self.last_test_succeeded = (
+            test_status == MessagingConnectionTestStatus.succeeded
+        )
+        self.save(db)
+
 
 def default_messaging_config_name(service_type: str) -> str:
     """
@@ -192,7 +208,7 @@ def default_messaging_config_name(service_type: str) -> str:
 
     Returns a name to be used in a default messaging config for the given type.
     """
-    return f"Default Messaging Config [{service_type}]"
+    return MessagingServiceType(service_type).human_readable
 
 
 def default_messaging_config_key(service_type: str) -> str:
@@ -201,4 +217,8 @@ def default_messaging_config_key(service_type: str) -> str:
 
     Returns a key to be used in a default messaging config for the given type.
     """
-    return f"default_messaging_config_{service_type.lower()}"
+    # Historically, when a key was not supplied, the service type string itself
+    # was used as the key (e.g. "mailgun").  Reverting to that behaviour keeps
+    # backward-compatibility with existing tests and API consumers that may rely
+    # on this convention.
+    return service_type.lower()

fides/api/models/messaging_template.py

@@ -58,6 +58,20 @@ DEFAULT_MESSAGING_TEMPLATES: Dict[str, Any] = {
             "body": "Your erasure request has been completed.",
         },
     },
+    MessagingActionType.MANUAL_TASK_DIGEST.value: {
+        "label": "Manual task digest",
+        "content": {
+            "subject": "Weekly DSR Summary from __ORGANIZATION_NAME__",
+            "body": "This is your weekly summary of open data subject requests tasks from __ORGANIZATION_NAME__ that require your attention. Please review the tasks in our Privacy Center to ensure timely completion.",
+        },
+    },
+    MessagingActionType.EXTERNAL_USER_WELCOME.value: {
+        "label": "External user welcome",
+        "content": {
+            "subject": "Welcome to our Privacy Center",
+            "body": "An account has been created for you by the privacy team at __ORG_NAME__ so that you can assist them in responding to data subject requests, as required by the GDPR and many US state laws, for users that have data in your systems. A portal is available to view and take action on these requests at: __PORTAL_LINK__. Through this portal you can upload or provide user data for users who have requested a copy of their information and attest that users have been removed from your system in the case of an erasure request. The provided link is valid for 30 days and you will be asked to verify your email address each time you access the portal. If you have any questions, please reach out first to your contacts at: __ORG_NAME__",
+        },
+    },
 }
 
 

fides/api/models/policy.py

@@ -142,6 +142,9 @@ class Policy(Base):
         except IndexError:
             return None
 
+    def get_all_action_types(self) -> Set[ActionType]:
+        return set(rule.action_type for rule in self.rules)  # type: ignore[attr-defined]
+
     def generate_masking_secrets(self) -> Optional[List[MaskingSecretCache]]:
         """
         Returns a list of masking secrets for the masking strategies in the policy.
@@ -177,6 +180,30 @@ class Policy(Base):
             for category in node.get_data_categories()
         )
 
+    def to_safe_dict(self) -> Dict[str, Any]:
+        """
+        Returns a dictionary representation of the Policy, excluding sensitive information.
+        """
+        return {
+            "id": self.id,
+            "name": self.name,
+            "key": self.key,
+            "drp_action": self.drp_action.value if self.drp_action else None,  # type: ignore[attr-defined]
+            "execution_timeframe": self.execution_timeframe,
+            "client_id": self.client_id,
+            "rules": [
+                {
+                    "id": rule.id,
+                    "name": rule.name,
+                    "key": rule.key,
+                    "action_type": rule.action_type.value,
+                    "masking_strategy": rule.masking_strategy,
+                    "storage_destination_id": rule.storage_destination_id,
+                }
+                for rule in self.rules  # type: ignore[attr-defined]
+            ],
+        }
+
 
 def _get_ref_from_taxonomy(
     fides_key: FidesKey,