ethyca-fides 2.69.3b1__py2.py3-none-any.whl → 2.70.0__py2.py3-none-any.whl

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -269,27 +269,30 @@ def privacy_request_csv_download(
     f = io.StringIO()
     csv_file = csv.writer(f)
 
+    privacy_request_ids: List[str] = [r.id for r in privacy_request_query]
+    denial_audit_log_query: Query = db.query(AuditLog).filter(
+        AuditLog.action == AuditLogAction.denied,
+        AuditLog.privacy_request_id.in_(privacy_request_ids),
+    )
+    denial_audit_logs: Dict[str, str] = {
+        r.privacy_request_id: r.message for r in denial_audit_log_query
+    }
+
+    identity_columns, custom_field_columns = get_variable_columns(privacy_request_query)
+
     csv_file.writerow(
         [
             "Status",
             "Request Type",
-            "Subject Identity",
-            "Custom Privacy Request Fields",
             "Time Received",
             "Reviewed By",
             "Request ID",
             "Time Approved/Denied",
             "Denial Reason",
         ]
+        + identity_columns
+        + with_prefix("Custom Field", list(custom_field_columns.values()))
     )
-    privacy_request_ids: List[str] = [r.id for r in privacy_request_query]
-    denial_audit_log_query: Query = db.query(AuditLog).filter(
-        AuditLog.action == AuditLogAction.denied,
-        AuditLog.privacy_request_id.in_(privacy_request_ids),
-    )
-    denial_audit_logs: Dict[str, str] = {
-        r.privacy_request_id: r.message for r in denial_audit_log_query
-    }
 
     for pr in privacy_request_query:
         denial_reason = (
@@ -298,19 +301,22 @@ def privacy_request_csv_download(
             else None
         )
 
-        csv_file.writerow(
-            [
-                pr.status.value if pr.status else None,
-                pr.policy.rules[0].action_type if len(pr.policy.rules) > 0 else None,
-                pr.get_persisted_identity().model_dump(mode="json"),
-                pr.get_persisted_custom_privacy_request_fields(),
-                pr.created_at,
-                pr.reviewed_by,
-                pr.id,
-                pr.reviewed_at,
-                denial_reason,
-            ]
-        )
+        static_cells = [
+            pr.status.value if pr.status else None,
+            pr.policy.rules[0].action_type if len(pr.policy.rules) > 0 else None,
+            pr.created_at,
+            pr.reviewed_by,
+            pr.id,
+            pr.reviewed_at,
+            denial_reason,
+        ]
+
+        identity_cells = extract_identity_cells(identity_columns, pr)
+        custom_field_cells = extract_custom_field_cells(custom_field_columns, pr)
+
+        row = static_cells + identity_cells + custom_field_cells
+
+        csv_file.writerow(row)
 
     f.seek(0)
     response = StreamingResponse(f, media_type="text/csv")
@@ -320,6 +326,64 @@ def privacy_request_csv_download(
     return response
 
 
+def get_variable_columns(
+    privacy_request_query: Query,
+) -> tuple[List[str], Dict[str, str]]:
+    identity_columns: Set[str] = set()
+    custom_field_columns: Dict[str, str] = {}
+
+    for pr in privacy_request_query:
+        identity_columns.update(
+            extract_identity_column_names(pr.get_persisted_identity().model_dump())
+        )
+        custom_field_columns.update(
+            extract_custom_field_column_names(
+                pr.get_persisted_custom_privacy_request_fields()
+            )
+        )
+
+    return list(identity_columns), custom_field_columns
+
+
+def extract_identity_column_names(identities: dict[str, Any]) -> Set[str]:
+    """Extract column names from identity data that have non-empty values."""
+    return {key for key, value in identities.items() if value}
+
+
+def extract_custom_field_column_names(custom_fields: dict[str, Any]) -> Dict[str, str]:
+    """Extract column names and labels from custom field data that have non-empty values."""
+    return {
+        key: value["label"]
+        for key, value in custom_fields.items()
+        if value.get("value")
+    }
+
+
+def with_prefix(prefix: str, items: List[str]) -> List[str]:
+    return [f"{prefix} {item}" for item in items]
+
+
+def extract_custom_field_cells(
+    custom_field_columns: Dict[str, str], pr: PrivacyRequest
+) -> List[str]:
+    custom_fields = pr.get_persisted_custom_privacy_request_fields()
+    return [
+        custom_fields.get(custom_field_column, {}).get("value")
+        for custom_field_column in custom_field_columns
+    ]
+
+
+def extract_identity_cells(
+    identity_columns: List[str], pr: PrivacyRequest
+) -> List[str]:
+    identity = pr.get_persisted_identity()
+    identity_dict = identity.model_dump()
+    return [
+        identity_dict.get(identity_column)  # type: ignore
+        for identity_column in identity_columns
+    ]
+
+
 def execution_and_audit_logs_by_dataset_name(
     self: PrivacyRequest,
 ) -> DefaultDict[str, List[Union["AuditLog", "ExecutionLog"]]]:

fides/api/db/base.py

@@ -22,6 +22,7 @@ from fides.api.models.detection_discovery.monitor_task import (
     MonitorTaskExecutionLog,
 )
 from fides.api.models.detection_discovery.web_monitor import WebMonitorGroupJob
+from fides.api.models.event_audit import EventAudit
 from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.fides_cloud import FidesCloud
 from fides.api.models.fides_user import FidesUser

fides/api/db/seed.py

@@ -2,6 +2,7 @@
 Provides functions that seed the application with data.
 """
 
+import os
 from typing import Dict, List, Optional
 
 from fideslang.default_taxonomy import DEFAULT_TAXONOMY
@@ -17,6 +18,7 @@ from fides.api.common_exceptions import KeyOrNameAlreadyExists
 from fides.api.db.base_class import FidesBase
 from fides.api.db.ctl_session import sync_session
 from fides.api.db.system import upsert_system
+from fides.api.models.application_config import ApplicationConfig
 from fides.api.models.client import ClientDetail
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.datasetconfig import DatasetConfig
@@ -37,7 +39,16 @@ from fides.api.schemas.connection_configuration.saas_config_template_values impo
     SaasConnectionTemplateValues,
 )
 from fides.api.schemas.dataset import DatasetConfigCtlDataset
+from fides.api.schemas.messaging.messaging import (
+    MessagingConfigRequest,
+    MessagingServiceDetailsMailgun,
+    MessagingServiceType,
+)
+from fides.api.schemas.messaging.shared_schemas import MessagingSecretsMailgunDocs
 from fides.api.schemas.policy import ActionType, DrpAction
+from fides.api.service.messaging.messaging_crud_service import (
+    create_or_update_messaging_config,
+)
 from fides.api.util.connection_util import patch_connection_configs
 from fides.api.util.data_category import get_user_data_categories
 from fides.api.util.errors import AlreadyExistsError, QueryError
@@ -486,6 +497,29 @@ async def load_samples(async_session: AsyncSession) -> None:
     except QueryError:  # pragma: no cover
         pass  # The upsert_resources function will log any error
 
+    if os.getenv("MAILGUN_DOMAIN") and os.getenv("MAILGUN_API_KEY"):
+        log.info("Loading Mailgun messaging config")
+        with sync_session() as db:
+            messaging_config = create_or_update_messaging_config(
+                db=db,
+                config=MessagingConfigRequest(
+                    key="my_mailgun_config",
+                    name="Mailgun",
+                    service_type=MessagingServiceType.mailgun,
+                    details=MessagingServiceDetailsMailgun(
+                        domain=os.getenv("MAILGUN_DOMAIN")
+                    ),
+                ),
+            )
+            messaging_config.set_secrets(
+                db=db,
+                messaging_secrets=MessagingSecretsMailgunDocs(
+                    mailgun_api_key=os.getenv("MAILGUN_API_KEY")
+                ),
+            )
+            CONFIG.notifications.notification_service_type = "mailgun"
+            ApplicationConfig.update_config_set(db, CONFIG)
+
 
 def load_default_organization(db: Session) -> None:
     """

fides/api/models/asset.py

@@ -34,6 +34,7 @@ class ConsentStatus(str, Enum):
     with_consent = "with_consent"
     without_consent = "without_consent"
     exempt = "exempt"
+    not_applicable = "not_applicable"
     unknown = "unknown"
 
 

fides/api/models/event_audit.py

@@ -0,0 +1,68 @@
+"""EventAudit model and related enums for comprehensive audit logging."""
+
+from enum import Enum as EnumType
+
+from sqlalchemy import Column, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+
+
+class EventAuditType(str, EnumType):
+    """Hierarchical event types for audit logging - variable depth as needed."""
+
+    # System
+    system_updated = "system.updated"
+
+    # System group
+    system_group_created = "system_group.created"
+    system_group_updated = "system_group.updated"
+    system_group_deleted = "system_group.deleted"
+
+    # Taxonomy
+    taxonomy_created = "taxonomy.created"
+    taxonomy_updated = "taxonomy.updated"
+    taxonomy_deleted = "taxonomy.deleted"
+    taxonomy_usage_updated = "taxonomy.usage.updated"
+    taxonomy_usage_deleted = "taxonomy.usage.deleted"
+    taxonomy_element_created = "taxonomy.element.created"
+    taxonomy_element_updated = "taxonomy.element.updated"
+    taxonomy_element_deleted = "taxonomy.element.deleted"
+
+
+class EventAuditStatus(str, EnumType):
+    """Status enum for event audit logging."""
+
+    succeeded = "succeeded"
+    failed = "failed"
+
+
+class EventAudit(Base):
+    """Audit log for significant business events across the Fides platform."""
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return "event_audit"
+
+    # Uses EventAuditType values but left as String to avoid future migrations
+    event_type = Column(String, index=True, nullable=False)
+    user_id = Column(String, nullable=True, index=True)
+
+    # Resource information
+    resource_type = Column(String, nullable=True, index=True)
+    resource_identifier = Column(String, nullable=True, index=True)
+
+    # User-friendly description of event
+    description = Column(Text, nullable=True)
+
+    # Structured data about event
+    event_details = Column(JSONB, nullable=True)
+
+    # Status of the event
+    status = Column(
+        EnumColumn(EventAuditStatus),
+        index=True,
+        nullable=False,
+    )

fides/api/models/messaging.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
+from datetime import datetime
 from typing import Dict, Optional, Type, Union
 
 from loguru import logger
 from pydantic import ValidationError
-from sqlalchemy import Column, Enum, String
+from sqlalchemy import Boolean, Column, DateTime, Enum, String
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.orm import Session
@@ -18,17 +19,18 @@ from fides.api.db.base_class import Base, JSONTypeOverride
 from fides.api.schemas.messaging.messaging import (
     EMAIL_MESSAGING_SERVICES,
     SMS_MESSAGING_SERVICES,
-    SUPPORTED_MESSAGING_SERVICE_SECRETS,
+    MessagingConnectionTestStatus,
     MessagingMethod,
+    MessagingServiceType,
+)
+from fides.api.schemas.messaging.shared_schemas import (
+    SUPPORTED_MESSAGING_SERVICE_SECRETS,
     MessagingServiceSecretsAWS_SES,
     MessagingServiceSecretsMailchimpTransactional,
     MessagingServiceSecretsMailgun,
     MessagingServiceSecretsTwilioEmail,
     MessagingServiceSecretsTwilioSMS,
-    MessagingServiceType,
-)
-from fides.api.schemas.messaging.messaging_secrets_docs_only import (
-    possible_messaging_secrets,
+    PossibleMessagingSecrets,
 )
 from fides.api.util.logger import Pii
 from fides.config import CONFIG
@@ -48,7 +50,7 @@ def get_messaging_method(
 
 def get_schema_for_secrets(
     service_type: MessagingServiceType,
-    secrets: possible_messaging_secrets,
+    secrets: PossibleMessagingSecrets,
 ) -> SUPPORTED_MESSAGING_SERVICE_SECRETS:
     """
     Returns the secrets that pertain to `service_type` represented as a Pydantic schema
@@ -87,6 +89,8 @@ class MessagingConfig(Base):
         Enum(MessagingServiceType), index=True, unique=True, nullable=False
     )
     details = Column(MutableDict.as_mutable(JSONB), nullable=True)
+    last_test_timestamp = Column(DateTime(timezone=True))
+    last_test_succeeded = Column(Boolean)
     secrets = Column(
         MutableDict.as_mutable(
             StringEncryptedType(
@@ -136,7 +140,7 @@ class MessagingConfig(Base):
         self,
         *,
         db: Session,
-        messaging_secrets: possible_messaging_secrets,
+        messaging_secrets: PossibleMessagingSecrets,
     ) -> None:
         """Creates or updates secrets associated with a config id"""
 
@@ -185,6 +189,18 @@ class MessagingConfig(Base):
                 f"Unknown notification_service_type {active_default_messaging_type} configured"
             )
 
+    def update_test_status(
+        self, test_status: MessagingConnectionTestStatus, db: Session
+    ) -> None:
+        """
+        Updates last_test_timestamp and last_test_succeeded after an attempt to send a test message.
+        """
+        self.last_test_timestamp = datetime.now()
+        self.last_test_succeeded = (
+            test_status == MessagingConnectionTestStatus.succeeded
+        )
+        self.save(db)
+
 
 def default_messaging_config_name(service_type: str) -> str:
     """
@@ -192,7 +208,7 @@ def default_messaging_config_name(service_type: str) -> str:
 
     Returns a name to be used in a default messaging config for the given type.
     """
-    return f"Default Messaging Config [{service_type}]"
+    return MessagingServiceType(service_type).human_readable
 
 
 def default_messaging_config_key(service_type: str) -> str:
@@ -201,4 +217,8 @@ def default_messaging_config_key(service_type: str) -> str:
 
     Returns a key to be used in a default messaging config for the given type.
     """
-    return f"default_messaging_config_{service_type.lower()}"
+    # Historically, when a key was not supplied, the service type string itself
+    # was used as the key (e.g. "mailgun").  Reverting to that behaviour keeps
+    # backward-compatibility with existing tests and API consumers that may rely
+    # on this convention.
+    return service_type.lower()

fides/api/models/taxonomy.py

@@ -85,10 +85,17 @@ class Taxonomy(Base, FidesBase):
             raise ValidationError(
                 f"Cannot create taxonomy '{fides_key}'. This is a taxonomy managed by the system."
             )
-        applies_to = data.pop("applies_to", [])
+        applies_to = data.get("applies_to", [])
+
+        # Create a copy of data without applies_to to avoid modifying the input
+        data_without_applies_to = {
+            key: value for key, value in data.items() if key != "applies_to"
+        }
 
         # Create the taxonomy
-        taxonomy: Taxonomy = super().create(db=db, data=data, check_name=check_name)
+        taxonomy: Taxonomy = super().create(
+            db=db, data=data_without_applies_to, check_name=check_name
+        )
 
         # Reconcile allowed usages if applies_to was provided
         if applies_to:
@@ -98,10 +105,15 @@ class Taxonomy(Base, FidesBase):
 
     def update(self, db: Session, *, data: Dict[str, Any]) -> "Taxonomy":
         """Update a Taxonomy with proper handling of applies_to."""
-        applies_to = data.pop("applies_to", None)
+        applies_to = data.get("applies_to", None)
+
+        # Create a copy of data without applies_to to avoid modifying the input
+        data_without_applies_to = {
+            key: value for key, value in data.items() if key != "applies_to"
+        }
 
         # Update the base fields
-        super().update(db=db, data=data)
+        super().update(db=db, data=data_without_applies_to)
 
         # If applies_to was provided, reconcile allowed usages
         if applies_to is not None:

fides/api/schemas/application_config.py

@@ -53,8 +53,11 @@ class NotificationApplicationConfig(FidesSchema):
 
     @field_validator("notification_service_type", mode="before")
     @classmethod
-    def validate_notification_service_type(cls, value: str) -> Optional[str]:
+    def validate_notification_service_type(cls, value: Optional[str]) -> Optional[str]:
         """Ensure the provided type is a valid value."""
+        if value is None:
+            return None  # Allow None values for disabling messaging
+
         value = value.lower()  # force lowercase for safety
         try:
             MessagingServiceType[value]

fides/api/schemas/messaging/messaging.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from datetime import datetime
 from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
@@ -7,15 +8,14 @@ from fideslang.default_taxonomy import DEFAULT_TAXONOMY
 from fideslang.validation import FidesKey
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
-from fides.api.custom_types import PhoneNumber, SafeStr
+from fides.api.custom_types import SafeStr
 from fides.api.schemas import Msg
 from fides.api.schemas.api import BulkResponse, BulkUpdateFailed
-from fides.api.schemas.connection_configuration.connection_secrets_base_aws import (
-    BaseAWSSchema,
-)
+from fides.api.schemas.messaging.shared_schemas import PossibleMessagingSecrets
 from fides.api.schemas.privacy_preference import MinimalPrivacyPreferenceHistorySchema
 from fides.api.schemas.privacy_request import Consent
 from fides.api.schemas.property import MinimalProperty
+from fides.api.schemas.redis_cache import IdentityBase
 
 
 class MessagingMethod(Enum):
@@ -44,6 +44,25 @@ class MessagingServiceType(Enum):
                 return member
         return None
 
+    @property
+    def human_readable(self) -> str:
+        """
+        Human-readable mapping for MessagingServiceType
+        """
+        readable_mapping: Dict[str, str] = {
+            MessagingServiceType.mailgun.value: "Mailgun",
+            MessagingServiceType.twilio_text.value: "Twilio SMS",
+            MessagingServiceType.twilio_email.value: "Twilio Email",
+            MessagingServiceType.mailchimp_transactional.value: "Mailchimp Transactional",
+            MessagingServiceType.aws_ses.value: "AWS SES",
+        }
+        try:
+            return readable_mapping[self.value]
+        except KeyError:
+            raise NotImplementedError(
+                "Add new MessagingServiceType to human_readable mapping"
+            )
+
 
 EMAIL_MESSAGING_SERVICES: Tuple[str, ...] = (
     MessagingServiceType.mailgun.value,
@@ -85,6 +104,12 @@ CONFIGURABLE_MESSAGING_ACTION_TYPES: Tuple[str, ...] = (
 )
 
 
+class MessagingTestBodyParams(BaseModel):
+    """Body params required for testing messaging service"""
+
+    to_identity: IdentityBase
+
+
 class ErrorNotificationBodyParams(BaseModel):
     """Body params required for privacy request error notifications."""
 
@@ -339,56 +364,6 @@ class MessagingServiceSecrets(Enum):
     AWS_ASSUME_ROLE_ARN = "aws_assume_role_arn"
 
 
-class MessagingServiceSecretsMailchimpTransactional(BaseModel):
-    """The secrets required to connect to Mailchimp Transactional."""
-
-    mailchimp_transactional_api_key: str
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessagingServiceSecretsMailgun(BaseModel):
-    """The secrets required to connect to Mailgun."""
-
-    mailgun_api_key: str
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessagingServiceSecretsTwilioSMS(BaseModel):
-    """The secrets required to connect to Twilio SMS."""
-
-    twilio_account_sid: str
-    twilio_auth_token: str
-    twilio_messaging_service_sid: Optional[str] = None
-    twilio_sender_phone_number: Optional[PhoneNumber] = None
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="before")
-    @classmethod
-    def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        sender_phone = values.get("twilio_sender_phone_number")
-        if not values.get("twilio_messaging_service_sid") and not sender_phone:
-            raise ValueError(
-                "Either the twilio_messaging_service_sid or the twilio_sender_phone_number should be supplied."
-            )
-        return values
-
-
-class MessagingServiceSecretsTwilioEmail(BaseModel):
-    """The secrets required to connect to twilio email."""
-
-    twilio_api_key: str
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessagingServiceSecretsAWS_SES(BaseAWSSchema):
-    """
-    The secrets required to connect to AWS SES.
-    Inherits basic AWS authentication schema.
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-
 class MessagingConfigBase(BaseModel):
     """Base model shared by messaging config related models"""
 
@@ -453,8 +428,9 @@ class MessagingConfigRequestBase(MessagingConfigBase):
 class MessagingConfigRequest(MessagingConfigRequestBase):
     """Messaging Config Request Schema"""
 
-    name: str
+    name: Optional[str] = None
     key: Optional[FidesKey] = None
+    secrets: Optional[PossibleMessagingSecrets] = None
 
 
 class MessagingConfigResponse(MessagingConfigBase):
@@ -462,16 +438,8 @@ class MessagingConfigResponse(MessagingConfigBase):
 
     name: str
     key: FidesKey
-    model_config = ConfigDict(from_attributes=True, use_enum_values=True)
-
-
-SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[
-    MessagingServiceSecretsMailgun,
-    MessagingServiceSecretsTwilioSMS,
-    MessagingServiceSecretsTwilioEmail,
-    MessagingServiceSecretsMailchimpTransactional,
-    MessagingServiceSecretsAWS_SES,
-]
+    last_test_timestamp: Optional[datetime] = None
+    last_test_succeeded: Optional[bool] = None
 
 
 class MessagingConnectionTestStatus(Enum):
@@ -479,7 +447,6 @@ class MessagingConnectionTestStatus(Enum):
 
     succeeded = "succeeded"
     failed = "failed"
-    skipped = "skipped"
 
 
 class TestMessagingStatusMessage(Msg):

fides/api/schemas/messaging/messaging_secrets_docs_only.py

@@ -1,7 +1,7 @@
 from typing import Union
 
 from fides.api.schemas.base_class import NoValidationSchema
-from fides.api.schemas.messaging.messaging import (
+from fides.api.schemas.messaging.shared_schemas import (
     MessagingServiceSecretsAWS_SES,
     MessagingServiceSecretsMailchimpTransactional,
     MessagingServiceSecretsMailgun,