ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/schemas/connection_configuration/connection_config.py

@@ -13,6 +13,7 @@ from fides.api.schemas.connection_configuration import connection_secrets_schema
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.saas.saas_config import SaaSConfigBase
 from fides.api.util.connection_type import get_connection_type_secret_schema
+from fides.api.util.masking_util import mask_sensitive_fields
 
 
 class CreateConnectionConfiguration(BaseModel):
@@ -41,36 +42,6 @@ class CreateConnectionConfigurationWithSecrets(CreateConnectionConfiguration):
     model_config = ConfigDict(from_attributes=True, extra="ignore")
 
 
-def mask_sensitive_fields(
-    connection_secrets: Dict[str, Any], secret_schema: Dict[str, Any]
-) -> Dict[str, Any]:
-    """
-    Mask sensitive fields in the given secrets based on the provided schema.
-    This function traverses the given secrets dictionary and uses the provided schema to
-    identify fields that have been marked as sensitive. The function replaces the sensitive
-    field values with a mask string ('********').
-    Args:
-        connection_secrets (Dict[str, Any]): The secrets to be masked.
-        secret_schema (Dict[str, Any]): The schema defining which fields are sensitive.
-    Returns:
-        Dict[str, Any]: The secrets dictionary with sensitive fields masked.
-    """
-    if connection_secrets is None:
-        return connection_secrets
-
-    secret_schema_properties = secret_schema["properties"]
-    new_connection_secrets = {}
-
-    for key, value in connection_secrets.items():
-        if key in secret_schema_properties:
-            if secret_schema_properties.get(key, {}).get("sensitive", False):
-                new_connection_secrets[key] = "**********"
-            else:
-                new_connection_secrets[key] = value
-
-    return new_connection_secrets
-
-
 class ConnectionConfigSecretsMixin(BaseModel):
     """
     A schema mixin to declare a connection config `secrets` attribute

fides/api/schemas/connection_configuration/connection_oauth_config.py

@@ -0,0 +1,42 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from fides.api.schemas.base_class import NoValidationSchema
+
+
+class OAuthGrantType(str, Enum):
+    """OAuth2 grant types supported by the system"""
+
+    client_credentials = "client_credentials"
+
+
+class OAuthConfigSchema(BaseModel):
+    """Schema for OAuth2 configuration used in API requests"""
+
+    grant_type: Optional[OAuthGrantType] = None
+    token_url: Optional[str] = None
+    scope: Optional[str] = None
+    client_id: Optional[str] = None
+    client_secret: Optional[str] = Field(json_schema_extra={"sensitive": True})
+
+    model_config = ConfigDict(
+        from_attributes=True, use_enum_values=True, extra="forbid"
+    )
+
+
+class OAuthConfigResponse(BaseModel):
+    """Schema for OAuth2 configuration used in API responses with masked sensitive fields"""
+
+    grant_type: OAuthGrantType
+    token_url: str
+    scope: Optional[str] = None
+    client_id: str
+    client_secret: str = "**********"  # Always masked in responses
+
+    model_config = ConfigDict(from_attributes=True, use_enum_values=True)
+
+
+class OAuthConfigDocsSchema(OAuthConfigSchema, NoValidationSchema):
+    """OAuth2 Config Schema for API Docs"""

fides/api/schemas/connection_configuration/connection_secrets_mongodb.py

@@ -1,4 +1,4 @@
-from typing import ClassVar, List
+from typing import ClassVar, List, Optional
 
 from pydantic import Field
 
@@ -18,7 +18,7 @@ class MongoDBSchema(ConnectionConfigSecretsSchema):
     port: int = Field(
         27017,
         title="Port",
-        description="The network port number on which the server is listening for incoming connections (default: 27017).",
+        description="The network port number on which the server is listening for incoming connections (default: 27017). Port will be ignored if using SRV.",
     )
     username: str = Field(
         title="Username",
@@ -30,10 +30,22 @@ class MongoDBSchema(ConnectionConfigSecretsSchema):
         json_schema_extra={"sensitive": True},
     )
     defaultauthdb: str = Field(
-        title="Default Auth DB",
+        title="Default auth DB",
         description="Used to specify the default authentication database.",
     )
 
+    use_srv: Optional[bool] = Field(
+        False,
+        title="Use SRV",
+        description="Enable SRV record lookup for service discovery (mongodb+srv://). Required for MongoDB Atlas. Enables SSL by default.",
+    )
+
+    ssl_enabled: Optional[bool] = Field(
+        None,
+        title="SSL enabled",
+        description="Enable SSL/TLS encryption. With SRV: defaults to enabled (can override). Without SRV: defaults to disabled.",
+    )
+
     _required_components: ClassVar[List[str]] = [
         "host",
         "username",

fides/api/schemas/connection_configuration/connection_secrets_mssql.py

@@ -39,6 +39,11 @@ class MicrosoftSQLServerSchema(ConnectionConfigSecretsSchema):
         title="Database",
         description="The name of the specific database within the database server that you want to connect to.",
     )
+    read_only_connection: Optional[bool] = Field(
+        default=False,
+        title="Read Only Connection",
+        description="Whether to connect to the database in read-only mode.",
+    )
 
     _required_components: ClassVar[List[str]] = [
         "host",

fides/api/schemas/messaging/messaging.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from datetime import datetime
 from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
@@ -7,15 +8,14 @@ from fideslang.default_taxonomy import DEFAULT_TAXONOMY
 from fideslang.validation import FidesKey
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
-from fides.api.custom_types import PhoneNumber, SafeStr
+from fides.api.custom_types import SafeStr
 from fides.api.schemas import Msg
 from fides.api.schemas.api import BulkResponse, BulkUpdateFailed
-from fides.api.schemas.connection_configuration.connection_secrets_base_aws import (
-    BaseAWSSchema,
-)
+from fides.api.schemas.messaging.shared_schemas import PossibleMessagingSecrets
 from fides.api.schemas.privacy_preference import MinimalPrivacyPreferenceHistorySchema
 from fides.api.schemas.privacy_request import Consent
 from fides.api.schemas.property import MinimalProperty
+from fides.api.schemas.redis_cache import IdentityBase
 
 
 class MessagingMethod(Enum):
@@ -44,6 +44,25 @@ class MessagingServiceType(Enum):
                 return member
         return None
 
+    @property
+    def human_readable(self) -> str:
+        """
+        Human-readable mapping for MessagingServiceType
+        """
+        readable_mapping: Dict[str, str] = {
+            MessagingServiceType.mailgun.value: "Mailgun",
+            MessagingServiceType.twilio_text.value: "Twilio SMS",
+            MessagingServiceType.twilio_email.value: "Twilio Email",
+            MessagingServiceType.mailchimp_transactional.value: "Mailchimp Transactional",
+            MessagingServiceType.aws_ses.value: "AWS SES",
+        }
+        try:
+            return readable_mapping[self.value]
+        except KeyError:
+            raise NotImplementedError(
+                "Add new MessagingServiceType to human_readable mapping"
+            )
+
 
 EMAIL_MESSAGING_SERVICES: Tuple[str, ...] = (
     MessagingServiceType.mailgun.value,
@@ -70,6 +89,7 @@ class MessagingActionType(str, Enum):
     PRIVACY_REQUEST_REVIEW_APPROVE = "privacy_request_review_approve"
     USER_INVITE = "user_invite"
     EXTERNAL_USER_WELCOME = "external_user_welcome"
+    MANUAL_TASK_DIGEST = "manual_task_digest"
     TEST_MESSAGE = "test_message"
 
 
@@ -82,9 +102,17 @@ CONFIGURABLE_MESSAGING_ACTION_TYPES: Tuple[str, ...] = (
     MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION.value,
     MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY.value,
     MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE.value,
+    MessagingActionType.MANUAL_TASK_DIGEST.value,
+    MessagingActionType.EXTERNAL_USER_WELCOME.value,
 )
 
 
+class MessagingTestBodyParams(BaseModel):
+    """Body params required for testing messaging service"""
+
+    to_identity: IdentityBase
+
+
 class ErrorNotificationBodyParams(BaseModel):
     """Body params required for privacy request error notifications."""
 
@@ -194,6 +222,18 @@ class ExternalUserWelcomeBodyParams(BaseModel):
     org_name: str
 
 
+class ManualTaskDigestBodyParams(BaseModel):
+    """Body params required to send manual task digest emails"""
+
+    vendor_contact_name: str
+    organization_name: str
+    portal_url: str
+    imminent_task_count: int
+    upcoming_task_count: int
+    total_task_count: int
+    company_logo_url: Optional[str] = None
+
+
 class FidesopsMessage(
     BaseModel,
 ):
@@ -231,6 +271,7 @@ class FidesopsMessage(
             MessagingActionType.PRIVACY_REQUEST_ERROR_NOTIFICATION: ErrorNotificationBodyParams,
             MessagingActionType.USER_INVITE: UserInviteBodyParams,
             MessagingActionType.EXTERNAL_USER_WELCOME: ExternalUserWelcomeBodyParams,
+            MessagingActionType.MANUAL_TASK_DIGEST: ManualTaskDigestBodyParams,
         }
 
         valid_body_params = valid_body_params_for_action_type.get(
@@ -339,56 +380,6 @@ class MessagingServiceSecrets(Enum):
     AWS_ASSUME_ROLE_ARN = "aws_assume_role_arn"
 
 
-class MessagingServiceSecretsMailchimpTransactional(BaseModel):
-    """The secrets required to connect to Mailchimp Transactional."""
-
-    mailchimp_transactional_api_key: str
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessagingServiceSecretsMailgun(BaseModel):
-    """The secrets required to connect to Mailgun."""
-
-    mailgun_api_key: str
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessagingServiceSecretsTwilioSMS(BaseModel):
-    """The secrets required to connect to Twilio SMS."""
-
-    twilio_account_sid: str
-    twilio_auth_token: str
-    twilio_messaging_service_sid: Optional[str] = None
-    twilio_sender_phone_number: Optional[PhoneNumber] = None
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="before")
-    @classmethod
-    def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        sender_phone = values.get("twilio_sender_phone_number")
-        if not values.get("twilio_messaging_service_sid") and not sender_phone:
-            raise ValueError(
-                "Either the twilio_messaging_service_sid or the twilio_sender_phone_number should be supplied."
-            )
-        return values
-
-
-class MessagingServiceSecretsTwilioEmail(BaseModel):
-    """The secrets required to connect to twilio email."""
-
-    twilio_api_key: str
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessagingServiceSecretsAWS_SES(BaseAWSSchema):
-    """
-    The secrets required to connect to AWS SES.
-    Inherits basic AWS authentication schema.
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-
 class MessagingConfigBase(BaseModel):
     """Base model shared by messaging config related models"""
 
@@ -453,8 +444,9 @@ class MessagingConfigRequestBase(MessagingConfigBase):
 class MessagingConfigRequest(MessagingConfigRequestBase):
     """Messaging Config Request Schema"""
 
-    name: str
+    name: Optional[str] = None
     key: Optional[FidesKey] = None
+    secrets: Optional[PossibleMessagingSecrets] = None
 
 
 class MessagingConfigResponse(MessagingConfigBase):
@@ -462,16 +454,8 @@ class MessagingConfigResponse(MessagingConfigBase):
 
     name: str
     key: FidesKey
-    model_config = ConfigDict(from_attributes=True, use_enum_values=True)
-
-
-SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[
-    MessagingServiceSecretsMailgun,
-    MessagingServiceSecretsTwilioSMS,
-    MessagingServiceSecretsTwilioEmail,
-    MessagingServiceSecretsMailchimpTransactional,
-    MessagingServiceSecretsAWS_SES,
-]
+    last_test_timestamp: Optional[datetime] = None
+    last_test_succeeded: Optional[bool] = None
 
 
 class MessagingConnectionTestStatus(Enum):
@@ -479,7 +463,6 @@ class MessagingConnectionTestStatus(Enum):
 
     succeeded = "succeeded"
     failed = "failed"
-    skipped = "skipped"
 
 
 class TestMessagingStatusMessage(Msg):

fides/api/schemas/messaging/messaging_secrets_docs_only.py

@@ -1,7 +1,7 @@
 from typing import Union
 
 from fides.api.schemas.base_class import NoValidationSchema
-from fides.api.schemas.messaging.messaging import (
+from fides.api.schemas.messaging.shared_schemas import (
     MessagingServiceSecretsAWS_SES,
     MessagingServiceSecretsMailchimpTransactional,
     MessagingServiceSecretsMailgun,

fides/api/schemas/messaging/shared_schemas.py

@@ -0,0 +1,102 @@
+from typing import Any, Dict, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, model_validator
+
+from fides.api.custom_types import PhoneNumber
+from fides.api.schemas.base_class import NoValidationSchema
+from fides.api.schemas.connection_configuration.connection_secrets_base_aws import (
+    BaseAWSSchema,
+)
+
+
+class MessagingServiceSecretsMailgun(BaseModel):
+    """The secrets required to connect to Mailgun."""
+
+    mailgun_api_key: str
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessagingServiceSecretsTwilioSMS(BaseModel):
+    """The secrets required to connect to Twilio SMS."""
+
+    twilio_account_sid: str
+    twilio_auth_token: str
+    twilio_messaging_service_sid: Optional[str] = None
+    twilio_sender_phone_number: Optional[PhoneNumber] = None
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        sender_phone = values.get("twilio_sender_phone_number")
+        if not values.get("twilio_messaging_service_sid") and not sender_phone:
+            raise ValueError(
+                "Either the twilio_messaging_service_sid or the twilio_sender_phone_number should be supplied."
+            )
+        return values
+
+
+class MessagingServiceSecretsTwilioEmail(BaseModel):
+    """The secrets required to connect to twilio email."""
+
+    twilio_api_key: str
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessagingServiceSecretsMailchimpTransactional(BaseModel):
+    """The secrets required to connect to Mailchimp Transactional."""
+
+    mailchimp_transactional_api_key: str
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessagingServiceSecretsMailchimpTransactionalDocs(
+    MessagingServiceSecretsMailchimpTransactional,
+    NoValidationSchema,
+):
+    """The secrets required to connect Mailchimp Transactional, for documentation"""
+
+
+class MessagingSecretsMailgunDocs(MessagingServiceSecretsMailgun, NoValidationSchema):
+    """The secrets required to connect to Mailgun, for documentation"""
+
+
+class MessagingSecretsTwilioSMSDocs(
+    MessagingServiceSecretsTwilioSMS, NoValidationSchema
+):
+    """The secrets required to connect to Twilio sms, for documentation"""
+
+
+class MessagingSecretsTwilioEmailDocs(
+    MessagingServiceSecretsTwilioEmail, NoValidationSchema
+):
+    """The secrets required to connect to Twilio email, for documentation"""
+
+
+class MessagingServiceSecretsAWS_SES(BaseAWSSchema):
+    """The secrets required to connect to AWS SES."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessagingServiceSecretsAWS_SESDocs(
+    MessagingServiceSecretsAWS_SES, NoValidationSchema
+):
+    """The secrets required to connect to AWS SES, for documentation"""
+
+
+SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[
+    MessagingServiceSecretsMailgun,
+    MessagingServiceSecretsTwilioSMS,
+    MessagingServiceSecretsTwilioEmail,
+    MessagingServiceSecretsMailchimpTransactional,
+    MessagingServiceSecretsAWS_SES,
+]
+
+PossibleMessagingSecrets = Union[
+    MessagingSecretsMailgunDocs,
+    MessagingSecretsTwilioSMSDocs,
+    MessagingSecretsTwilioEmailDocs,
+    MessagingServiceSecretsMailchimpTransactionalDocs,
+    MessagingServiceSecretsAWS_SESDocs,
+]

fides/api/schemas/oauth.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional
+from typing import Dict, Literal, Optional
 
 from fastapi import Form
 from fastapi.openapi.models import OAuthFlows
@@ -14,6 +14,7 @@ class AccessToken(BaseModel):
     """A wrapper for the access_code returned upon successful authentication"""
 
     access_token: str
+    token_type: Literal["Bearer"] = "Bearer"
 
 
 # NOTE: Adapted from

fides/api/schemas/privacy_center_config.py

@@ -1,7 +1,14 @@
 from abc import ABC
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
 
-from pydantic import ConfigDict, Field, field_validator, model_validator
+from pydantic import (
+    ConfigDict,
+    Discriminator,
+    Field,
+    Tag,
+    field_validator,
+    model_validator,
+)
 
 from fides.api.models.location_regulation_selections import PrivacyNoticeRegion
 from fides.api.schemas.base_class import FidesSchema
@@ -86,9 +93,26 @@ class LocationCustomPrivacyRequestField(BaseCustomPrivacyRequestField):
         return values
 
 
-# Create a simple union type - Pydantic will use the field_type to determine which model to use
-CustomPrivacyRequestFieldUnion = Union[
-    LocationCustomPrivacyRequestField, CustomPrivacyRequestField
+# Create a discriminated union type using the field_type to properly distinguish between types
+def get_field_type_discriminator(v: Any) -> str:
+    """Discriminator function for CustomPrivacyRequestFieldUnion"""
+    if isinstance(v, dict):
+        field_type = v.get("field_type")
+    else:
+        # For model instances, get field_type attribute
+        field_type = getattr(v, "field_type", None)
+
+    if field_type == "location":
+        return "location"
+    return "custom"
+
+
+CustomPrivacyRequestFieldUnion = Annotated[
+    Union[
+        Annotated[LocationCustomPrivacyRequestField, Tag("location")],
+        Annotated[CustomPrivacyRequestField, Tag("custom")],
+    ],
+    Discriminator(get_field_type_discriminator),
 ]
 
 

fides/api/schemas/privacy_request.py

@@ -1,9 +1,10 @@
 from datetime import datetime
 from enum import Enum as EnumType
 from typing import Any, Dict, List, Optional, Type, Union
+from uuid import UUID
 
 from fideslang.validation import FidesKey
-from pydantic import ConfigDict, Field, field_serializer, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
 
 from fides.api.custom_types import SafeStr
 from fides.api.graph.config import CollectionAddress
@@ -13,8 +14,12 @@ from fides.api.schemas.api import BulkResponse, BulkUpdateFailed
 from fides.api.schemas.base_class import FidesSchema
 from fides.api.schemas.policy import ActionType, CurrentStep
 from fides.api.schemas.policy import PolicyResponse as PolicySchema
-from fides.api.schemas.redis_cache import CustomPrivacyRequestField, Identity
-from fides.api.schemas.user import PrivacyRequestReviewer
+from fides.api.schemas.redis_cache import (
+    CustomPrivacyRequestField,
+    Identity,
+    MultiValue,
+)
+from fides.api.schemas.user import PrivacyRequestUser
 from fides.api.util.collection_util import Row
 from fides.api.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key
 from fides.api.util.enums import ColumnSort
@@ -306,6 +311,23 @@ class PrivacyRequestStatus(str, EnumType):
     requires_manual_finalization = "requires_manual_finalization"
     canceled = "canceled"
     error = "error"
+    duplicate = "duplicate"  # Request identified as duplicate of another request
+
+
+class IdentityValue(BaseModel):
+    """Represents an identity value with a label in API responses.
+
+    The value field accepts MultiValue types which match what LabeledIdentity supports:
+    - int
+    - str
+    - List[Union[int, str]]
+
+    This allows the schema to accept list values that were previously causing
+    validation errors.
+    """
+
+    label: str
+    value: Optional[MultiValue] = None
 
 
 class PrivacyRequestResponse(FidesSchema):
@@ -316,7 +338,9 @@ class PrivacyRequestResponse(FidesSchema):
     started_processing_at: Optional[datetime] = None
     reviewed_at: Optional[datetime] = None
     reviewed_by: Optional[str] = None
-    reviewer: Optional[PrivacyRequestReviewer] = None
+    submitted_by: Optional[str] = None
+    reviewer: Optional[PrivacyRequestUser] = None
+    submitter: Optional[PrivacyRequestUser] = None
     finished_processing_at: Optional[datetime] = None
     identity_verified_at: Optional[datetime] = None
     paused_at: Optional[datetime] = None
@@ -326,7 +350,7 @@ class PrivacyRequestResponse(FidesSchema):
     # as it is an API response field, and we don't want to reveal any more
     # about our PII structure than is explicitly stored in the cache on request
     # creation.
-    identity: Optional[Dict[str, Union[Optional[str], Dict[str, Any]]]] = None
+    identity: Optional[Dict[str, Union[Optional[str], IdentityValue]]] = None
     custom_privacy_request_fields: Optional[Dict[str, Any]] = None
     policy: PolicySchema
     action_required_details: Optional[CheckpointActionRequiredDetails] = None
@@ -340,6 +364,8 @@ class PrivacyRequestResponse(FidesSchema):
     deleted_by: Optional[str] = None
     finalized_at: Optional[datetime] = None
     finalized_by: Optional[str] = None
+    duplicate_request_group_id: Optional[UUID] = None
+
     model_config = ConfigDict(from_attributes=True, use_enum_values=True)
 
 
@@ -353,10 +379,15 @@ class PrivacyRequestVerboseResponse(PrivacyRequestResponse):
     model_config = ConfigDict(populate_by_name=True)
 
 
+# Batch size for bulk privacy request operations. Used for request validation and
+# automatic batching to avoid memory issues with large request lists.
+BULK_PRIVACY_REQUEST_BATCH_SIZE = 50
+
+
 class ReviewPrivacyRequestIds(FidesSchema):
     """Pass in a list of privacy request ids"""
 
-    request_ids: List[str] = Field(..., max_length=50)
+    request_ids: List[str] = Field(..., max_length=BULK_PRIVACY_REQUEST_BATCH_SIZE)
 
 
 class DenyPrivacyRequests(ReviewPrivacyRequestIds):
@@ -365,6 +396,12 @@ class DenyPrivacyRequests(ReviewPrivacyRequestIds):
     reason: Optional[SafeStr] = None
 
 
+class CancelPrivacyRequests(ReviewPrivacyRequestIds):
+    """Pass in a list of privacy request ids and cancellation reason"""
+
+    reason: Optional[SafeStr] = None
+
+
 class BulkPostPrivacyRequests(BulkResponse):
     """Schema with mixed success/failure responses for Bulk Create of PrivacyRequest responses."""
 
@@ -453,7 +490,7 @@ class PrivacyRequestFilter(FidesSchema):
     errored_gt: Optional[datetime] = None
     external_id: Optional[str] = None
     location: Optional[str] = None
-    action_type: Optional[ActionType] = None
+    action_type: Optional[Union[ActionType, List[ActionType]]] = None
     verbose: Optional[bool] = False
     include_identities: Optional[bool] = False
     include_custom_privacy_request_fields: Optional[bool] = False