ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/db/seed.py

@@ -2,6 +2,7 @@
 Provides functions that seed the application with data.
 """
 
+import os
 from typing import Dict, List, Optional
 
 from fideslang.default_taxonomy import DEFAULT_TAXONOMY
@@ -17,6 +18,7 @@ from fides.api.common_exceptions import KeyOrNameAlreadyExists
 from fides.api.db.base_class import FidesBase
 from fides.api.db.ctl_session import sync_session
 from fides.api.db.system import upsert_system
+from fides.api.models.application_config import ApplicationConfig
 from fides.api.models.client import ClientDetail
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.datasetconfig import DatasetConfig
@@ -37,7 +39,16 @@ from fides.api.schemas.connection_configuration.saas_config_template_values impo
     SaasConnectionTemplateValues,
 )
 from fides.api.schemas.dataset import DatasetConfigCtlDataset
+from fides.api.schemas.messaging.messaging import (
+    MessagingConfigRequest,
+    MessagingServiceDetailsMailgun,
+    MessagingServiceType,
+)
+from fides.api.schemas.messaging.shared_schemas import MessagingSecretsMailgunDocs
 from fides.api.schemas.policy import ActionType, DrpAction
+from fides.api.service.messaging.messaging_crud_service import (
+    create_or_update_messaging_config,
+)
 from fides.api.util.connection_util import patch_connection_configs
 from fides.api.util.data_category import get_user_data_categories
 from fides.api.util.errors import AlreadyExistsError, QueryError
@@ -377,7 +388,7 @@ async def load_samples(async_session: AsyncSession) -> None:
                     )  # not supported by this API!
                     instantiate_connection_from_template(
                         db=db_session,
-                        saas_connector_type=connection.saas_connector_type,
+                        connector_template_type=connection.saas_connector_type,
                         template_values=SaasConnectionTemplateValues.model_validate(
                             saas_template_data
                         ),
@@ -486,6 +497,29 @@ async def load_samples(async_session: AsyncSession) -> None:
     except QueryError:  # pragma: no cover
         pass  # The upsert_resources function will log any error
 
+    if os.getenv("MAILGUN_DOMAIN") and os.getenv("MAILGUN_API_KEY"):
+        log.info("Loading Mailgun messaging config")
+        with sync_session() as db:
+            messaging_config = create_or_update_messaging_config(
+                db=db,
+                config=MessagingConfigRequest(
+                    key="my_mailgun_config",
+                    name="Mailgun",
+                    service_type=MessagingServiceType.mailgun,
+                    details=MessagingServiceDetailsMailgun(
+                        domain=os.getenv("MAILGUN_DOMAIN")
+                    ),
+                ),
+            )
+            messaging_config.set_secrets(
+                db=db,
+                messaging_secrets=MessagingSecretsMailgunDocs(
+                    mailgun_api_key=os.getenv("MAILGUN_API_KEY")
+                ),
+            )
+            CONFIG.notifications.notification_service_type = "mailgun"
+            ApplicationConfig.update_config_set(db, CONFIG)
+
 
 def load_default_organization(db: Session) -> None:
     """

fides/api/email_templates/get_email_template.py

@@ -9,6 +9,7 @@ from fides.api.email_templates.template_names import (
     CONSENT_REQUEST_VERIFICATION_TEMPLATE,
     EMAIL_ERASURE_REQUEST_FULFILLMENT,
     EXTERNAL_USER_WELCOME,
+    MANUAL_TASK_DIGEST,
     PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE,
     PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE,
     PRIVACY_REQUEST_ERROR_NOTIFICATION_TEMPLATE,
@@ -59,6 +60,8 @@ def get_email_template(  # pylint: disable=too-many-return-statements, too-many-
         return template_env.get_template(USER_INVITE)
     if action_type == MessagingActionType.EXTERNAL_USER_WELCOME:
         return template_env.get_template(EXTERNAL_USER_WELCOME)
+    if action_type == MessagingActionType.MANUAL_TASK_DIGEST:
+        return template_env.get_template(MANUAL_TASK_DIGEST)
 
     logger.error("No corresponding template linked to the {}", action_type)
     raise EmailTemplateUnhandledActionType(

fides/api/email_templates/template_names.py

@@ -11,3 +11,4 @@ PRIVACY_REQUEST_REVIEW_APPROVE_TEMPLATE = "privacy_request_review_approve.html"
 TEST_MESSAGE_TEMPLATE = "test_message.html"
 USER_INVITE = "user_invite.html"
 EXTERNAL_USER_WELCOME = "external_user_welcome.html"
+MANUAL_TASK_DIGEST = "manual_task_digest.html"

fides/api/email_templates/templates/external_user_welcome.html

@@ -8,16 +8,20 @@
 <main>
     <p>Hello {{ display_name }},</p>
 
-    <p>An account has been created for you by the privacy team at {{ org_name }} so that you can assist them in responding to data subject requests, as required by the GDPR and many US state laws, for users that have data in your systems.</p>
+    {% if custom_intro %}
+        <div>{{ custom_intro | safe }}</div>
+    {% else %}
+        <p>An account has been created for you by the privacy team at {{ org_name }} so that you can assist them in responding to data subject requests, as required by the GDPR and many US state laws, for users that have data in your systems.</p>
 
-    <p>A portal is available to view and take action on these requests at: <a href="{{ portal_link }}">{{ portal_link }}</a>. Through this portal you can upload or provide user data for users who have requested a copy of their information and attest that users have been removed from your system in the case of an erasure request.</p>
+        <p>A portal is available to view and take action on these requests at: <a href="{{ portal_link }}">{{ portal_link }}</a>. Through this portal you can upload or provide user data for users who have requested a copy of their information and attest that users have been removed from your system in the case of an erasure request.</p>
 
-    <p>The provided link is valid for 30 days and you will be asked to verify your email address each time you access the portal.</p>
+        <p>The provided link is valid for 30 days and you will be asked to verify your email address each time you access the portal.</p>
 
-    <p>If you have any questions, please reach out first to your contacts at: {{ org_name }}</p>
+        <p>If you have any questions, please reach out first to your contacts at: {{ org_name }}</p>
+    {% endif %}
 
     <p>Regards,<br/>
     {{ org_name }} Privacy Team</p>
 </main>
 </body>
-</html> 
+</html>

fides/api/email_templates/templates/manual_task_digest.html

@@ -0,0 +1,316 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Weekly DSR Summary</title>
+    <style>
+        /* Reset styles for email clients */
+        body, table, td, p, a, li, blockquote {
+            -webkit-text-size-adjust: 100%;
+            -ms-text-size-adjust: 100%;
+        }
+
+        table, td {
+            mso-table-lspace: 0pt;
+            mso-table-rspace: 0pt;
+        }
+
+        img {
+            -ms-interpolation-mode: bicubic;
+            border: 0;
+            height: auto;
+            line-height: 100%;
+            outline: none;
+            text-decoration: none;
+        }
+
+        /* Main styles */
+        body {
+            margin: 0;
+            padding: 0;
+            width: 100% !important;
+            min-width: 100%;
+            height: 100%;
+            background-color: #f8fafc;
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+            font-size: 16px;
+            line-height: 1.6;
+            color: #334155;
+        }
+
+        .email-container {
+            max-width: 600px;
+            margin: 0 auto;
+            background-color: #ffffff;
+            border-radius: 8px;
+            box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1);
+            overflow: hidden;
+        }
+
+        .logo-section {
+            background-color: #ffffff;
+            padding: 24px;
+            text-align: center;
+            border-bottom: 1px solid #e5e5e5;
+        }
+
+        .logo-container {
+            margin-bottom: 0;
+        }
+
+        .header {
+            background-color: #000000;
+            padding: 32px 24px;
+            text-align: center;
+        }
+
+        .logo {
+            max-width: 200px;
+            max-height: 60px;
+            width: auto;
+            height: auto;
+            display: block;
+            margin: 0 auto;
+        }
+
+        .logo-placeholder {
+            width: 200px;
+            height: 60px;
+            background-color: #333333;
+            border: 2px dashed #666666;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            margin: 0 auto;
+            color: #999999;
+            font-size: 14px;
+            font-weight: 500;
+            text-align: center;
+            border-radius: 4px;
+        }
+
+        .header h1 {
+            color: #ffffff;
+            font-size: 24px;
+            font-weight: 600;
+            margin: 0;
+            letter-spacing: -0.025em;
+        }
+
+        .content {
+            padding: 32px 24px;
+        }
+
+        .greeting {
+            font-size: 18px;
+            color: #000000;
+            margin-bottom: 24px;
+            font-weight: 500;
+        }
+
+        .intro-text {
+            margin-bottom: 32px;
+            color: #333333;
+            line-height: 1.7;
+        }
+
+        .section {
+            margin-bottom: 32px;
+        }
+
+        .section-title {
+            font-size: 20px;
+            font-weight: 600;
+            color: #000000;
+            margin-bottom: 16px;
+            padding-bottom: 8px;
+            border-bottom: 2px solid #e5e5e5;
+        }
+
+        .task-count {
+            background-color: #f5f5f5;
+            border-left: 4px solid #000000;
+            padding: 16px 20px;
+            margin-bottom: 20px;
+            border-radius: 0 6px 6px 0;
+        }
+
+        .task-count-text {
+            font-size: 18px;
+            font-weight: 600;
+            color: #000000;
+            margin: 0;
+        }
+
+        .cta-button {
+            display: inline-block;
+            background-color: #000000;
+            color: #ffffff !important;
+            text-decoration: none;
+            padding: 12px 24px;
+            border-radius: 6px;
+            font-weight: 600;
+            font-size: 16px;
+            text-align: center;
+            transition: all 0.2s ease;
+            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+        }
+
+        .cta-button:hover {
+            background-color: #333333;
+            box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
+            transform: translateY(-1px);
+        }
+
+        .footer {
+            background-color: #f5f5f5;
+            padding: 24px;
+            text-align: center;
+            border-top: 1px solid #e5e5e5;
+        }
+
+        .footer-text {
+            color: #666666;
+            font-size: 14px;
+            line-height: 1.6;
+            margin: 0;
+        }
+
+        .signature {
+            margin-top: 24px;
+            color: #333333;
+            font-weight: 500;
+        }
+
+        /* Responsive styles */
+        @media only screen and (max-width: 600px) {
+            .email-container {
+                margin: 0;
+                border-radius: 0;
+            }
+
+            .header, .content, .footer {
+                padding: 24px 16px;
+            }
+
+            .header h1 {
+                font-size: 20px;
+            }
+
+            .greeting {
+                font-size: 16px;
+            }
+
+            .section-title {
+                font-size: 18px;
+            }
+
+            .cta-button {
+                width: 100%;
+                box-sizing: border-box;
+            }
+        }
+
+        /* Dark mode support */
+        @media (prefers-color-scheme: dark) {
+            body {
+                background-color: #000000;
+            }
+
+            .email-container {
+                background-color: #1a1a1a;
+            }
+
+            .greeting, .section-title {
+                color: #ffffff;
+            }
+
+            .intro-text, .footer-text, .signature {
+                color: #cccccc;
+            }
+
+            .task-count {
+                background-color: #333333;
+                border-left-color: #ffffff;
+            }
+
+            .task-count-text {
+                color: #ffffff;
+            }
+
+            .footer {
+                background-color: #333333;
+                border-top-color: #666666;
+            }
+
+            .logo-section {
+                background-color: #1a1a1a;
+                border-bottom-color: #666666;
+            }
+
+            .logo-placeholder {
+                background-color: #555555;
+                border-color: #888888;
+                color: #cccccc;
+            }
+        }
+    </style>
+</head>
+<body>
+    <div style="padding: 20px 0;">
+        <div class="email-container">
+            <!-- Logo Section -->
+            <div class="logo-section">
+                <div class="logo-container">
+                    {% if company_logo_url %}
+                        <img src="{{ company_logo_url }}" alt="{{ organization_name }} Logo" class="logo">
+                    {% else %}
+                        <div class="logo-placeholder">{{ organization_name }}</div>
+                    {% endif %}
+                </div>
+            </div>
+
+            <!-- Main Content -->
+            <div class="content">
+                <div class="greeting">Hi {{ vendor_contact_name }},</div>
+
+                <div class="intro-text">
+                    {{ intro_text | default("This is your weekly summary of open data subject requests tasks from " + organization_name + " that require your attention. Please review the tasks listed below in our <a href='" + portal_url + "' style='color: #000000; text-decoration: underline; font-weight: 600;'>Privacy Center</a> to ensure timely completion.") | safe }}
+                </div>
+
+                <!-- Tasks Due Soon Section -->
+                <div class="section">
+                    <div class="section-title">Tasks Due Soon</div>
+
+                    <div class="task-count">
+                        <p class="task-count-text">You have {{ total_task_count }} request{{ 's' if total_task_count != 1 else '' }} coming due, including {{ imminent_task_count }} within the next 7 days and {{ upcoming_task_count }} due in the next period.</p>
+                    </div>
+
+                    <div style="text-align: center;">
+                        <a href="{{ portal_url }}" class="cta-button">View All Tasks</a>
+                    </div>
+                </div>
+
+                <!-- Closing -->
+                <div style="margin-top: 32px;">
+                    <p style="color: #333333; margin-bottom: 24px;">
+                        {{ closing_text | default("Thank you for your partnership in processing these requests efficiently. If you have any questions or issues, please reach out to your contacts at " + organization_name + ".") | safe }}
+                    </p>
+
+                    <div class="signature">
+                        {{ signature | default("Best regards,<br>The " + organization_name + " Privacy Team") | safe }}
+                    </div>
+                </div>
+            </div>
+
+            <!-- Footer -->
+            <div class="footer">
+                <p class="footer-text">
+                    This email was sent from your Privacy Center. Please do not reply to this email.
+                </p>
+            </div>
+        </div>
+    </div>
+</body>
+</html>

fides/api/main.py

@@ -47,6 +47,7 @@ from fides.api.service.privacy_request.email_batch_service import (
 from fides.api.service.privacy_request.request_service import (
     initiate_interrupted_task_requeue_poll,
     initiate_poll_for_exited_privacy_request_tasks,
+    initiate_polling_task_requeue,
     initiate_scheduled_dsr_data_removal,
 )
 
@@ -102,6 +103,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_poll_for_exited_privacy_request_tasks()
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
+    initiate_polling_task_requeue()
     initiate_bcrypt_migration_task()
     initiate_post_upgrade_index_creation()
 

fides/api/migrations/post_upgrade_index_creation.py

@@ -33,6 +33,9 @@ This means that after the constraint is created, the index will have the same na
 """
 
 POST_UPGRADE_INDEX_CREATION = "post_upgrade_index_creation"
+# This is the threshold for the number of rows in a table to create the index during application startup
+# The value follows the pattern established in migrations for the tables in the TABLE_OBJECT_MAP
+INDEX_ROW_COUNT_THRESHOLD = 1000000
 
 TABLE_OBJECT_MAP: Dict[str, List[Dict[str, str]]] = {
     "currentprivacypreferencev2": [
@@ -93,6 +96,13 @@ TABLE_OBJECT_MAP: Dict[str, List[Dict[str, str]]] = {
             "type": "index",
         },
     ],
+    "providedidentity": [
+        {
+            "name": "ix_providedidentity_privacy_request_id",
+            "statement": "CREATE INDEX CONCURRENTLY ix_providedidentity_privacy_request_id ON providedidentity (privacy_request_id)",
+            "type": "index",
+        },
+    ],
     "servednoticehistory": [
         {
             "name": "ix_servednoticehistory_hashed_external_id",
@@ -153,6 +163,16 @@ TABLE_OBJECT_MAP: Dict[str, List[Dict[str, str]]] = {
             "statement": "CREATE INDEX CONCURRENTLY ix_stagedresource_system_vendor_consent ON stagedresource (system_id, vendor_id, (meta->>'consent_aggregated'))",
             "type": "index",
         },
+        {
+            "name": "idx_stagedresource_user_categories_gin",
+            "statement": "CREATE INDEX CONCURRENTLY idx_stagedresource_user_categories_gin ON stagedresource USING GIN (user_assigned_data_categories)",
+            "type": "index",
+        },
+        {
+            "name": "idx_stagedresource_classifications_gin",
+            "statement": "CREATE INDEX CONCURRENTLY idx_stagedresource_classifications_gin ON stagedresource USING GIN (classifications)",
+            "type": "index",
+        },
     ],
 }
 

fides/api/models/asset.py

@@ -17,7 +17,7 @@ from sqlalchemy import (
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.mutable import MutableDict
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, selectinload
 
 from fides.api.db.base_class import Base
 from fides.api.db.util import EnumColumn
@@ -29,9 +29,12 @@ class ConsentStatus(str, Enum):
     Consent status of the asset
     """
 
+    cmp_error = "cmp_error"
+    pre_consent = "pre_consent"
     with_consent = "with_consent"
     without_consent = "without_consent"
     exempt = "exempt"
+    not_applicable = "not_applicable"
     unknown = "unknown"
 
 
@@ -44,6 +47,7 @@ class Asset(Base):
     name = Column(String, index=True, nullable=False)
     asset_type = Column(String, index=True, nullable=False)
     domain = Column(String, index=True)
+    duration = Column(String, nullable=True)
     parent = Column(ARRAY(String), server_default="{}", nullable=False)
     parent_domain = Column(String)
     locations = Column(ARRAY(String), server_default="{}", nullable=False)
@@ -187,5 +191,17 @@ class Asset(Base):
                 .where(System.fides_key == system_fides_key)
             )
 
+        # Explicitly eager load the `system` relationship to make this query's
+        # performance predictable and prevent N+1 issues.
+        query = query.options(selectinload(cls.system))  # type: ignore[attr-defined]
         result = await async_session.execute(query)
         return result.scalars().all()
+
+    # Expose related System attributes for API serialization convenience
+    @property
+    def system_name(self) -> Optional[str]:
+        return self.system.name if self.system else None
+
+    @property
+    def system_fides_key(self) -> Optional[str]:
+        return self.system.fides_key if self.system else None

fides/api/models/attachment.py

@@ -52,6 +52,7 @@ class AttachmentReferenceType(str, EnumType):
     privacy_request = "privacy_request"
     comment = "comment"
     manual_task_submission = "manual_task_submission"
+    request_task = "request_task"
 
 
 class AttachmentReference(Base):