ethyca-fides 2.65.0rc10__py2.py3-none-any.whl → 2.65.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.65.0rc10.dist-info → ethyca_fides-2.65.1.dist-info}/METADATA +1 -1
- {ethyca_fides-2.65.0rc10.dist-info → ethyca_fides-2.65.1.dist-info}/RECORD +108 -108
- fides/_version.py +3 -3
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +24 -1
- fides/api/oauth/roles.py +2 -0
- fides/api/schemas/api.py +6 -0
- fides/api/service/connectors/base_email_connector.py +1 -1
- fides/api/service/connectors/base_erasure_email_connector.py +39 -2
- fides/api/service/connectors/consent_email_connector.py +1 -1
- fides/api/service/connectors/dynamic_erasure_email_connector.py +14 -48
- fides/api/service/connectors/erasure_email_connector.py +14 -8
- fides/api/service/privacy_request/email_batch_service.py +68 -52
- fides/api/tasks/scheduled/scheduler.py +5 -0
- fides/api/util/lock.py +3 -0
- fides/common/api/scope_registry.py +4 -0
- fides/common/api/v1/urn_registry.py +3 -0
- fides/config/execution_settings.py +8 -0
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3988-e8e0424b117d813b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-fefefb8600b6fc6a.js → _app-be66c01c2b1de442.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-93f0fed887d9e98e.js +1 -0
- fides/ui-build/static/admin/_next/static/css/{5f8af79f94072e0f.css → d522df24a125b46f.css} +1 -1
- fides/ui-build/static/admin/_next/static/e4X4A9FU3w3buLKgvkCKW/_buildManifest.js +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/S2fFUpl9ABwA2MWvCfWz7/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3988-046b3a2866b5cb99.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-b1772281866cb827.js +0 -1
- {ethyca_fides-2.65.0rc10.dist-info → ethyca_fides-2.65.1.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.65.0rc10.dist-info → ethyca_fides-2.65.1.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.65.0rc10.dist-info → ethyca_fides-2.65.1.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.65.0rc10.dist-info → ethyca_fides-2.65.1.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{S2fFUpl9ABwA2MWvCfWz7 → e4X4A9FU3w3buLKgvkCKW}/_ssgManifest.js +0 -0
|
@@ -22,7 +22,6 @@ from fides.api.schemas.connection_configuration.connection_secrets_dynamic_erasu
|
|
|
22
22
|
DynamicErasureEmailSchema,
|
|
23
23
|
)
|
|
24
24
|
from fides.api.schemas.policy import ActionType
|
|
25
|
-
from fides.api.schemas.privacy_request import PrivacyRequestStatus
|
|
26
25
|
from fides.api.service.connectors.base_connector import BaseConnector
|
|
27
26
|
from fides.api.service.connectors.base_erasure_email_connector import (
|
|
28
27
|
BaseErasureEmailConnector,
|
|
@@ -67,9 +66,11 @@ class DynamicErasureEmailConnector(BaseErasureEmailConnector):
|
|
|
67
66
|
def get_config(self, configuration: ConnectionConfig) -> DynamicErasureEmailSchema:
|
|
68
67
|
return DynamicErasureEmailSchema(**configuration.secrets or {})
|
|
69
68
|
|
|
70
|
-
def batch_email_send(self, privacy_requests: Query) -> None:
|
|
69
|
+
def batch_email_send(self, privacy_requests: Query, batch_id: str) -> None:
|
|
71
70
|
logger.debug(
|
|
72
|
-
"Starting
|
|
71
|
+
"Starting batch erasure email {} for connector: {} ...",
|
|
72
|
+
batch_id,
|
|
73
|
+
self.configuration.key,
|
|
73
74
|
)
|
|
74
75
|
|
|
75
76
|
db: Session = Session.object_session(self.configuration)
|
|
@@ -133,14 +134,16 @@ class DynamicErasureEmailConnector(BaseErasureEmailConnector):
|
|
|
133
134
|
|
|
134
135
|
if not batched_identities:
|
|
135
136
|
logger.info(
|
|
136
|
-
"Skipping erasure email send for connector
|
|
137
|
+
"Skipping erasure email send for connector '{}' and batch '{}'. "
|
|
137
138
|
"No corresponding user identities or email addresses found for pending privacy requests.",
|
|
138
139
|
self.configuration.key,
|
|
140
|
+
batch_id,
|
|
139
141
|
)
|
|
140
142
|
return
|
|
141
143
|
|
|
142
144
|
logger.info(
|
|
143
|
-
"Sending
|
|
145
|
+
"Sending batch erasure email {} for connector {}...",
|
|
146
|
+
batch_id,
|
|
144
147
|
self.configuration.key,
|
|
145
148
|
)
|
|
146
149
|
|
|
@@ -155,15 +158,14 @@ class DynamicErasureEmailConnector(BaseErasureEmailConnector):
|
|
|
155
158
|
test_mode=False,
|
|
156
159
|
)
|
|
157
160
|
except MessageDispatchException as exc:
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
self.configuration.key,
|
|
161
|
-
exc,
|
|
162
|
-
)
|
|
161
|
+
message = f"Dynamic batch erasure email {batch_id} for connector {self.configuration.key} failed with exception {exc}"
|
|
162
|
+
logger.error(message)
|
|
163
163
|
self.error_all_privacy_requests(
|
|
164
164
|
db,
|
|
165
|
-
privacy_requests
|
|
166
|
-
|
|
165
|
+
privacy_requests.filter(
|
|
166
|
+
PrivacyRequest.id.notin_(skipped_privacy_requests)
|
|
167
|
+
),
|
|
168
|
+
message,
|
|
167
169
|
)
|
|
168
170
|
raise exc
|
|
169
171
|
|
|
@@ -470,42 +472,6 @@ class DynamicErasureEmailConnector(BaseErasureEmailConnector):
|
|
|
470
472
|
|
|
471
473
|
return email, vendor
|
|
472
474
|
|
|
473
|
-
def error_all_privacy_requests(
|
|
474
|
-
self, db: Session, privacy_requests: Query, failure_reason: str
|
|
475
|
-
) -> None:
|
|
476
|
-
"""
|
|
477
|
-
Creates an ExecutionLog with status error for each privacy request in the batch, and sets the
|
|
478
|
-
privacy request status to error.
|
|
479
|
-
"""
|
|
480
|
-
for privacy_request in privacy_requests:
|
|
481
|
-
self.error_privacy_request(db, privacy_request, failure_reason)
|
|
482
|
-
|
|
483
|
-
def error_privacy_request(
|
|
484
|
-
self,
|
|
485
|
-
db: Session,
|
|
486
|
-
privacy_request: PrivacyRequest,
|
|
487
|
-
failure_reason: Optional[str],
|
|
488
|
-
) -> None:
|
|
489
|
-
"""
|
|
490
|
-
Creates an ExecutionLog with status error for the privacy request, using the failure_reason
|
|
491
|
-
as the message, and sets the privacy request status to error.
|
|
492
|
-
"""
|
|
493
|
-
ExecutionLog.create(
|
|
494
|
-
db=db,
|
|
495
|
-
data={
|
|
496
|
-
"connection_key": self.configuration.key,
|
|
497
|
-
"dataset_name": self.configuration.name_or_key,
|
|
498
|
-
"collection_name": self.configuration.name_or_key,
|
|
499
|
-
"privacy_request_id": privacy_request.id,
|
|
500
|
-
"action_type": ActionType.erasure,
|
|
501
|
-
"status": ExecutionLogStatus.error,
|
|
502
|
-
"message": failure_reason
|
|
503
|
-
or "An error occurred when trying to send the erasure email",
|
|
504
|
-
},
|
|
505
|
-
)
|
|
506
|
-
privacy_request.status = PrivacyRequestStatus.error
|
|
507
|
-
privacy_request.save(db)
|
|
508
|
-
|
|
509
475
|
def test_connection(self) -> Optional[ConnectionTestStatus]:
|
|
510
476
|
"""
|
|
511
477
|
Sends an email to the "test_email" configured, just to establish that the email workflow is working.
|
|
@@ -9,7 +9,7 @@ from fides.api.models.connectionconfig import (
|
|
|
9
9
|
ConnectionTestStatus,
|
|
10
10
|
ConnectionType,
|
|
11
11
|
)
|
|
12
|
-
from fides.api.models.privacy_request import ExecutionLog
|
|
12
|
+
from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest
|
|
13
13
|
from fides.api.models.worker_task import ExecutionLogStatus
|
|
14
14
|
from fides.api.schemas.connection_configuration import EmailSchema
|
|
15
15
|
from fides.api.schemas.policy import ActionType
|
|
@@ -67,7 +67,7 @@ class GenericErasureEmailConnector(BaseErasureEmailConnector):
|
|
|
67
67
|
return ConnectionTestStatus.failed
|
|
68
68
|
return ConnectionTestStatus.succeeded
|
|
69
69
|
|
|
70
|
-
def batch_email_send(self, privacy_requests: Query) -> None:
|
|
70
|
+
def batch_email_send(self, privacy_requests: Query, batch_id: str) -> None:
|
|
71
71
|
skipped_privacy_requests: List[str] = []
|
|
72
72
|
batched_identities: List[str] = []
|
|
73
73
|
db = Session.object_session(self.configuration)
|
|
@@ -85,14 +85,16 @@ class GenericErasureEmailConnector(BaseErasureEmailConnector):
|
|
|
85
85
|
|
|
86
86
|
if not batched_identities:
|
|
87
87
|
logger.info(
|
|
88
|
-
"Skipping erasure email send for connector
|
|
88
|
+
"Skipping erasure email send for connector '{}' and batch '{}'. "
|
|
89
89
|
"No corresponding user identities found for pending privacy requests.",
|
|
90
90
|
self.configuration.key,
|
|
91
|
+
batch_id,
|
|
91
92
|
)
|
|
92
93
|
return
|
|
93
94
|
|
|
94
95
|
logger.info(
|
|
95
|
-
"Sending
|
|
96
|
+
"Sending batch erasure email {} for connector {}...",
|
|
97
|
+
batch_id,
|
|
96
98
|
self.configuration.key,
|
|
97
99
|
)
|
|
98
100
|
|
|
@@ -105,10 +107,14 @@ class GenericErasureEmailConnector(BaseErasureEmailConnector):
|
|
|
105
107
|
test_mode=False,
|
|
106
108
|
)
|
|
107
109
|
except MessageDispatchException as exc:
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
110
|
+
message = f"Batch erasure email {batch_id} for connector {self.configuration.key} failed with exception {exc}"
|
|
111
|
+
logger.error(message)
|
|
112
|
+
self.error_all_privacy_requests(
|
|
113
|
+
db,
|
|
114
|
+
privacy_requests.filter(
|
|
115
|
+
PrivacyRequest.id.notin_(skipped_privacy_requests)
|
|
116
|
+
),
|
|
117
|
+
message,
|
|
112
118
|
)
|
|
113
119
|
raise
|
|
114
120
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import uuid
|
|
1
2
|
from enum import Enum
|
|
2
3
|
|
|
3
4
|
from loguru import logger
|
|
@@ -14,12 +15,15 @@ from fides.api.service.privacy_request.request_runner_service import (
|
|
|
14
15
|
get_erasure_email_connection_configs,
|
|
15
16
|
)
|
|
16
17
|
from fides.api.tasks import DatabaseTask, celery_app
|
|
17
|
-
from fides.api.tasks.scheduled.scheduler import scheduler
|
|
18
|
+
from fides.api.tasks.scheduled.scheduler import create_cron_trigger, scheduler
|
|
19
|
+
from fides.api.util.lock import redis_lock
|
|
18
20
|
from fides.config import get_config
|
|
19
21
|
from fides.service.privacy_request.privacy_request_service import queue_privacy_request
|
|
20
22
|
|
|
21
23
|
CONFIG = get_config()
|
|
22
24
|
BATCH_EMAIL_SEND = "batch_email_send"
|
|
25
|
+
BATCH_EMAIL_SEND_LOCK = "batch_email_send_lock"
|
|
26
|
+
BATCH_EMAIL_SEND_LOCK_TIMEOUT = 600
|
|
23
27
|
|
|
24
28
|
|
|
25
29
|
class EmailExitState(Enum):
|
|
@@ -31,63 +35,74 @@ class EmailExitState(Enum):
|
|
|
31
35
|
missing_required_data = "missing_required_data"
|
|
32
36
|
email_send_failed = "email_send_failed"
|
|
33
37
|
complete = "complete"
|
|
38
|
+
email_send_already_running = "email_send_already_running"
|
|
34
39
|
|
|
35
40
|
|
|
36
41
|
@celery_app.task(base=DatabaseTask, bind=True)
|
|
37
42
|
def send_email_batch(self: DatabaseTask) -> EmailExitState:
|
|
38
43
|
"""Sends emails for each relevant connector with applicable user details batched together."""
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
consent_configs: Query = get_consent_email_connection_configs(session)
|
|
56
|
-
erasure_configs: Query = get_erasure_email_connection_configs(session)
|
|
57
|
-
combined_configs = consent_configs.union_all(erasure_configs)
|
|
58
|
-
if not combined_configs.first():
|
|
59
|
-
requeue_privacy_requests_after_email_send(privacy_requests, session)
|
|
60
|
-
logger.info(
|
|
61
|
-
"Skipping batch email send with status: {}",
|
|
62
|
-
EmailExitState.no_applicable_connectors.value,
|
|
44
|
+
with redis_lock(
|
|
45
|
+
lock_key=BATCH_EMAIL_SEND_LOCK, timeout=BATCH_EMAIL_SEND_LOCK_TIMEOUT
|
|
46
|
+
) as lock:
|
|
47
|
+
if not lock:
|
|
48
|
+
return EmailExitState.email_send_already_running
|
|
49
|
+
|
|
50
|
+
batch_id = str(uuid.uuid4())
|
|
51
|
+
logger.info("Starting batch email send {}...", batch_id)
|
|
52
|
+
with self.get_new_session() as session:
|
|
53
|
+
privacy_requests: Query = (
|
|
54
|
+
session.query(PrivacyRequest)
|
|
55
|
+
.filter(
|
|
56
|
+
PrivacyRequest.status == PrivacyRequestStatus.awaiting_email_send
|
|
57
|
+
)
|
|
58
|
+
.filter(PrivacyRequest.deleted_at.is_(None))
|
|
59
|
+
.order_by(PrivacyRequest.created_at.asc()) # oldest first
|
|
63
60
|
)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
for connection_config in erasure_configs:
|
|
69
|
-
get_connector(connection_config).batch_email_send( # type: ignore
|
|
70
|
-
filter_privacy_requests_by_action_type(
|
|
71
|
-
privacy_requests, ActionType.erasure
|
|
72
|
-
)
|
|
61
|
+
if not privacy_requests.first():
|
|
62
|
+
logger.info(
|
|
63
|
+
"Skipping batch email send with status: {}",
|
|
64
|
+
EmailExitState.no_applicable_privacy_requests.value,
|
|
73
65
|
)
|
|
66
|
+
return EmailExitState.no_applicable_privacy_requests
|
|
67
|
+
|
|
68
|
+
consent_configs: Query = get_consent_email_connection_configs(session)
|
|
69
|
+
erasure_configs: Query = get_erasure_email_connection_configs(session)
|
|
70
|
+
combined_configs = consent_configs.union_all(erasure_configs)
|
|
71
|
+
if not combined_configs.first():
|
|
72
|
+
requeue_privacy_requests_after_email_send(privacy_requests, session)
|
|
73
|
+
logger.info(
|
|
74
|
+
"Skipping batch email send with status: {}",
|
|
75
|
+
EmailExitState.no_applicable_connectors.value,
|
|
76
|
+
)
|
|
77
|
+
return EmailExitState.no_applicable_connectors
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
# erasure
|
|
81
|
+
for connection_config in erasure_configs:
|
|
82
|
+
get_connector(connection_config).batch_email_send( # type: ignore
|
|
83
|
+
filter_privacy_requests_by_action_type(
|
|
84
|
+
privacy_requests, ActionType.erasure
|
|
85
|
+
),
|
|
86
|
+
batch_id,
|
|
87
|
+
)
|
|
74
88
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
89
|
+
# consent
|
|
90
|
+
for connection_config in consent_configs:
|
|
91
|
+
get_connector(connection_config).batch_email_send( # type: ignore
|
|
92
|
+
filter_privacy_requests_by_action_type(
|
|
93
|
+
privacy_requests, ActionType.consent
|
|
94
|
+
),
|
|
95
|
+
batch_id,
|
|
80
96
|
)
|
|
97
|
+
except MessageDispatchException as exc:
|
|
98
|
+
logger.error(
|
|
99
|
+
"Batch email send for connector failed with exception: '{}'",
|
|
100
|
+
exc,
|
|
81
101
|
)
|
|
82
|
-
|
|
83
|
-
logger.error(
|
|
84
|
-
"Batch email send for connector failed with exception: '{}'",
|
|
85
|
-
exc,
|
|
86
|
-
)
|
|
87
|
-
return EmailExitState.email_send_failed
|
|
102
|
+
return EmailExitState.email_send_failed
|
|
88
103
|
|
|
89
|
-
|
|
90
|
-
|
|
104
|
+
requeue_privacy_requests_after_email_send(privacy_requests, session)
|
|
105
|
+
return EmailExitState.complete
|
|
91
106
|
|
|
92
107
|
|
|
93
108
|
def filter_privacy_requests_by_action_type(
|
|
@@ -135,6 +150,11 @@ def initiate_scheduled_batch_email_send() -> None:
|
|
|
135
150
|
|
|
136
151
|
assert scheduler.running, "Scheduler is not running! Cannot add Batch Email job."
|
|
137
152
|
|
|
153
|
+
cron_trigger = create_cron_trigger(
|
|
154
|
+
cron_expression=CONFIG.execution.email_send_cron_expression,
|
|
155
|
+
timezone=CONFIG.execution.email_send_timezone,
|
|
156
|
+
)
|
|
157
|
+
|
|
138
158
|
logger.info("Initiating scheduler for batch email send")
|
|
139
159
|
scheduler.add_job(
|
|
140
160
|
func=send_email_batch,
|
|
@@ -142,9 +162,5 @@ def initiate_scheduled_batch_email_send() -> None:
|
|
|
142
162
|
id=BATCH_EMAIL_SEND,
|
|
143
163
|
coalesce=False,
|
|
144
164
|
replace_existing=True,
|
|
145
|
-
trigger=
|
|
146
|
-
minute="0",
|
|
147
|
-
hour="12",
|
|
148
|
-
day_of_week="mon",
|
|
149
|
-
timezone="US/Eastern",
|
|
165
|
+
trigger=cron_trigger,
|
|
150
166
|
)
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
2
2
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
3
|
+
from apscheduler.triggers.cron import CronTrigger
|
|
3
4
|
|
|
4
5
|
scheduler = BackgroundScheduler()
|
|
5
6
|
async_scheduler = AsyncIOScheduler()
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def create_cron_trigger(cron_expression: str, timezone: str) -> CronTrigger:
|
|
10
|
+
return CronTrigger.from_crontab(cron_expression, timezone=timezone)
|
fides/api/util/lock.py
CHANGED
|
@@ -24,6 +24,9 @@ def redis_lock(lock_key: str, timeout: int) -> Generator[Lock | None, None, None
|
|
|
24
24
|
If the lock is acquired, it yields the lock object.
|
|
25
25
|
If the lock is not acquired, it yields None.
|
|
26
26
|
The lock is automatically released on exiting the context.
|
|
27
|
+
|
|
28
|
+
Timeout is the maximum number of seconds for the lock to be held,
|
|
29
|
+
after which the lock will be released.
|
|
27
30
|
"""
|
|
28
31
|
lock = get_redis_lock(lock_key, timeout)
|
|
29
32
|
|
|
@@ -48,6 +48,7 @@ PRIVACY_NOTICE = "privacy-notice"
|
|
|
48
48
|
PRIVACY_PREFERENCE_HISTORY = "privacy-preference-history"
|
|
49
49
|
PRIVACY_REQUEST = "privacy-request"
|
|
50
50
|
PRIVACY_REQUEST_ACCESS_RESULTS = "privacy-request-access-results"
|
|
51
|
+
PRIVACY_REQUEST_EMAIL_INTEGRATIONS = "privacy-request-email-integrations"
|
|
51
52
|
PRIVACY_REQUEST_NOTIFICATIONS = "privacy-request-notifications"
|
|
52
53
|
READ = "read"
|
|
53
54
|
REGISTER = "register"
|
|
@@ -58,6 +59,7 @@ REVIEW = "review"
|
|
|
58
59
|
RULE = "rule"
|
|
59
60
|
SAAS_CONFIG = "saas_config"
|
|
60
61
|
SCOPE = "scope"
|
|
62
|
+
SEND = "send"
|
|
61
63
|
STORAGE = "storage"
|
|
62
64
|
SYSTEM = "system"
|
|
63
65
|
SYSTEM_MANAGER = "system_manager"
|
|
@@ -176,6 +178,7 @@ PRIVACY_PREFERENCE_HISTORY_READ = f"{PRIVACY_PREFERENCE_HISTORY}:{READ}"
|
|
|
176
178
|
PRIVACY_REQUEST_CALLBACK_RESUME = f"{PRIVACY_REQUEST}:{RESUME}" # User has permission to restart a paused privacy request
|
|
177
179
|
PRIVACY_REQUEST_CREATE = f"{PRIVACY_REQUEST}:{CREATE}"
|
|
178
180
|
PRIVACY_REQUEST_DELETE = f"{PRIVACY_REQUEST}:{DELETE}"
|
|
181
|
+
PRIVACY_REQUEST_EMAIL_INTEGRATIONS_SEND = f"{PRIVACY_REQUEST_EMAIL_INTEGRATIONS}:{SEND}"
|
|
179
182
|
PRIVACY_REQUEST_MANUAL_STEPS_REVIEW = f"{PRIVACY_REQUEST}:{MANUAL_STEPS}:{REVIEW}"
|
|
180
183
|
PRIVACY_REQUEST_MANUAL_STEPS_RESPOND = f"{PRIVACY_REQUEST}:{MANUAL_STEPS}:{RESPOND}"
|
|
181
184
|
PRIVACY_REQUEST_NOTIFICATIONS_CREATE_OR_UPDATE = (
|
|
@@ -316,6 +319,7 @@ SCOPE_DOCS = {
|
|
|
316
319
|
PRIVACY_REQUEST_CALLBACK_RESUME: "Restart paused privacy requests",
|
|
317
320
|
PRIVACY_REQUEST_READ_ACCESS_RESULTS: "Download access data for the privacy request",
|
|
318
321
|
PRIVACY_REQUEST_DELETE: "Remove privacy requests",
|
|
322
|
+
PRIVACY_REQUEST_EMAIL_INTEGRATIONS_SEND: "Send email for email integrations for the privacy request",
|
|
319
323
|
PRIVACY_REQUEST_MANUAL_STEPS_RESPOND: "Respond to manual steps for the privacy request",
|
|
320
324
|
PRIVACY_REQUEST_MANUAL_STEPS_REVIEW: "Review manual steps for the privacy request",
|
|
321
325
|
PRIVACY_REQUEST_NOTIFICATIONS_CREATE_OR_UPDATE: "",
|
|
@@ -76,6 +76,9 @@ POLICY_DETAIL = "/dsr/policy/{policy_key}"
|
|
|
76
76
|
# Privacy request URLs
|
|
77
77
|
PRIVACY_REQUESTS = "/privacy-request"
|
|
78
78
|
PRIVACY_REQUEST_APPROVE = "/privacy-request/administrate/approve"
|
|
79
|
+
PRIVACY_REQUEST_BATCH_EMAIL_SEND = (
|
|
80
|
+
"/privacy-request/administrate/process-awaiting-email-send"
|
|
81
|
+
)
|
|
79
82
|
PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated"
|
|
80
83
|
PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry"
|
|
81
84
|
PRIVACY_REQUEST_BULK_SOFT_DELETE = "/privacy-request/bulk/soft-delete"
|
|
@@ -65,4 +65,12 @@ class ExecutionSettings(FidesSettings):
|
|
|
65
65
|
default=True,
|
|
66
66
|
description="Whether fuzzy search is enabled for privacy request lookups.",
|
|
67
67
|
)
|
|
68
|
+
email_send_cron_expression: str = Field(
|
|
69
|
+
default="0 12 * * mon",
|
|
70
|
+
description="The cron expression to send batch emails for DSR email integration. Defaults to weekly on Mondays at 12pm (noon).",
|
|
71
|
+
)
|
|
72
|
+
email_send_timezone: str = Field(
|
|
73
|
+
default="US/Eastern",
|
|
74
|
+
description="The timezone to send batch emails for DSR email integration.",
|
|
75
|
+
)
|
|
68
76
|
model_config = SettingsConfigDict(env_prefix=ENV_PREFIX)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/
|
|
1
|
+
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/d522df24a125b46f.css" as="style"/><link rel="stylesheet" href="/_next/static/css/d522df24a125b46f.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-8fc1c717e1c0f7ae.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-be66c01c2b1de442.js" defer=""></script><script src="/_next/static/chunks/pages/404-ac335065bc49f818.js" defer=""></script><script src="/_next/static/e4X4A9FU3w3buLKgvkCKW/_buildManifest.js" defer=""></script><script src="/_next/static/e4X4A9FU3w3buLKgvkCKW/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"e4X4A9FU3w3buLKgvkCKW","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[3988],{59969:function(e,t,i){var n=i(24246),r=i(86920);i(27378);var l=i(8557);t.Z=e=>{let{daysLeft:t,includeText:i,status:s}=e;if(!t||s===l.q.COMPLETE||s===l.q.CANCELED||s===l.q.DENIED||s===l.q.IDENTITY_UNVERIFIED)return(0,n.jsx)("span",{children:"-"});let a="";switch(!0){case t>=10:a="success";break;case t<10&&t>4:a="warning";break;case t<5:a="error"}return(0,n.jsx)(r.j8w,{color:a,children:i?"".concat(t," days left"):t})}},18225:function(e,t,i){var n=i(24246),r=i(86920);t.Z=e=>{let{alignment:t="center",...i}=e;return(0,n.jsx)(r.kCb,{boxSize:"full",align:"center",justify:t,children:(0,n.jsx)(r.$jN,{color:"primary",...i})})}},58754:function(e,t,i){var n=i(24246),r=i(86920),l=i(70788);t.Z=e=>{let{heading:t,breadcrumbItems:i,isSticky:s=!0,children:a,rightContent:d,style:o,...c}=e;return(0,n.jsxs)("div",{...c,style:s?{position:"sticky",top:"-24px",paddingTop:"24px",paddingBottom:"24px",paddingLeft:"40px",marginLeft:"-40px",paddingRight:"40px",marginRight:"-40px",marginTop:"-24px",left:0,zIndex:20,backgroundColor:"white",...o}:{paddingBottom:"24px",...o},children:[(0,n.jsxs)(r.jqI,{justify:"space-between",children:["string"==typeof t?(0,n.jsx)(r.lQT,{className:i||a?"pb-4":void 0,level:1,"data-testid":"page-heading",children:t}):t,d&&(0,n.jsx)("div",{"data-testid":"page-header-right-content",children:d})]}),!!i&&(0,n.jsx)(l.m,{className:a?"pb-4":void 0,items:i,"data-testid":"page-breadcrumb"}),a]})}},19904:function(e,t,i){i.d(t,{Tg:function(){return s}});var n=i(24246),r=i(16134),l=i(31793);let s=e=>(0,r.C)(l.uu).filter(t=>e.includes(t)).length>0;t.ZP=e=>{let{scopes:t,children:i}=e;return s(t)?(0,n.jsx)(n.Fragment,{children:i}):null}},14047:function(e,t,i){i.d(t,{H:function(){return l},V:function(){return n.V}});var n=i(84306),r=i(812);let l=()=>{let{errorAlert:e}=(0,n.V)();return{handleError:t=>{let i="An unexpected error occurred. Please try again.";(0,r.Ot)(t)?i=t.data.detail:(0,r.tB)(t)&&(i=t.data.detail[0].msg),e(i)}}}},84306:function(e,t,i){i.d(t,{V:function(){return l}});var n=i(24246),r=i(86920);let l=()=>{let e=(0,r.pmc)();return{errorAlert:(t,i,l)=>{let s={...l,position:(null==l?void 0:l.position)||"top",render:e=>{let{onClose:l}=e;return(0,n.jsxs)(r.bZj,{alignItems:"normal",status:"error","data-testid":"error-alert",children:[(0,n.jsx)(r.zMQ,{}),(0,n.jsxs)(r.xuv,{children:[i&&(0,n.jsx)(r.CdC,{children:i}),(0,n.jsx)(r.XaZ,{children:t})]}),(0,n.jsx)(r.PZ7,{onClick:l,position:"relative",right:0,size:"sm",top:-1})]})}};(null==l?void 0:l.id)&&e.isActive(l.id)?e.update(l.id,s):e(s)},successAlert:(t,i,l)=>{let s={...l,position:(null==l?void 0:l.position)||"top",render:e=>{let{onClose:l}=e;return(0,n.jsxs)(r.bZj,{alignItems:"normal",status:"success",variant:"subtle","data-testid":"success-alert",children:[(0,n.jsx)(r.zMQ,{}),(0,n.jsxs)(r.xuv,{children:[i&&(0,n.jsx)(r.CdC,{children:i}),(0,n.jsx)(r.XaZ,{children:t})]}),(0,n.jsx)(r.PZ7,{onClick:l,position:"relative",right:0,size:"sm",top:-1})]})}};(null==l?void 0:l.id)&&e.isActive(l.id)?e.update(l.id,s):e(s)}}}},70788:function(e,t,i){i.d(t,{m:function(){return o}});var n=i(24246),r=i(86920),l=i(79894),s=i.n(l),a=i(27378);let{Text:d}=r.AntTypography,o=e=>{let{items:t,...i}=e,l=(0,a.useMemo)(()=>null==t?void 0:t.map((e,i)=>{let l=i===t.length-1,a={...e},o=a.onClick&&!a.href;return("string"==typeof a.title&&(a.title=(0,n.jsx)(d,{style:{color:"inherit",maxWidth:l?void 0:400},ellipsis:!l,children:a.title})),o)?a.title=(0,n.jsx)(r.wpx,{type:"text",size:"small",icon:a.icon,onClick:a.onClick,className:"ant-breadcrumb-link -mt-px px-1 text-inherit",children:a.title}):(a.icon&&(a.title=(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)("span",{className:"anticon align-text-bottom",children:a.icon}),a.title]})),a.href&&a.title&&(a.title=(0,n.jsx)(s(),{href:a.href,className:"ant-breadcrumb-link",children:a.title}),delete a.href)),a}),[t]);return(0,n.jsx)(r.zrq,{items:l,...i})}},57526:function(e,t,i){var n=i(24246),r=i(86920),l=i(27378);t.Z=e=>{let{isOpen:t,onClose:i,onApproveRequest:s,isLoading:a,subjectRequest:d}=e,{identity:o,identity_verified_at:c,custom_privacy_request_fields:u}=d,h=(0,l.useCallback)(()=>{s().then(()=>{i()})},[s,i]);return(0,n.jsxs)(r.u_l,{isOpen:t,onClose:i,size:"lg",isCentered:!0,children:[(0,n.jsx)(r.ZAr,{}),(0,n.jsxs)(r.hzk,{children:[(0,n.jsx)(r.xBx,{children:"Privacy request approval"}),(0,n.jsxs)(r.fef,{paddingTop:0,paddingBottom:0,children:[(0,n.jsx)(r.xvT,{color:"gray.500",fontSize:"14px",marginBottom:4,children:"Are you sure you want to approve this privacy request?"}),(0,n.jsxs)(r.QI$,{children:[Object.entries(o).filter(e=>{let[,{value:t}]=e;return null!==t}).map(e=>{let[t,{value:i,label:l}]=e;return(0,n.jsx)(r.HCh,{children:(0,n.jsxs)(r.kCb,{alignItems:"flex-start",children:[(0,n.jsxs)(r.xvT,{mr:2,fontSize:"sm",color:"gray.900",fontWeight:"500",children:[l,":"]}),(0,n.jsx)(r.xvT,{color:"gray.600",fontWeight:"500",fontSize:"sm",mr:2,children:i}),"(",c?"Verified":"Unverified",")"]},t)},t)}),u&&Object.entries(u).filter(e=>{let[,t]=e;return t.value}).map(e=>{let[t,i]=e;return(0,n.jsx)(r.HCh,{children:(0,n.jsxs)(r.kCb,{alignItems:"flex-start",children:[(0,n.jsxs)(r.xvT,{mr:2,fontSize:"sm",color:"gray.900",fontWeight:"500",children:[i.label,":"]}),(0,n.jsxs)(r.xvT,{color:"gray.600",fontWeight:"500",fontSize:"sm",mr:2,children:[Array.isArray(i.value)?i.value.join(", "):i.value," "]}),"(Unverified)"]},t)},t)})]})]}),(0,n.jsx)(r.mzw,{children:(0,n.jsxs)(r.MIq,{columns:2,width:"100%",children:[(0,n.jsx)(r.wpx,{onClick:i,className:"mr-3","data-testid":"cancel-btn",children:"Cancel"}),(0,n.jsx)(r.wpx,{type:"primary","data-testid":"continue-btn",onClick:h,loading:a,children:"Confirm"})]})})]})]})}},66548:function(e,t,i){var n=i(24246),r=i(40324),l=i(86920),s=i(34090),a=i(27378),d=i(55484);let o={denialReason:""};t.Z=e=>{let{isOpen:t,onClose:i,onDenyRequest:c}=e,u=(0,a.useCallback)((e,t)=>{let{setSubmitting:n}=t;c(e.denialReason).then(()=>{n(!1),i()})},[c,i]);return(0,n.jsxs)(l.u_l,{isOpen:t,onClose:i,isCentered:!0,returnFocusOnClose:!1,children:[(0,n.jsx)(l.ZAr,{}),(0,n.jsx)(l.hzk,{width:"100%",maxWidth:"456px","data-testid":"deny-privacy-request-modal",children:(0,n.jsx)(s.J9,{initialValues:o,validationSchema:d.Ry({denialReason:d.Z_().required().label("Reason for denial")}),onSubmit:u,children:e=>{let{isSubmitting:t,dirty:a,isValid:d}=e;return(0,n.jsxs)(s.l0,{children:[(0,n.jsx)(l.xBx,{children:"Privacy request denial"}),(0,n.jsx)(l.fef,{color:"gray.500",fontSize:"14px",children:"Please enter a reason for denying this privacy request. Please note: this can be seen by the user in their notification email."}),(0,n.jsx)(l.fef,{children:(0,n.jsx)(r.Ks,{name:"denialReason",textAreaProps:{focusBorderColor:"primary.600",resize:"none"}})}),(0,n.jsxs)(l.mzw,{className:"flex w-full gap-4",children:[(0,n.jsx)(l.wpx,{disabled:t,onClick:i,className:"grow",children:"Cancel"}),(0,n.jsx)(l.wpx,{htmlType:"submit",type:"primary",disabled:!a||!d,loading:t,className:"grow","data-testid":"deny-privacy-request-modal-btn",children:"Confirm"})]})]})}})})]})}},45489:function(e,t,i){i.d(t,{Z:function(){return r}});var n=i(90867);let r=e=>{let{subjectRequest:t}=e,[i,r]=(0,n.RW)({fixedCacheKey:t.id}),[l,s]=(0,n.F1)({fixedCacheKey:t.id}),[a,d]=(0,n.rC)({fixedCacheKey:t.id}),o=s.isLoading||r.isLoading;return{approveRequest:i,approveRequestResult:r,denyRequest:l,denyRequestResult:s,handleApproveRequest:()=>i(t),handleDenyRequest:e=>l({id:t.id,reason:e}),handleDeleteRequest:()=>a(t),softDeleteRequestResult:d,isLoading:o}}},72281:function(e,t,i){i.d(t,{G:function(){return l},d:function(){return s}});var n=i(86677),r=i(27378);let l={REQUEST:"request",MANUAL_TASK:"manual-tasks"},s=()=>{let e=(0,n.useRouter)(),[t,i]=(0,r.useState)(l.REQUEST),s=(0,r.useMemo)(()=>({request:!0,manualTask:!0}),[]),a=(0,r.useCallback)(()=>{let{tab:t}=e.query;return t&&"string"==typeof t&&Object.values(l).includes(t)?t:null},[e.query]),d=(0,r.useCallback)(t=>{e.replace({pathname:e.pathname,query:{tab:t}},void 0,{shallow:!0})},[e]);return(0,r.useEffect)(()=>{if(!e.isReady)return;let t=a();if(t&&(t===l.REQUEST||t===l.MANUAL_TASK&&s.manualTask)){i(t);return}i(l.REQUEST),d(l.REQUEST)},[e.isReady,e.query,s,a,d]),{activeTab:t,handleTabChange:(0,r.useCallback)(e=>{Object.values(l).includes(e)&&(i(e),d(e))},[d]),availableTabs:s}}}}]);
|