ethyca-fides 2.63.1b4__py2.py3-none-any.whl → 2.63.1rc0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/RECORD +114 -126
- fides/_version.py +3 -3
- fides/api/db/base.py +0 -2
- fides/api/main.py +0 -1
- fides/api/models/attachment.py +23 -36
- fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +46 -264
- fides/api/service/privacy_request/dsr_package/templates/collection_index.html +9 -34
- fides/api/service/privacy_request/dsr_package/templates/item.html +37 -0
- fides/api/service/privacy_request/dsr_package/templates/main.css +2 -45
- fides/api/service/privacy_request/dsr_package/templates/welcome.html +8 -12
- fides/api/service/privacy_request/request_runner_service.py +139 -258
- fides/api/service/storage/gcs.py +3 -15
- fides/api/service/storage/s3.py +14 -28
- fides/api/service/storage/util.py +7 -45
- fides/api/tasks/storage.py +91 -85
- fides/api/util/cache.py +1 -77
- fides/config/redis_settings.py +8 -99
- fides/service/messaging/aws_ses_service.py +1 -5
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/{X2nvWLg2_-vsCTkhSWpzw → PEElhfUdgE5bJjiyu5QCD}/_buildManifest.js +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-8cab04871908cfeb.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-150d40428245ee0c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-20cdb2c8a03deae1.js +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +0 -160
- fides/api/models/manual_tasks/__init__.py +0 -8
- fides/api/models/manual_tasks/manual_task.py +0 -110
- fides/api/models/manual_tasks/manual_task_log.py +0 -100
- fides/api/schemas/manual_tasks/__init__.py +0 -0
- fides/api/schemas/manual_tasks/manual_task_schemas.py +0 -79
- fides/api/schemas/manual_tasks/manual_task_status.py +0 -151
- fides/api/service/privacy_request/attachment_handling.py +0 -132
- fides/api/service/privacy_request/dsr_package/templates/attachments_index.html +0 -33
- fides/api/tasks/csv_utils.py +0 -170
- fides/api/tasks/encryption_utils.py +0 -42
- fides/service/manual_tasks/__init__.py +0 -0
- fides/service/manual_tasks/manual_task_service.py +0 -150
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-c583a61302f02add.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-20d20a8d1736f7c4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-0e557d79e1e43c2b.js +0 -1
- {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{X2nvWLg2_-vsCTkhSWpzw → PEElhfUdgE5bJjiyu5QCD}/_ssgManifest.js +0 -0
@@ -1,151 +0,0 @@
|
|
1
|
-
from datetime import datetime, timezone
|
2
|
-
from enum import Enum as EnumType
|
3
|
-
from typing import Optional
|
4
|
-
|
5
|
-
from sqlalchemy.orm import Session
|
6
|
-
|
7
|
-
|
8
|
-
class StatusTransitionNotAllowed(Exception):
|
9
|
-
"""Exception raised when a status transition is not allowed."""
|
10
|
-
|
11
|
-
def __init__(self, message: str):
|
12
|
-
self.message = message
|
13
|
-
super().__init__(self.message)
|
14
|
-
|
15
|
-
|
16
|
-
class StatusType(str, EnumType):
|
17
|
-
"""Enum for manual task status."""
|
18
|
-
|
19
|
-
pending = "pending"
|
20
|
-
in_progress = "in_progress"
|
21
|
-
completed = "completed"
|
22
|
-
failed = "failed"
|
23
|
-
|
24
|
-
@classmethod
|
25
|
-
def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
|
26
|
-
"""Get valid transitions from the current status.
|
27
|
-
|
28
|
-
Args:
|
29
|
-
current_status: The current status
|
30
|
-
|
31
|
-
Returns:
|
32
|
-
list[StatusType]: List of valid transitions
|
33
|
-
"""
|
34
|
-
if current_status == cls.pending:
|
35
|
-
return [cls.in_progress, cls.failed]
|
36
|
-
if current_status == cls.in_progress:
|
37
|
-
return [cls.completed, cls.failed]
|
38
|
-
if current_status == cls.completed:
|
39
|
-
return []
|
40
|
-
if current_status == cls.failed:
|
41
|
-
return [cls.pending, cls.in_progress]
|
42
|
-
return []
|
43
|
-
|
44
|
-
|
45
|
-
class StatusTransitionMixin:
|
46
|
-
"""Mixin for handling status transitions.
|
47
|
-
|
48
|
-
This mixin provides methods for managing status transitions and completion tracking.
|
49
|
-
It can be used by any model that needs status management.
|
50
|
-
"""
|
51
|
-
|
52
|
-
# These should be overridden by the implementing class
|
53
|
-
status: StatusType
|
54
|
-
completed_at: Optional[datetime]
|
55
|
-
completed_by_id: Optional[str]
|
56
|
-
|
57
|
-
def _get_valid_transitions(self) -> list[StatusType]:
|
58
|
-
"""Get valid transitions from the current status.
|
59
|
-
|
60
|
-
Returns:
|
61
|
-
list[StatusType]: List of valid transitions
|
62
|
-
"""
|
63
|
-
return StatusType.get_valid_transitions(self.status)
|
64
|
-
|
65
|
-
def _validate_status_transition(self, new_status: StatusType) -> None:
|
66
|
-
"""Validate that a status transition is allowed.
|
67
|
-
|
68
|
-
Args:
|
69
|
-
new_status: The new status to transition to
|
70
|
-
|
71
|
-
Raises:
|
72
|
-
StatusTransitionNotAllowed: If the transition is not allowed
|
73
|
-
"""
|
74
|
-
# Don't allow transitions to the same status
|
75
|
-
if new_status == self.status:
|
76
|
-
raise StatusTransitionNotAllowed(
|
77
|
-
f"Invalid status transition: already in status {new_status}"
|
78
|
-
)
|
79
|
-
|
80
|
-
# Get valid transitions for current status
|
81
|
-
valid_transitions = self._get_valid_transitions()
|
82
|
-
if new_status not in valid_transitions:
|
83
|
-
raise StatusTransitionNotAllowed(
|
84
|
-
f"Invalid status transition from {self.status} to {new_status}. "
|
85
|
-
f"Valid transitions are: {valid_transitions}"
|
86
|
-
)
|
87
|
-
|
88
|
-
def update_status(
|
89
|
-
self, db: Session, new_status: StatusType, user_id: Optional[str] = None
|
90
|
-
) -> None:
|
91
|
-
"""Update the status with validation and completion handling.
|
92
|
-
|
93
|
-
Args:
|
94
|
-
db: Database session
|
95
|
-
new_status: New status to set
|
96
|
-
user_id: Optional user ID who is making the change
|
97
|
-
"""
|
98
|
-
self._validate_status_transition(new_status)
|
99
|
-
|
100
|
-
if new_status == StatusType.completed:
|
101
|
-
self.completed_at = datetime.now(timezone.utc)
|
102
|
-
self.completed_by_id = user_id
|
103
|
-
elif new_status == StatusType.pending:
|
104
|
-
# Reset completion fields if going back to pending
|
105
|
-
self.completed_at = None
|
106
|
-
self.completed_by_id = None
|
107
|
-
|
108
|
-
self.status = new_status
|
109
|
-
db.add(self)
|
110
|
-
db.commit()
|
111
|
-
|
112
|
-
def mark_completed(self, db: Session, user_id: str) -> None:
|
113
|
-
"""Mark as completed.
|
114
|
-
|
115
|
-
Args:
|
116
|
-
db: Database session
|
117
|
-
user_id: user ID who completed the task
|
118
|
-
"""
|
119
|
-
self.update_status(db, StatusType.completed, user_id)
|
120
|
-
|
121
|
-
def mark_failed(self, db: Session) -> None:
|
122
|
-
"""Mark as failed."""
|
123
|
-
self.update_status(db, StatusType.failed)
|
124
|
-
|
125
|
-
def start_progress(self, db: Session) -> None:
|
126
|
-
"""Mark as in progress."""
|
127
|
-
self.update_status(db, StatusType.in_progress)
|
128
|
-
|
129
|
-
def reset_to_pending(self, db: Session) -> None:
|
130
|
-
"""Reset to pending status."""
|
131
|
-
self.update_status(db, StatusType.pending)
|
132
|
-
|
133
|
-
@property
|
134
|
-
def is_completed(self) -> bool:
|
135
|
-
"""Check if completed."""
|
136
|
-
return self.status == StatusType.completed
|
137
|
-
|
138
|
-
@property
|
139
|
-
def is_failed(self) -> bool:
|
140
|
-
"""Check if failed."""
|
141
|
-
return self.status == StatusType.failed
|
142
|
-
|
143
|
-
@property
|
144
|
-
def is_in_progress(self) -> bool:
|
145
|
-
"""Check if in progress."""
|
146
|
-
return self.status == StatusType.in_progress
|
147
|
-
|
148
|
-
@property
|
149
|
-
def is_pending(self) -> bool:
|
150
|
-
"""Check if pending."""
|
151
|
-
return self.status == StatusType.pending
|
@@ -1,132 +0,0 @@
|
|
1
|
-
import time as time_module
|
2
|
-
from dataclasses import dataclass
|
3
|
-
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
4
|
-
|
5
|
-
from loguru import logger
|
6
|
-
|
7
|
-
from fides.api.models.attachment import Attachment, AttachmentType
|
8
|
-
from fides.api.schemas.storage.storage import StorageDetails
|
9
|
-
|
10
|
-
|
11
|
-
@dataclass
|
12
|
-
class AttachmentData:
|
13
|
-
"""Data structure for attachment metadata and content.
|
14
|
-
Using a dataclass rather than a Pydantic model here for the following reasons:
|
15
|
-
- The data structure is simple and doesn't need complex validation.
|
16
|
-
- The fields being used have already been validated and are properly typed.
|
17
|
-
- The class is used internally for data transfer, not for API serialization.
|
18
|
-
- Performance is important since this is used in a data processing pipeline.
|
19
|
-
"""
|
20
|
-
|
21
|
-
file_name: str
|
22
|
-
file_size: Optional[int]
|
23
|
-
download_url: Optional[str]
|
24
|
-
content_type: str
|
25
|
-
bucket_name: str
|
26
|
-
file_key: str
|
27
|
-
storage_key: str
|
28
|
-
|
29
|
-
def to_upload_dict(self) -> Dict[str, Any]:
|
30
|
-
"""Convert to dictionary for upload, including presigned URL."""
|
31
|
-
return {
|
32
|
-
"file_name": self.file_name,
|
33
|
-
"file_size": self.file_size,
|
34
|
-
"download_url": self.download_url,
|
35
|
-
"content_type": self.content_type,
|
36
|
-
}
|
37
|
-
|
38
|
-
def to_storage_dict(self) -> Dict[str, Any]:
|
39
|
-
"""Convert to dictionary for storage, including the elements needed to recreated the presigned URL."""
|
40
|
-
return {
|
41
|
-
"file_name": self.file_name,
|
42
|
-
"file_size": self.file_size,
|
43
|
-
"content_type": self.content_type,
|
44
|
-
"bucket_name": self.bucket_name,
|
45
|
-
"file_key": self.file_key,
|
46
|
-
"storage_key": self.storage_key,
|
47
|
-
}
|
48
|
-
|
49
|
-
|
50
|
-
def get_attachments_content(
|
51
|
-
loaded_attachments: List[Attachment],
|
52
|
-
) -> Iterator[AttachmentData]:
|
53
|
-
"""
|
54
|
-
Retrieves all attachments associated with a privacy request that are marked to be included with the access package.
|
55
|
-
Yields AttachmentData objects containing attachment metadata and download urls.
|
56
|
-
Uses generators to minimize memory usage.
|
57
|
-
|
58
|
-
Args:
|
59
|
-
loaded_attachments: List of Attachment objects to process
|
60
|
-
|
61
|
-
Yields:
|
62
|
-
AttachmentData object containing attachment metadata and url
|
63
|
-
"""
|
64
|
-
start_time = time_module.time()
|
65
|
-
processed_count = 0
|
66
|
-
skipped_count = 0
|
67
|
-
error_count = 0
|
68
|
-
total_size = 0
|
69
|
-
|
70
|
-
for attachment in loaded_attachments:
|
71
|
-
if attachment.attachment_type != AttachmentType.include_with_access_package:
|
72
|
-
skipped_count += 1
|
73
|
-
continue
|
74
|
-
|
75
|
-
try:
|
76
|
-
# Get size and download URL using retrieve_attachment
|
77
|
-
size, url = attachment.retrieve_attachment()
|
78
|
-
total_size += size if size else 0
|
79
|
-
if url is None:
|
80
|
-
logger.warning(
|
81
|
-
"No download URL retrieved for attachment {}", attachment.file_name
|
82
|
-
)
|
83
|
-
skipped_count += 1
|
84
|
-
continue
|
85
|
-
|
86
|
-
processed_count += 1
|
87
|
-
yield AttachmentData(
|
88
|
-
file_name=attachment.file_name,
|
89
|
-
file_size=size,
|
90
|
-
download_url=str(url) if url else None,
|
91
|
-
content_type=attachment.content_type,
|
92
|
-
bucket_name=attachment.config.details[StorageDetails.BUCKET.value],
|
93
|
-
file_key=attachment.file_key,
|
94
|
-
storage_key=attachment.storage_key,
|
95
|
-
)
|
96
|
-
|
97
|
-
except Exception as e:
|
98
|
-
error_count += 1
|
99
|
-
logger.error(
|
100
|
-
"Error processing attachment {}: {}", attachment.file_name, str(e)
|
101
|
-
)
|
102
|
-
continue
|
103
|
-
|
104
|
-
# Log final metrics
|
105
|
-
time_taken = time_module.time() - start_time
|
106
|
-
logger.bind(
|
107
|
-
time_to_process=time_taken,
|
108
|
-
total_attachments=len(loaded_attachments),
|
109
|
-
processed_attachments=processed_count,
|
110
|
-
skipped_attachments=skipped_count,
|
111
|
-
error_attachments=error_count,
|
112
|
-
total_size_bytes=total_size,
|
113
|
-
).info("Attachment processing complete")
|
114
|
-
|
115
|
-
|
116
|
-
def process_attachments_for_upload(
|
117
|
-
attachments: Iterator[AttachmentData],
|
118
|
-
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
119
|
-
"""
|
120
|
-
Process attachments into separate upload and storage formats.
|
121
|
-
Returns both formats:
|
122
|
-
- upload_attachments: Used for uploading to access packages
|
123
|
-
- storage_attachments: Used for saving filtered access results
|
124
|
-
"""
|
125
|
-
upload_attachments = []
|
126
|
-
storage_attachments = []
|
127
|
-
|
128
|
-
for attachment in attachments:
|
129
|
-
storage_attachments.append(attachment.to_storage_dict())
|
130
|
-
upload_attachments.append(attachment.to_upload_dict())
|
131
|
-
|
132
|
-
return upload_attachments, storage_attachments
|
@@ -1,33 +0,0 @@
|
|
1
|
-
<html>
|
2
|
-
<head>
|
3
|
-
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600">
|
4
|
-
<link rel="stylesheet" href="../data/main.css">
|
5
|
-
</head>
|
6
|
-
<body>
|
7
|
-
<div class="container">
|
8
|
-
<div class="header"></div>
|
9
|
-
<div class="content">
|
10
|
-
<div class="button-container">
|
11
|
-
<a href="../welcome.html">
|
12
|
-
<div class="button"><img src="../data/back.svg"></div>
|
13
|
-
<span>Back to main page</span>
|
14
|
-
</a>
|
15
|
-
</div>
|
16
|
-
<h1>Attachments</h1>
|
17
|
-
<p class="expiration-notice">Note: All download links will expire in 7 days.</p>
|
18
|
-
<div class="table table-hover">
|
19
|
-
<div class="table-row">
|
20
|
-
<div class="table-cell" style="text-align: left;">File Name</div>
|
21
|
-
<div class="table-cell" style="text-align: left;">Size</div>
|
22
|
-
</div>
|
23
|
-
{% for name, info in data.items() %}
|
24
|
-
<a href="{{ info.url }}" class="table-row" target="_blank">
|
25
|
-
<div class="table-cell" style="text-align: left;">{{ name }}</div>
|
26
|
-
<div class="table-cell" style="text-align: left;">{{ info.size }}</div>
|
27
|
-
</a>
|
28
|
-
{% endfor %}
|
29
|
-
</div>
|
30
|
-
</div>
|
31
|
-
</div>
|
32
|
-
</body>
|
33
|
-
</html>
|
fides/api/tasks/csv_utils.py
DELETED
@@ -1,170 +0,0 @@
|
|
1
|
-
import zipfile
|
2
|
-
from io import BytesIO
|
3
|
-
from typing import Any, Optional
|
4
|
-
|
5
|
-
import pandas as pd
|
6
|
-
|
7
|
-
from fides.api.tasks.encryption_utils import encrypt_access_request_results
|
8
|
-
from fides.config import CONFIG
|
9
|
-
|
10
|
-
|
11
|
-
def create_csv_from_dataframe(df: pd.DataFrame) -> BytesIO:
|
12
|
-
"""Create a CSV file from a pandas DataFrame.
|
13
|
-
|
14
|
-
Args:
|
15
|
-
df: The DataFrame to convert to CSV
|
16
|
-
|
17
|
-
Returns:
|
18
|
-
BytesIO: A file-like object containing the CSV data
|
19
|
-
"""
|
20
|
-
buffer = BytesIO()
|
21
|
-
df.to_csv(buffer, index=False, encoding=CONFIG.security.encoding)
|
22
|
-
buffer.seek(0)
|
23
|
-
return buffer
|
24
|
-
|
25
|
-
|
26
|
-
def create_attachment_csv(attachments: list[dict[str, Any]]) -> Optional[BytesIO]:
|
27
|
-
"""Create a CSV file containing attachment metadata.
|
28
|
-
|
29
|
-
Args:
|
30
|
-
attachments: List of attachment dictionaries
|
31
|
-
privacy_request_id: The ID of the privacy request for encryption
|
32
|
-
|
33
|
-
Returns:
|
34
|
-
Optional[BytesIO]: A file-like object containing the CSV data, or None if no attachments
|
35
|
-
"""
|
36
|
-
if not attachments:
|
37
|
-
return None
|
38
|
-
|
39
|
-
# Filter out invalid attachments and create a list of valid ones
|
40
|
-
valid_attachments = []
|
41
|
-
for a in attachments:
|
42
|
-
if not isinstance(a, dict):
|
43
|
-
continue
|
44
|
-
|
45
|
-
# Check if the attachment has at least one of the required fields
|
46
|
-
if not any(
|
47
|
-
key in a
|
48
|
-
for key in ["file_name", "file_size", "content_type", "download_url"]
|
49
|
-
):
|
50
|
-
continue
|
51
|
-
|
52
|
-
valid_attachments.append(
|
53
|
-
{
|
54
|
-
"file_name": a.get("file_name", ""),
|
55
|
-
"file_size": a.get("file_size", 0),
|
56
|
-
"content_type": a.get("content_type", "application/octet-stream"),
|
57
|
-
"download_url": a.get("download_url", ""),
|
58
|
-
}
|
59
|
-
)
|
60
|
-
|
61
|
-
# Return None if there are no valid attachments
|
62
|
-
if not valid_attachments:
|
63
|
-
return None
|
64
|
-
|
65
|
-
df = pd.DataFrame(valid_attachments)
|
66
|
-
|
67
|
-
if df.empty:
|
68
|
-
return None
|
69
|
-
|
70
|
-
return create_csv_from_dataframe(df)
|
71
|
-
|
72
|
-
|
73
|
-
def _write_attachment_csv(
|
74
|
-
zip_file: zipfile.ZipFile,
|
75
|
-
key: str,
|
76
|
-
idx: int,
|
77
|
-
attachments: list[dict[str, Any]],
|
78
|
-
privacy_request_id: str,
|
79
|
-
) -> None:
|
80
|
-
"""Write attachment data to a CSV file in the zip archive.
|
81
|
-
|
82
|
-
Args:
|
83
|
-
zip_file: The zip file to write to
|
84
|
-
key: The key for the data
|
85
|
-
idx: The index of the item in the list
|
86
|
-
attachments: List of attachment dictionaries
|
87
|
-
privacy_request_id: The ID of the privacy request for encryption
|
88
|
-
"""
|
89
|
-
buffer = create_attachment_csv(attachments)
|
90
|
-
if buffer:
|
91
|
-
zip_file.writestr(
|
92
|
-
f"{key}/{idx + 1}/attachments.csv",
|
93
|
-
encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
|
94
|
-
)
|
95
|
-
|
96
|
-
|
97
|
-
def _write_item_csv(
|
98
|
-
zip_file: zipfile.ZipFile,
|
99
|
-
key: str,
|
100
|
-
items: list[dict[str, Any]],
|
101
|
-
privacy_request_id: str,
|
102
|
-
) -> None:
|
103
|
-
"""Write item data to a CSV file in the zip archive.
|
104
|
-
|
105
|
-
Args:
|
106
|
-
zip_file: The zip file to write to
|
107
|
-
key: The key for the data
|
108
|
-
items: List of items to write
|
109
|
-
privacy_request_id: The ID of the privacy request for encryption
|
110
|
-
"""
|
111
|
-
if items:
|
112
|
-
df = pd.DataFrame(items)
|
113
|
-
buffer = create_csv_from_dataframe(df)
|
114
|
-
zip_file.writestr(
|
115
|
-
f"{key}.csv",
|
116
|
-
encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
|
117
|
-
)
|
118
|
-
|
119
|
-
|
120
|
-
def _write_simple_csv(
|
121
|
-
zip_file: zipfile.ZipFile,
|
122
|
-
key: str,
|
123
|
-
value: Any,
|
124
|
-
privacy_request_id: str,
|
125
|
-
) -> None:
|
126
|
-
"""Write simple key-value data to a CSV file in the zip archive.
|
127
|
-
|
128
|
-
Args:
|
129
|
-
zip_file: The zip file to write to
|
130
|
-
key: The key for the data
|
131
|
-
value: The value to write
|
132
|
-
privacy_request_id: The ID of the privacy request for encryption
|
133
|
-
"""
|
134
|
-
df = pd.json_normalize({key: value})
|
135
|
-
buffer = create_csv_from_dataframe(df)
|
136
|
-
zip_file.writestr(
|
137
|
-
f"{key}.csv",
|
138
|
-
encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
|
139
|
-
)
|
140
|
-
|
141
|
-
|
142
|
-
def write_csv_to_zip(
|
143
|
-
zip_file: zipfile.ZipFile, data: dict[str, Any], privacy_request_id: str
|
144
|
-
) -> None:
|
145
|
-
"""Write data to a zip file in CSV format.
|
146
|
-
|
147
|
-
Args:
|
148
|
-
zip_file: The zip file to write to
|
149
|
-
data: The data to convert to CSV
|
150
|
-
privacy_request_id: The ID of the privacy request for encryption
|
151
|
-
"""
|
152
|
-
for key, value in data.items():
|
153
|
-
if (
|
154
|
-
isinstance(value, list)
|
155
|
-
and value
|
156
|
-
and all(isinstance(item, dict) for item in value)
|
157
|
-
):
|
158
|
-
# Handle lists of dictionaries
|
159
|
-
items: list[dict[str, Any]] = []
|
160
|
-
for item in value:
|
161
|
-
# Extract attachments if they exist
|
162
|
-
attachments = item.pop("attachments", [])
|
163
|
-
if attachments:
|
164
|
-
_write_attachment_csv(
|
165
|
-
zip_file, key, len(items), attachments, privacy_request_id
|
166
|
-
)
|
167
|
-
items.append(item)
|
168
|
-
_write_item_csv(zip_file, key, items, privacy_request_id)
|
169
|
-
else:
|
170
|
-
_write_simple_csv(zip_file, key, value, privacy_request_id)
|
@@ -1,42 +0,0 @@
|
|
1
|
-
import secrets
|
2
|
-
from typing import Optional, Union
|
3
|
-
|
4
|
-
from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
|
5
|
-
from fides.api.util.cache import get_cache, get_encryption_cache_key
|
6
|
-
from fides.api.util.encryption.aes_gcm_encryption_scheme import (
|
7
|
-
encrypt_to_bytes_verify_secrets_length,
|
8
|
-
)
|
9
|
-
from fides.config import CONFIG
|
10
|
-
|
11
|
-
|
12
|
-
def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> str:
|
13
|
-
"""Encrypt data with encryption key if provided, otherwise return unencrypted data.
|
14
|
-
|
15
|
-
Args:
|
16
|
-
data: The data to encrypt
|
17
|
-
request_id: The ID of the privacy request for encryption key lookup
|
18
|
-
|
19
|
-
Returns:
|
20
|
-
str: The encrypted data as a string
|
21
|
-
"""
|
22
|
-
cache = get_cache()
|
23
|
-
encryption_cache_key = get_encryption_cache_key(
|
24
|
-
privacy_request_id=request_id,
|
25
|
-
encryption_attr="key",
|
26
|
-
)
|
27
|
-
if isinstance(data, bytes):
|
28
|
-
data = data.decode(CONFIG.security.encoding)
|
29
|
-
|
30
|
-
encryption_key: Optional[str] = cache.get(encryption_cache_key)
|
31
|
-
if not encryption_key:
|
32
|
-
return data
|
33
|
-
|
34
|
-
bytes_encryption_key: bytes = encryption_key.encode(
|
35
|
-
encoding=CONFIG.security.encoding
|
36
|
-
)
|
37
|
-
nonce: bytes = secrets.token_bytes(CONFIG.security.aes_gcm_nonce_length)
|
38
|
-
# b64encode the entire nonce and the encrypted message together
|
39
|
-
return bytes_to_b64_str(
|
40
|
-
nonce
|
41
|
-
+ encrypt_to_bytes_verify_secrets_length(data, bytes_encryption_key, nonce)
|
42
|
-
)
|
File without changes
|
@@ -1,150 +0,0 @@
|
|
1
|
-
from typing import Optional
|
2
|
-
|
3
|
-
from loguru import logger
|
4
|
-
from sqlalchemy import select
|
5
|
-
from sqlalchemy.orm import Session
|
6
|
-
|
7
|
-
from fides.api.models.fides_user import FidesUser
|
8
|
-
from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
|
9
|
-
from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
|
10
|
-
from fides.api.schemas.manual_tasks.manual_task_schemas import (
|
11
|
-
ManualTaskLogStatus,
|
12
|
-
ManualTaskParentEntityType,
|
13
|
-
ManualTaskReferenceType,
|
14
|
-
ManualTaskType,
|
15
|
-
)
|
16
|
-
|
17
|
-
|
18
|
-
class ManualTaskService:
|
19
|
-
def __init__(self, db: Session):
|
20
|
-
self.db = db
|
21
|
-
|
22
|
-
def get_task(
|
23
|
-
self,
|
24
|
-
task_id: Optional[str] = None,
|
25
|
-
parent_entity_id: Optional[str] = None,
|
26
|
-
parent_entity_type: Optional[ManualTaskParentEntityType] = None,
|
27
|
-
task_type: Optional[ManualTaskType] = None,
|
28
|
-
) -> Optional[ManualTask]:
|
29
|
-
"""Get the manual task using provided filters.
|
30
|
-
|
31
|
-
Args:
|
32
|
-
task_id: The task ID
|
33
|
-
parent_entity_id: The parent entity ID
|
34
|
-
parent_entity_type: The parent entity type
|
35
|
-
task_type: The task type
|
36
|
-
|
37
|
-
Returns:
|
38
|
-
Optional[ManualTask]: The manual task for the connection, if it exists
|
39
|
-
"""
|
40
|
-
if not any([task_id, parent_entity_id, parent_entity_type, task_type]):
|
41
|
-
logger.warning("No filters provided to get_task. Returning None.")
|
42
|
-
return None
|
43
|
-
|
44
|
-
stmt = select(ManualTask) # type: ignore[arg-type]
|
45
|
-
if task_id:
|
46
|
-
stmt = stmt.where(ManualTask.id == task_id)
|
47
|
-
if parent_entity_id:
|
48
|
-
stmt = stmt.where(ManualTask.parent_entity_id == parent_entity_id)
|
49
|
-
if parent_entity_type:
|
50
|
-
stmt = stmt.where(ManualTask.parent_entity_type == parent_entity_type)
|
51
|
-
if task_type:
|
52
|
-
stmt = stmt.where(ManualTask.task_type == task_type)
|
53
|
-
return self.db.execute(stmt).scalar_one_or_none()
|
54
|
-
|
55
|
-
# User Management
|
56
|
-
def assign_users_to_task(
|
57
|
-
self, db: Session, task: ManualTask, user_ids: list[str]
|
58
|
-
) -> None:
|
59
|
-
"""Assigns users to this task. We can assign one or more users to a task.
|
60
|
-
|
61
|
-
Args:
|
62
|
-
db: Database session
|
63
|
-
task: The task to assign users to
|
64
|
-
user_ids: List of user IDs to assign
|
65
|
-
"""
|
66
|
-
user_ids = list(set(user_ids))
|
67
|
-
if not user_ids:
|
68
|
-
raise ValueError("User ID is required for assignment")
|
69
|
-
|
70
|
-
# Create new user assignment
|
71
|
-
for user_id in user_ids:
|
72
|
-
# if user is already assigned, skip
|
73
|
-
if user_id in task.assigned_users:
|
74
|
-
continue
|
75
|
-
# verify user exists
|
76
|
-
user = db.query(FidesUser).filter_by(id=user_id).first()
|
77
|
-
if not user:
|
78
|
-
ManualTaskLog.create_error_log(
|
79
|
-
db=db,
|
80
|
-
task_id=task.id,
|
81
|
-
message=f"Failed to add user {user_id} to task {task.id}: user does not exist",
|
82
|
-
details={"user_id": user_id},
|
83
|
-
)
|
84
|
-
continue
|
85
|
-
|
86
|
-
ManualTaskReference.create(
|
87
|
-
db=db,
|
88
|
-
data={
|
89
|
-
"task_id": task.id,
|
90
|
-
"reference_id": user_id,
|
91
|
-
"reference_type": ManualTaskReferenceType.assigned_user,
|
92
|
-
},
|
93
|
-
)
|
94
|
-
|
95
|
-
# Log the user assignment
|
96
|
-
ManualTaskLog.create_log(
|
97
|
-
db=db,
|
98
|
-
task_id=task.id,
|
99
|
-
status=ManualTaskLogStatus.updated,
|
100
|
-
message=f"User {user_id} assigned to task",
|
101
|
-
details={"assigned_user_id": user_id},
|
102
|
-
)
|
103
|
-
|
104
|
-
def unassign_users_from_task(
|
105
|
-
self, db: Session, task: ManualTask, user_ids: list[str]
|
106
|
-
) -> None:
|
107
|
-
"""Remove the user assignment from this task.
|
108
|
-
|
109
|
-
Args:
|
110
|
-
db: Database session
|
111
|
-
task: The task to unassign users from
|
112
|
-
user_ids: List of user IDs to unassign
|
113
|
-
"""
|
114
|
-
user_ids = list(set(user_ids))
|
115
|
-
if not user_ids:
|
116
|
-
raise ValueError("User ID is required for unassignment")
|
117
|
-
|
118
|
-
# Get references to unassign
|
119
|
-
references_to_unassign = (
|
120
|
-
db.query(ManualTaskReference)
|
121
|
-
.filter(
|
122
|
-
ManualTaskReference.task_id == task.id,
|
123
|
-
ManualTaskReference.reference_type
|
124
|
-
== ManualTaskReferenceType.assigned_user,
|
125
|
-
ManualTaskReference.reference_id.in_(user_ids),
|
126
|
-
)
|
127
|
-
.all()
|
128
|
-
)
|
129
|
-
|
130
|
-
# Delete references and log unassignments
|
131
|
-
for ref in references_to_unassign:
|
132
|
-
ref.delete(db)
|
133
|
-
ManualTaskLog.create_log(
|
134
|
-
db=db,
|
135
|
-
task_id=task.id,
|
136
|
-
status=ManualTaskLogStatus.updated,
|
137
|
-
message=f"User {ref.reference_id} unassigned from task",
|
138
|
-
details={"unassigned_user_id": ref.reference_id},
|
139
|
-
)
|
140
|
-
|
141
|
-
# Check if any users weren't unassigned
|
142
|
-
unassigned_user_ids = [ref.reference_id for ref in references_to_unassign]
|
143
|
-
left_over_user_ids = [
|
144
|
-
user_id for user_id in user_ids if user_id not in unassigned_user_ids
|
145
|
-
]
|
146
|
-
if left_over_user_ids:
|
147
|
-
logger.warning(
|
148
|
-
f"Failed to unassign users {left_over_user_ids} from task {task.id}: "
|
149
|
-
"users were not assigned to the task"
|
150
|
-
)
|