ethyca-fides 2.63.1b4__py2.py3-none-any.whl → 2.63.1rc0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/RECORD +114 -126
  3. fides/_version.py +3 -3
  4. fides/api/db/base.py +0 -2
  5. fides/api/main.py +0 -1
  6. fides/api/models/attachment.py +23 -36
  7. fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +46 -264
  8. fides/api/service/privacy_request/dsr_package/templates/collection_index.html +9 -34
  9. fides/api/service/privacy_request/dsr_package/templates/item.html +37 -0
  10. fides/api/service/privacy_request/dsr_package/templates/main.css +2 -45
  11. fides/api/service/privacy_request/dsr_package/templates/welcome.html +8 -12
  12. fides/api/service/privacy_request/request_runner_service.py +139 -258
  13. fides/api/service/storage/gcs.py +3 -15
  14. fides/api/service/storage/s3.py +14 -28
  15. fides/api/service/storage/util.py +7 -45
  16. fides/api/tasks/storage.py +91 -85
  17. fides/api/util/cache.py +1 -77
  18. fides/config/redis_settings.py +8 -99
  19. fides/service/messaging/aws_ses_service.py +1 -5
  20. fides/ui-build/static/admin/404.html +1 -1
  21. fides/ui-build/static/admin/_next/static/{X2nvWLg2_-vsCTkhSWpzw → PEElhfUdgE5bJjiyu5QCD}/_buildManifest.js +1 -1
  22. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-8cab04871908cfeb.js +1 -0
  23. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-150d40428245ee0c.js +1 -0
  24. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-20cdb2c8a03deae1.js +1 -0
  25. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  26. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  27. fides/ui-build/static/admin/add-systems.html +1 -1
  28. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  29. fides/ui-build/static/admin/consent/configure.html +1 -1
  30. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  31. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  32. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  33. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  34. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  35. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  36. fides/ui-build/static/admin/consent/properties.html +1 -1
  37. fides/ui-build/static/admin/consent/reporting.html +1 -1
  38. fides/ui-build/static/admin/consent.html +1 -1
  39. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  40. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  41. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  42. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  43. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  44. fides/ui-build/static/admin/data-catalog.html +1 -1
  45. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  46. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  47. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  48. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  49. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  50. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  51. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  52. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  53. fides/ui-build/static/admin/datamap.html +1 -1
  54. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  55. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  56. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  57. fides/ui-build/static/admin/dataset/new.html +1 -1
  58. fides/ui-build/static/admin/dataset.html +1 -1
  59. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  60. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  61. fides/ui-build/static/admin/datastore-connection.html +1 -1
  62. fides/ui-build/static/admin/index.html +1 -1
  63. fides/ui-build/static/admin/integrations/[id].html +1 -1
  64. fides/ui-build/static/admin/integrations.html +1 -1
  65. fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
  66. fides/ui-build/static/admin/lib/fides-headless.js +1 -1
  67. fides/ui-build/static/admin/lib/fides-preview.js +1 -1
  68. fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
  69. fides/ui-build/static/admin/lib/fides.js +2 -2
  70. fides/ui-build/static/admin/login/[provider].html +1 -1
  71. fides/ui-build/static/admin/login.html +1 -1
  72. fides/ui-build/static/admin/messaging/[id].html +1 -1
  73. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  74. fides/ui-build/static/admin/messaging.html +1 -1
  75. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  76. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  77. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  78. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  79. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  80. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  81. fides/ui-build/static/admin/poc/forms.html +1 -1
  82. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  83. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  84. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  85. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  86. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  87. fides/ui-build/static/admin/privacy-requests.html +1 -1
  88. fides/ui-build/static/admin/properties/[id].html +1 -1
  89. fides/ui-build/static/admin/properties/add-property.html +1 -1
  90. fides/ui-build/static/admin/properties.html +1 -1
  91. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  92. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  93. fides/ui-build/static/admin/settings/about.html +1 -1
  94. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  95. fides/ui-build/static/admin/settings/consent.html +1 -1
  96. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  97. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  98. fides/ui-build/static/admin/settings/domains.html +1 -1
  99. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  100. fides/ui-build/static/admin/settings/locations.html +1 -1
  101. fides/ui-build/static/admin/settings/organization.html +1 -1
  102. fides/ui-build/static/admin/settings/regulations.html +1 -1
  103. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  104. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  105. fides/ui-build/static/admin/systems.html +1 -1
  106. fides/ui-build/static/admin/taxonomy.html +1 -1
  107. fides/ui-build/static/admin/user-management/new.html +1 -1
  108. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  109. fides/ui-build/static/admin/user-management.html +1 -1
  110. fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +0 -160
  111. fides/api/models/manual_tasks/__init__.py +0 -8
  112. fides/api/models/manual_tasks/manual_task.py +0 -110
  113. fides/api/models/manual_tasks/manual_task_log.py +0 -100
  114. fides/api/schemas/manual_tasks/__init__.py +0 -0
  115. fides/api/schemas/manual_tasks/manual_task_schemas.py +0 -79
  116. fides/api/schemas/manual_tasks/manual_task_status.py +0 -151
  117. fides/api/service/privacy_request/attachment_handling.py +0 -132
  118. fides/api/service/privacy_request/dsr_package/templates/attachments_index.html +0 -33
  119. fides/api/tasks/csv_utils.py +0 -170
  120. fides/api/tasks/encryption_utils.py +0 -42
  121. fides/service/manual_tasks/__init__.py +0 -0
  122. fides/service/manual_tasks/manual_task_service.py +0 -150
  123. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-c583a61302f02add.js +0 -1
  124. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-20d20a8d1736f7c4.js +0 -1
  125. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-0e557d79e1e43c2b.js +0 -1
  126. {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/WHEEL +0 -0
  127. {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/entry_points.txt +0 -0
  128. {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/licenses/LICENSE +0 -0
  129. {ethyca_fides-2.63.1b4.dist-info → ethyca_fides-2.63.1rc0.dist-info}/top_level.txt +0 -0
  130. /fides/ui-build/static/admin/_next/static/{X2nvWLg2_-vsCTkhSWpzw → PEElhfUdgE5bJjiyu5QCD}/_ssgManifest.js +0 -0
@@ -1,151 +0,0 @@
1
- from datetime import datetime, timezone
2
- from enum import Enum as EnumType
3
- from typing import Optional
4
-
5
- from sqlalchemy.orm import Session
6
-
7
-
8
- class StatusTransitionNotAllowed(Exception):
9
- """Exception raised when a status transition is not allowed."""
10
-
11
- def __init__(self, message: str):
12
- self.message = message
13
- super().__init__(self.message)
14
-
15
-
16
- class StatusType(str, EnumType):
17
- """Enum for manual task status."""
18
-
19
- pending = "pending"
20
- in_progress = "in_progress"
21
- completed = "completed"
22
- failed = "failed"
23
-
24
- @classmethod
25
- def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
26
- """Get valid transitions from the current status.
27
-
28
- Args:
29
- current_status: The current status
30
-
31
- Returns:
32
- list[StatusType]: List of valid transitions
33
- """
34
- if current_status == cls.pending:
35
- return [cls.in_progress, cls.failed]
36
- if current_status == cls.in_progress:
37
- return [cls.completed, cls.failed]
38
- if current_status == cls.completed:
39
- return []
40
- if current_status == cls.failed:
41
- return [cls.pending, cls.in_progress]
42
- return []
43
-
44
-
45
- class StatusTransitionMixin:
46
- """Mixin for handling status transitions.
47
-
48
- This mixin provides methods for managing status transitions and completion tracking.
49
- It can be used by any model that needs status management.
50
- """
51
-
52
- # These should be overridden by the implementing class
53
- status: StatusType
54
- completed_at: Optional[datetime]
55
- completed_by_id: Optional[str]
56
-
57
- def _get_valid_transitions(self) -> list[StatusType]:
58
- """Get valid transitions from the current status.
59
-
60
- Returns:
61
- list[StatusType]: List of valid transitions
62
- """
63
- return StatusType.get_valid_transitions(self.status)
64
-
65
- def _validate_status_transition(self, new_status: StatusType) -> None:
66
- """Validate that a status transition is allowed.
67
-
68
- Args:
69
- new_status: The new status to transition to
70
-
71
- Raises:
72
- StatusTransitionNotAllowed: If the transition is not allowed
73
- """
74
- # Don't allow transitions to the same status
75
- if new_status == self.status:
76
- raise StatusTransitionNotAllowed(
77
- f"Invalid status transition: already in status {new_status}"
78
- )
79
-
80
- # Get valid transitions for current status
81
- valid_transitions = self._get_valid_transitions()
82
- if new_status not in valid_transitions:
83
- raise StatusTransitionNotAllowed(
84
- f"Invalid status transition from {self.status} to {new_status}. "
85
- f"Valid transitions are: {valid_transitions}"
86
- )
87
-
88
- def update_status(
89
- self, db: Session, new_status: StatusType, user_id: Optional[str] = None
90
- ) -> None:
91
- """Update the status with validation and completion handling.
92
-
93
- Args:
94
- db: Database session
95
- new_status: New status to set
96
- user_id: Optional user ID who is making the change
97
- """
98
- self._validate_status_transition(new_status)
99
-
100
- if new_status == StatusType.completed:
101
- self.completed_at = datetime.now(timezone.utc)
102
- self.completed_by_id = user_id
103
- elif new_status == StatusType.pending:
104
- # Reset completion fields if going back to pending
105
- self.completed_at = None
106
- self.completed_by_id = None
107
-
108
- self.status = new_status
109
- db.add(self)
110
- db.commit()
111
-
112
- def mark_completed(self, db: Session, user_id: str) -> None:
113
- """Mark as completed.
114
-
115
- Args:
116
- db: Database session
117
- user_id: user ID who completed the task
118
- """
119
- self.update_status(db, StatusType.completed, user_id)
120
-
121
- def mark_failed(self, db: Session) -> None:
122
- """Mark as failed."""
123
- self.update_status(db, StatusType.failed)
124
-
125
- def start_progress(self, db: Session) -> None:
126
- """Mark as in progress."""
127
- self.update_status(db, StatusType.in_progress)
128
-
129
- def reset_to_pending(self, db: Session) -> None:
130
- """Reset to pending status."""
131
- self.update_status(db, StatusType.pending)
132
-
133
- @property
134
- def is_completed(self) -> bool:
135
- """Check if completed."""
136
- return self.status == StatusType.completed
137
-
138
- @property
139
- def is_failed(self) -> bool:
140
- """Check if failed."""
141
- return self.status == StatusType.failed
142
-
143
- @property
144
- def is_in_progress(self) -> bool:
145
- """Check if in progress."""
146
- return self.status == StatusType.in_progress
147
-
148
- @property
149
- def is_pending(self) -> bool:
150
- """Check if pending."""
151
- return self.status == StatusType.pending
@@ -1,132 +0,0 @@
1
- import time as time_module
2
- from dataclasses import dataclass
3
- from typing import Any, Dict, Iterator, List, Optional, Tuple
4
-
5
- from loguru import logger
6
-
7
- from fides.api.models.attachment import Attachment, AttachmentType
8
- from fides.api.schemas.storage.storage import StorageDetails
9
-
10
-
11
- @dataclass
12
- class AttachmentData:
13
- """Data structure for attachment metadata and content.
14
- Using a dataclass rather than a Pydantic model here for the following reasons:
15
- - The data structure is simple and doesn't need complex validation.
16
- - The fields being used have already been validated and are properly typed.
17
- - The class is used internally for data transfer, not for API serialization.
18
- - Performance is important since this is used in a data processing pipeline.
19
- """
20
-
21
- file_name: str
22
- file_size: Optional[int]
23
- download_url: Optional[str]
24
- content_type: str
25
- bucket_name: str
26
- file_key: str
27
- storage_key: str
28
-
29
- def to_upload_dict(self) -> Dict[str, Any]:
30
- """Convert to dictionary for upload, including presigned URL."""
31
- return {
32
- "file_name": self.file_name,
33
- "file_size": self.file_size,
34
- "download_url": self.download_url,
35
- "content_type": self.content_type,
36
- }
37
-
38
- def to_storage_dict(self) -> Dict[str, Any]:
39
- """Convert to dictionary for storage, including the elements needed to recreated the presigned URL."""
40
- return {
41
- "file_name": self.file_name,
42
- "file_size": self.file_size,
43
- "content_type": self.content_type,
44
- "bucket_name": self.bucket_name,
45
- "file_key": self.file_key,
46
- "storage_key": self.storage_key,
47
- }
48
-
49
-
50
- def get_attachments_content(
51
- loaded_attachments: List[Attachment],
52
- ) -> Iterator[AttachmentData]:
53
- """
54
- Retrieves all attachments associated with a privacy request that are marked to be included with the access package.
55
- Yields AttachmentData objects containing attachment metadata and download urls.
56
- Uses generators to minimize memory usage.
57
-
58
- Args:
59
- loaded_attachments: List of Attachment objects to process
60
-
61
- Yields:
62
- AttachmentData object containing attachment metadata and url
63
- """
64
- start_time = time_module.time()
65
- processed_count = 0
66
- skipped_count = 0
67
- error_count = 0
68
- total_size = 0
69
-
70
- for attachment in loaded_attachments:
71
- if attachment.attachment_type != AttachmentType.include_with_access_package:
72
- skipped_count += 1
73
- continue
74
-
75
- try:
76
- # Get size and download URL using retrieve_attachment
77
- size, url = attachment.retrieve_attachment()
78
- total_size += size if size else 0
79
- if url is None:
80
- logger.warning(
81
- "No download URL retrieved for attachment {}", attachment.file_name
82
- )
83
- skipped_count += 1
84
- continue
85
-
86
- processed_count += 1
87
- yield AttachmentData(
88
- file_name=attachment.file_name,
89
- file_size=size,
90
- download_url=str(url) if url else None,
91
- content_type=attachment.content_type,
92
- bucket_name=attachment.config.details[StorageDetails.BUCKET.value],
93
- file_key=attachment.file_key,
94
- storage_key=attachment.storage_key,
95
- )
96
-
97
- except Exception as e:
98
- error_count += 1
99
- logger.error(
100
- "Error processing attachment {}: {}", attachment.file_name, str(e)
101
- )
102
- continue
103
-
104
- # Log final metrics
105
- time_taken = time_module.time() - start_time
106
- logger.bind(
107
- time_to_process=time_taken,
108
- total_attachments=len(loaded_attachments),
109
- processed_attachments=processed_count,
110
- skipped_attachments=skipped_count,
111
- error_attachments=error_count,
112
- total_size_bytes=total_size,
113
- ).info("Attachment processing complete")
114
-
115
-
116
- def process_attachments_for_upload(
117
- attachments: Iterator[AttachmentData],
118
- ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
119
- """
120
- Process attachments into separate upload and storage formats.
121
- Returns both formats:
122
- - upload_attachments: Used for uploading to access packages
123
- - storage_attachments: Used for saving filtered access results
124
- """
125
- upload_attachments = []
126
- storage_attachments = []
127
-
128
- for attachment in attachments:
129
- storage_attachments.append(attachment.to_storage_dict())
130
- upload_attachments.append(attachment.to_upload_dict())
131
-
132
- return upload_attachments, storage_attachments
@@ -1,33 +0,0 @@
1
- <html>
2
- <head>
3
- <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600">
4
- <link rel="stylesheet" href="../data/main.css">
5
- </head>
6
- <body>
7
- <div class="container">
8
- <div class="header"></div>
9
- <div class="content">
10
- <div class="button-container">
11
- <a href="../welcome.html">
12
- <div class="button"><img src="../data/back.svg"></div>
13
- <span>Back to main page</span>
14
- </a>
15
- </div>
16
- <h1>Attachments</h1>
17
- <p class="expiration-notice">Note: All download links will expire in 7 days.</p>
18
- <div class="table table-hover">
19
- <div class="table-row">
20
- <div class="table-cell" style="text-align: left;">File Name</div>
21
- <div class="table-cell" style="text-align: left;">Size</div>
22
- </div>
23
- {% for name, info in data.items() %}
24
- <a href="{{ info.url }}" class="table-row" target="_blank">
25
- <div class="table-cell" style="text-align: left;">{{ name }}</div>
26
- <div class="table-cell" style="text-align: left;">{{ info.size }}</div>
27
- </a>
28
- {% endfor %}
29
- </div>
30
- </div>
31
- </div>
32
- </body>
33
- </html>
@@ -1,170 +0,0 @@
1
- import zipfile
2
- from io import BytesIO
3
- from typing import Any, Optional
4
-
5
- import pandas as pd
6
-
7
- from fides.api.tasks.encryption_utils import encrypt_access_request_results
8
- from fides.config import CONFIG
9
-
10
-
11
- def create_csv_from_dataframe(df: pd.DataFrame) -> BytesIO:
12
- """Create a CSV file from a pandas DataFrame.
13
-
14
- Args:
15
- df: The DataFrame to convert to CSV
16
-
17
- Returns:
18
- BytesIO: A file-like object containing the CSV data
19
- """
20
- buffer = BytesIO()
21
- df.to_csv(buffer, index=False, encoding=CONFIG.security.encoding)
22
- buffer.seek(0)
23
- return buffer
24
-
25
-
26
- def create_attachment_csv(attachments: list[dict[str, Any]]) -> Optional[BytesIO]:
27
- """Create a CSV file containing attachment metadata.
28
-
29
- Args:
30
- attachments: List of attachment dictionaries
31
- privacy_request_id: The ID of the privacy request for encryption
32
-
33
- Returns:
34
- Optional[BytesIO]: A file-like object containing the CSV data, or None if no attachments
35
- """
36
- if not attachments:
37
- return None
38
-
39
- # Filter out invalid attachments and create a list of valid ones
40
- valid_attachments = []
41
- for a in attachments:
42
- if not isinstance(a, dict):
43
- continue
44
-
45
- # Check if the attachment has at least one of the required fields
46
- if not any(
47
- key in a
48
- for key in ["file_name", "file_size", "content_type", "download_url"]
49
- ):
50
- continue
51
-
52
- valid_attachments.append(
53
- {
54
- "file_name": a.get("file_name", ""),
55
- "file_size": a.get("file_size", 0),
56
- "content_type": a.get("content_type", "application/octet-stream"),
57
- "download_url": a.get("download_url", ""),
58
- }
59
- )
60
-
61
- # Return None if there are no valid attachments
62
- if not valid_attachments:
63
- return None
64
-
65
- df = pd.DataFrame(valid_attachments)
66
-
67
- if df.empty:
68
- return None
69
-
70
- return create_csv_from_dataframe(df)
71
-
72
-
73
- def _write_attachment_csv(
74
- zip_file: zipfile.ZipFile,
75
- key: str,
76
- idx: int,
77
- attachments: list[dict[str, Any]],
78
- privacy_request_id: str,
79
- ) -> None:
80
- """Write attachment data to a CSV file in the zip archive.
81
-
82
- Args:
83
- zip_file: The zip file to write to
84
- key: The key for the data
85
- idx: The index of the item in the list
86
- attachments: List of attachment dictionaries
87
- privacy_request_id: The ID of the privacy request for encryption
88
- """
89
- buffer = create_attachment_csv(attachments)
90
- if buffer:
91
- zip_file.writestr(
92
- f"{key}/{idx + 1}/attachments.csv",
93
- encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
94
- )
95
-
96
-
97
- def _write_item_csv(
98
- zip_file: zipfile.ZipFile,
99
- key: str,
100
- items: list[dict[str, Any]],
101
- privacy_request_id: str,
102
- ) -> None:
103
- """Write item data to a CSV file in the zip archive.
104
-
105
- Args:
106
- zip_file: The zip file to write to
107
- key: The key for the data
108
- items: List of items to write
109
- privacy_request_id: The ID of the privacy request for encryption
110
- """
111
- if items:
112
- df = pd.DataFrame(items)
113
- buffer = create_csv_from_dataframe(df)
114
- zip_file.writestr(
115
- f"{key}.csv",
116
- encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
117
- )
118
-
119
-
120
- def _write_simple_csv(
121
- zip_file: zipfile.ZipFile,
122
- key: str,
123
- value: Any,
124
- privacy_request_id: str,
125
- ) -> None:
126
- """Write simple key-value data to a CSV file in the zip archive.
127
-
128
- Args:
129
- zip_file: The zip file to write to
130
- key: The key for the data
131
- value: The value to write
132
- privacy_request_id: The ID of the privacy request for encryption
133
- """
134
- df = pd.json_normalize({key: value})
135
- buffer = create_csv_from_dataframe(df)
136
- zip_file.writestr(
137
- f"{key}.csv",
138
- encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
139
- )
140
-
141
-
142
- def write_csv_to_zip(
143
- zip_file: zipfile.ZipFile, data: dict[str, Any], privacy_request_id: str
144
- ) -> None:
145
- """Write data to a zip file in CSV format.
146
-
147
- Args:
148
- zip_file: The zip file to write to
149
- data: The data to convert to CSV
150
- privacy_request_id: The ID of the privacy request for encryption
151
- """
152
- for key, value in data.items():
153
- if (
154
- isinstance(value, list)
155
- and value
156
- and all(isinstance(item, dict) for item in value)
157
- ):
158
- # Handle lists of dictionaries
159
- items: list[dict[str, Any]] = []
160
- for item in value:
161
- # Extract attachments if they exist
162
- attachments = item.pop("attachments", [])
163
- if attachments:
164
- _write_attachment_csv(
165
- zip_file, key, len(items), attachments, privacy_request_id
166
- )
167
- items.append(item)
168
- _write_item_csv(zip_file, key, items, privacy_request_id)
169
- else:
170
- _write_simple_csv(zip_file, key, value, privacy_request_id)
@@ -1,42 +0,0 @@
1
- import secrets
2
- from typing import Optional, Union
3
-
4
- from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
5
- from fides.api.util.cache import get_cache, get_encryption_cache_key
6
- from fides.api.util.encryption.aes_gcm_encryption_scheme import (
7
- encrypt_to_bytes_verify_secrets_length,
8
- )
9
- from fides.config import CONFIG
10
-
11
-
12
- def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> str:
13
- """Encrypt data with encryption key if provided, otherwise return unencrypted data.
14
-
15
- Args:
16
- data: The data to encrypt
17
- request_id: The ID of the privacy request for encryption key lookup
18
-
19
- Returns:
20
- str: The encrypted data as a string
21
- """
22
- cache = get_cache()
23
- encryption_cache_key = get_encryption_cache_key(
24
- privacy_request_id=request_id,
25
- encryption_attr="key",
26
- )
27
- if isinstance(data, bytes):
28
- data = data.decode(CONFIG.security.encoding)
29
-
30
- encryption_key: Optional[str] = cache.get(encryption_cache_key)
31
- if not encryption_key:
32
- return data
33
-
34
- bytes_encryption_key: bytes = encryption_key.encode(
35
- encoding=CONFIG.security.encoding
36
- )
37
- nonce: bytes = secrets.token_bytes(CONFIG.security.aes_gcm_nonce_length)
38
- # b64encode the entire nonce and the encrypted message together
39
- return bytes_to_b64_str(
40
- nonce
41
- + encrypt_to_bytes_verify_secrets_length(data, bytes_encryption_key, nonce)
42
- )
File without changes
@@ -1,150 +0,0 @@
1
- from typing import Optional
2
-
3
- from loguru import logger
4
- from sqlalchemy import select
5
- from sqlalchemy.orm import Session
6
-
7
- from fides.api.models.fides_user import FidesUser
8
- from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
9
- from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
10
- from fides.api.schemas.manual_tasks.manual_task_schemas import (
11
- ManualTaskLogStatus,
12
- ManualTaskParentEntityType,
13
- ManualTaskReferenceType,
14
- ManualTaskType,
15
- )
16
-
17
-
18
- class ManualTaskService:
19
- def __init__(self, db: Session):
20
- self.db = db
21
-
22
- def get_task(
23
- self,
24
- task_id: Optional[str] = None,
25
- parent_entity_id: Optional[str] = None,
26
- parent_entity_type: Optional[ManualTaskParentEntityType] = None,
27
- task_type: Optional[ManualTaskType] = None,
28
- ) -> Optional[ManualTask]:
29
- """Get the manual task using provided filters.
30
-
31
- Args:
32
- task_id: The task ID
33
- parent_entity_id: The parent entity ID
34
- parent_entity_type: The parent entity type
35
- task_type: The task type
36
-
37
- Returns:
38
- Optional[ManualTask]: The manual task for the connection, if it exists
39
- """
40
- if not any([task_id, parent_entity_id, parent_entity_type, task_type]):
41
- logger.warning("No filters provided to get_task. Returning None.")
42
- return None
43
-
44
- stmt = select(ManualTask) # type: ignore[arg-type]
45
- if task_id:
46
- stmt = stmt.where(ManualTask.id == task_id)
47
- if parent_entity_id:
48
- stmt = stmt.where(ManualTask.parent_entity_id == parent_entity_id)
49
- if parent_entity_type:
50
- stmt = stmt.where(ManualTask.parent_entity_type == parent_entity_type)
51
- if task_type:
52
- stmt = stmt.where(ManualTask.task_type == task_type)
53
- return self.db.execute(stmt).scalar_one_or_none()
54
-
55
- # User Management
56
- def assign_users_to_task(
57
- self, db: Session, task: ManualTask, user_ids: list[str]
58
- ) -> None:
59
- """Assigns users to this task. We can assign one or more users to a task.
60
-
61
- Args:
62
- db: Database session
63
- task: The task to assign users to
64
- user_ids: List of user IDs to assign
65
- """
66
- user_ids = list(set(user_ids))
67
- if not user_ids:
68
- raise ValueError("User ID is required for assignment")
69
-
70
- # Create new user assignment
71
- for user_id in user_ids:
72
- # if user is already assigned, skip
73
- if user_id in task.assigned_users:
74
- continue
75
- # verify user exists
76
- user = db.query(FidesUser).filter_by(id=user_id).first()
77
- if not user:
78
- ManualTaskLog.create_error_log(
79
- db=db,
80
- task_id=task.id,
81
- message=f"Failed to add user {user_id} to task {task.id}: user does not exist",
82
- details={"user_id": user_id},
83
- )
84
- continue
85
-
86
- ManualTaskReference.create(
87
- db=db,
88
- data={
89
- "task_id": task.id,
90
- "reference_id": user_id,
91
- "reference_type": ManualTaskReferenceType.assigned_user,
92
- },
93
- )
94
-
95
- # Log the user assignment
96
- ManualTaskLog.create_log(
97
- db=db,
98
- task_id=task.id,
99
- status=ManualTaskLogStatus.updated,
100
- message=f"User {user_id} assigned to task",
101
- details={"assigned_user_id": user_id},
102
- )
103
-
104
- def unassign_users_from_task(
105
- self, db: Session, task: ManualTask, user_ids: list[str]
106
- ) -> None:
107
- """Remove the user assignment from this task.
108
-
109
- Args:
110
- db: Database session
111
- task: The task to unassign users from
112
- user_ids: List of user IDs to unassign
113
- """
114
- user_ids = list(set(user_ids))
115
- if not user_ids:
116
- raise ValueError("User ID is required for unassignment")
117
-
118
- # Get references to unassign
119
- references_to_unassign = (
120
- db.query(ManualTaskReference)
121
- .filter(
122
- ManualTaskReference.task_id == task.id,
123
- ManualTaskReference.reference_type
124
- == ManualTaskReferenceType.assigned_user,
125
- ManualTaskReference.reference_id.in_(user_ids),
126
- )
127
- .all()
128
- )
129
-
130
- # Delete references and log unassignments
131
- for ref in references_to_unassign:
132
- ref.delete(db)
133
- ManualTaskLog.create_log(
134
- db=db,
135
- task_id=task.id,
136
- status=ManualTaskLogStatus.updated,
137
- message=f"User {ref.reference_id} unassigned from task",
138
- details={"unassigned_user_id": ref.reference_id},
139
- )
140
-
141
- # Check if any users weren't unassigned
142
- unassigned_user_ids = [ref.reference_id for ref in references_to_unassign]
143
- left_over_user_ids = [
144
- user_id for user_id in user_ids if user_id not in unassigned_user_ids
145
- ]
146
- if left_over_user_ids:
147
- logger.warning(
148
- f"Failed to unassign users {left_over_user_ids} from task {task.id}: "
149
- "users were not assigned to the task"
150
- )