ethyca-fides 2.63.1b3__py2.py3-none-any.whl → 2.63.1rc0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/RECORD +139 -141
  3. fides/_version.py +3 -3
  4. fides/api/alembic/migrations/versions/29e56fa1fdb3_add_monitor_tasks.py +147 -0
  5. fides/api/api/v1/endpoints/privacy_request_endpoints.py +4 -4
  6. fides/api/db/base.py +5 -3
  7. fides/api/main.py +0 -1
  8. fides/api/models/attachment.py +23 -36
  9. fides/api/models/connectionconfig.py +1 -1
  10. fides/api/models/detection_discovery/__init__.py +35 -0
  11. fides/api/models/detection_discovery/monitor_task.py +161 -0
  12. fides/api/models/field_types/__init__.py +5 -0
  13. fides/api/models/field_types/encrypted_large_data.py +151 -0
  14. fides/api/models/privacy_preference.py +1 -1
  15. fides/api/models/privacy_request/execution_log.py +3 -31
  16. fides/api/models/privacy_request/privacy_request.py +16 -3
  17. fides/api/models/privacy_request/request_task.py +36 -25
  18. fides/api/models/worker_task.py +96 -0
  19. fides/api/schemas/external_storage.py +22 -0
  20. fides/api/schemas/privacy_request.py +1 -12
  21. fides/api/service/connectors/base_erasure_email_connector.py +1 -1
  22. fides/api/service/connectors/consent_email_connector.py +2 -1
  23. fides/api/service/connectors/dynamic_erasure_email_connector.py +2 -1
  24. fides/api/service/connectors/erasure_email_connector.py +1 -1
  25. fides/api/service/external_data_storage.py +371 -0
  26. fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +46 -264
  27. fides/api/service/privacy_request/dsr_package/templates/collection_index.html +9 -34
  28. fides/api/service/privacy_request/dsr_package/templates/item.html +37 -0
  29. fides/api/service/privacy_request/dsr_package/templates/main.css +2 -45
  30. fides/api/service/privacy_request/dsr_package/templates/welcome.html +8 -12
  31. fides/api/service/privacy_request/request_runner_service.py +139 -258
  32. fides/api/service/privacy_request/request_service.py +1 -1
  33. fides/api/service/storage/gcs.py +3 -15
  34. fides/api/service/storage/s3.py +14 -28
  35. fides/api/service/storage/util.py +7 -45
  36. fides/api/task/create_request_tasks.py +1 -1
  37. fides/api/task/execute_request_tasks.py +9 -8
  38. fides/api/task/graph_task.py +22 -10
  39. fides/api/tasks/storage.py +91 -85
  40. fides/api/util/cache.py +1 -77
  41. fides/api/util/consent_util.py +1 -1
  42. fides/api/util/data_size.py +102 -0
  43. fides/api/util/encryption/aes_gcm_encryption_util.py +271 -0
  44. fides/config/redis_settings.py +8 -99
  45. fides/service/messaging/aws_ses_service.py +1 -5
  46. fides/service/privacy_request/privacy_request_service.py +1 -1
  47. fides/ui-build/static/admin/404.html +1 -1
  48. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  49. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  50. fides/ui-build/static/admin/add-systems.html +1 -1
  51. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  52. fides/ui-build/static/admin/consent/configure.html +1 -1
  53. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  54. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  55. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  56. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  57. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  58. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  59. fides/ui-build/static/admin/consent/properties.html +1 -1
  60. fides/ui-build/static/admin/consent/reporting.html +1 -1
  61. fides/ui-build/static/admin/consent.html +1 -1
  62. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  63. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  64. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  65. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  66. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  67. fides/ui-build/static/admin/data-catalog.html +1 -1
  68. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  69. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  70. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  71. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  72. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  73. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  74. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  75. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  76. fides/ui-build/static/admin/datamap.html +1 -1
  77. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  78. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  79. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  80. fides/ui-build/static/admin/dataset/new.html +1 -1
  81. fides/ui-build/static/admin/dataset.html +1 -1
  82. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  83. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  84. fides/ui-build/static/admin/datastore-connection.html +1 -1
  85. fides/ui-build/static/admin/index.html +1 -1
  86. fides/ui-build/static/admin/integrations/[id].html +1 -1
  87. fides/ui-build/static/admin/integrations.html +1 -1
  88. fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
  89. fides/ui-build/static/admin/lib/fides-headless.js +1 -1
  90. fides/ui-build/static/admin/lib/fides-preview.js +1 -1
  91. fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
  92. fides/ui-build/static/admin/lib/fides.js +2 -2
  93. fides/ui-build/static/admin/login/[provider].html +1 -1
  94. fides/ui-build/static/admin/login.html +1 -1
  95. fides/ui-build/static/admin/messaging/[id].html +1 -1
  96. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  97. fides/ui-build/static/admin/messaging.html +1 -1
  98. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  99. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  100. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  101. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  102. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  103. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  104. fides/ui-build/static/admin/poc/forms.html +1 -1
  105. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  106. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  107. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  108. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  109. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  110. fides/ui-build/static/admin/privacy-requests.html +1 -1
  111. fides/ui-build/static/admin/properties/[id].html +1 -1
  112. fides/ui-build/static/admin/properties/add-property.html +1 -1
  113. fides/ui-build/static/admin/properties.html +1 -1
  114. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  115. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  116. fides/ui-build/static/admin/settings/about.html +1 -1
  117. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  118. fides/ui-build/static/admin/settings/consent.html +1 -1
  119. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  120. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  121. fides/ui-build/static/admin/settings/domains.html +1 -1
  122. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  123. fides/ui-build/static/admin/settings/locations.html +1 -1
  124. fides/ui-build/static/admin/settings/organization.html +1 -1
  125. fides/ui-build/static/admin/settings/regulations.html +1 -1
  126. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  127. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  128. fides/ui-build/static/admin/systems.html +1 -1
  129. fides/ui-build/static/admin/taxonomy.html +1 -1
  130. fides/ui-build/static/admin/user-management/new.html +1 -1
  131. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  132. fides/ui-build/static/admin/user-management.html +1 -1
  133. fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +0 -160
  134. fides/api/models/manual_tasks/__init__.py +0 -8
  135. fides/api/models/manual_tasks/manual_task.py +0 -110
  136. fides/api/models/manual_tasks/manual_task_log.py +0 -100
  137. fides/api/schemas/manual_tasks/__init__.py +0 -0
  138. fides/api/schemas/manual_tasks/manual_task_schemas.py +0 -79
  139. fides/api/schemas/manual_tasks/manual_task_status.py +0 -151
  140. fides/api/service/privacy_request/attachment_handling.py +0 -132
  141. fides/api/service/privacy_request/dsr_package/templates/attachments_index.html +0 -33
  142. fides/api/tasks/csv_utils.py +0 -170
  143. fides/api/tasks/encryption_utils.py +0 -42
  144. fides/service/manual_tasks/__init__.py +0 -0
  145. fides/service/manual_tasks/manual_task_service.py +0 -150
  146. {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/WHEEL +0 -0
  147. {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/entry_points.txt +0 -0
  148. {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/licenses/LICENSE +0 -0
  149. {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/top_level.txt +0 -0
  150. /fides/api/models/{detection_discovery.py → detection_discovery/core.py} +0 -0
  151. /fides/ui-build/static/admin/_next/static/{ycPcko8qnif6BlkQ6MN4D → PEElhfUdgE5bJjiyu5QCD}/_buildManifest.js +0 -0
  152. /fides/ui-build/static/admin/_next/static/{ycPcko8qnif6BlkQ6MN4D → PEElhfUdgE5bJjiyu5QCD}/_ssgManifest.js +0 -0
@@ -0,0 +1,147 @@
1
+ """add_monitor_tasks
2
+
3
+ Revision ID: 29e56fa1fdb3
4
+ Revises: 5efcdf18438e
5
+ Create Date: 2025-06-11 14:40:08.384571
6
+
7
+ """
8
+
9
+ import sqlalchemy as sa
10
+ from alembic import op
11
+ from sqlalchemy.dialects import postgresql
12
+
13
+ # revision identifiers, used by Alembic.
14
+ revision = "29e56fa1fdb3"
15
+ down_revision = "5efcdf18438e"
16
+ branch_labels = None
17
+ depends_on = None
18
+
19
+
20
+ def upgrade():
21
+ op.create_table(
22
+ "monitortask",
23
+ sa.Column("id", sa.String(length=255), nullable=False),
24
+ sa.Column(
25
+ "created_at",
26
+ sa.DateTime(timezone=True),
27
+ server_default=sa.text("now()"),
28
+ nullable=True,
29
+ ),
30
+ sa.Column(
31
+ "updated_at",
32
+ sa.DateTime(timezone=True),
33
+ server_default=sa.text("now()"),
34
+ nullable=True,
35
+ ),
36
+ sa.Column("action_type", sa.String(), nullable=False),
37
+ sa.Column(
38
+ "status",
39
+ sa.Enum(
40
+ "in_processing",
41
+ "pending",
42
+ "complete",
43
+ "error",
44
+ "paused",
45
+ "retrying",
46
+ "skipped",
47
+ name="executionlogstatus",
48
+ native_enum=False,
49
+ ),
50
+ nullable=False,
51
+ ),
52
+ sa.Column("celery_id", sa.String(length=255), nullable=False),
53
+ sa.Column(
54
+ "task_arguments", postgresql.JSONB(astext_type=sa.Text()), nullable=True
55
+ ),
56
+ sa.Column("message", sa.String(), nullable=True),
57
+ sa.Column("monitor_config_id", sa.String(), nullable=False),
58
+ sa.Column("staged_resource_urns", sa.ARRAY(sa.String()), nullable=True),
59
+ sa.Column("child_resource_urns", sa.ARRAY(sa.String()), nullable=True),
60
+ sa.ForeignKeyConstraint(
61
+ ["monitor_config_id"], ["monitorconfig.id"], ondelete="CASCADE"
62
+ ),
63
+ sa.PrimaryKeyConstraint("id"),
64
+ sa.UniqueConstraint("celery_id"),
65
+ )
66
+ op.create_index(
67
+ op.f("ix_monitortask_action_type"), "monitortask", ["action_type"], unique=False
68
+ )
69
+ op.create_index(op.f("ix_monitortask_id"), "monitortask", ["id"], unique=False)
70
+ op.create_index(
71
+ op.f("ix_monitortask_monitor_config_id"),
72
+ "monitortask",
73
+ ["monitor_config_id"],
74
+ unique=False,
75
+ )
76
+ op.create_index(
77
+ op.f("ix_monitortask_status"), "monitortask", ["status"], unique=False
78
+ )
79
+ op.create_table(
80
+ "monitortaskexecutionlog",
81
+ sa.Column("id", sa.String(length=255), nullable=False),
82
+ sa.Column(
83
+ "status",
84
+ postgresql.ENUM(name="executionlogstatus", create_type=False),
85
+ nullable=False,
86
+ ),
87
+ sa.Column("message", sa.String(), nullable=True),
88
+ sa.Column(
89
+ "created_at",
90
+ sa.DateTime(timezone=True),
91
+ server_default=sa.text("clock_timestamp()"),
92
+ nullable=True,
93
+ ),
94
+ sa.Column(
95
+ "updated_at",
96
+ sa.DateTime(timezone=True),
97
+ server_default=sa.text("clock_timestamp()"),
98
+ nullable=True,
99
+ ),
100
+ sa.Column("celery_id", sa.String(length=255), nullable=False),
101
+ sa.Column("monitor_task_id", sa.String(), nullable=False),
102
+ sa.Column(
103
+ "run_type", sa.Enum("MANUAL", "SYSTEM", name="taskruntype"), nullable=False
104
+ ),
105
+ sa.ForeignKeyConstraint(
106
+ ["monitor_task_id"], ["monitortask.id"], ondelete="CASCADE"
107
+ ),
108
+ sa.PrimaryKeyConstraint("id"),
109
+ )
110
+ op.create_index(
111
+ op.f("ix_monitortaskexecutionlog_id"),
112
+ "monitortaskexecutionlog",
113
+ ["id"],
114
+ unique=False,
115
+ )
116
+ op.create_index(
117
+ op.f("ix_monitortaskexecutionlog_monitor_task_id"),
118
+ "monitortaskexecutionlog",
119
+ ["monitor_task_id"],
120
+ unique=False,
121
+ )
122
+ op.create_index(
123
+ op.f("ix_monitortaskexecutionlog_status"),
124
+ "monitortaskexecutionlog",
125
+ ["status"],
126
+ unique=False,
127
+ )
128
+
129
+
130
+ def downgrade():
131
+ op.drop_index(
132
+ op.f("ix_monitortaskexecutionlog_status"), table_name="monitortaskexecutionlog"
133
+ )
134
+ op.drop_index(
135
+ op.f("ix_monitortaskexecutionlog_monitor_task_id"),
136
+ table_name="monitortaskexecutionlog",
137
+ )
138
+ op.drop_index(
139
+ op.f("ix_monitortaskexecutionlog_id"), table_name="monitortaskexecutionlog"
140
+ )
141
+ op.drop_table("monitortaskexecutionlog")
142
+ op.drop_index(op.f("ix_monitortask_status"), table_name="monitortask")
143
+ op.drop_index(op.f("ix_monitortask_monitor_config_id"), table_name="monitortask")
144
+ op.drop_index(op.f("ix_monitortask_id"), table_name="monitortask")
145
+ op.drop_index(op.f("ix_monitortask_action_type"), table_name="monitortask")
146
+ op.drop_table("monitortask")
147
+ op.execute("DROP TYPE IF EXISTS taskruntype")
@@ -75,6 +75,7 @@ from fides.api.models.privacy_request import (
75
75
  ProvidedIdentity,
76
76
  RequestTask,
77
77
  )
78
+ from fides.api.models.worker_task import ExecutionLogStatus
78
79
  from fides.api.oauth.utils import (
79
80
  verify_callback_oauth_policy_pre_webhook,
80
81
  verify_callback_oauth_pre_approval_webhook,
@@ -91,7 +92,6 @@ from fides.api.schemas.privacy_request import (
91
92
  CheckpointActionRequired,
92
93
  DenyPrivacyRequests,
93
94
  ExecutionLogDetailResponse,
94
- ExecutionLogStatus,
95
95
  FilteredPrivacyRequestResults,
96
96
  LogEntry,
97
97
  ManualWebhookData,
@@ -1940,16 +1940,16 @@ def request_task_async_callback(
1940
1940
  ]:
1941
1941
  raise HTTPException(
1942
1942
  status_code=HTTP_400_BAD_REQUEST,
1943
- detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
1943
+ detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
1944
1944
  )
1945
1945
  if request_task.status != ExecutionLogStatus.awaiting_processing:
1946
1946
  raise HTTPException(
1947
1947
  status_code=HTTP_400_BAD_REQUEST,
1948
- detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with request task status '{request_task.status.value}'",
1948
+ detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with request task status '{request_task.status.value}'",
1949
1949
  )
1950
1950
  logger.info(
1951
1951
  "Callback received for {} task {} {}",
1952
- request_task.action_type.value,
1952
+ request_task.action_type,
1953
1953
  request_task.collection_address,
1954
1954
  request_task.id,
1955
1955
  )
fides/api/db/base.py CHANGED
@@ -16,7 +16,11 @@ from fides.api.models.custom_connector_template import CustomConnectorTemplate
16
16
  from fides.api.models.custom_report import CustomReport
17
17
  from fides.api.models.datasetconfig import DatasetConfig
18
18
  from fides.api.models.db_cache import DBCache
19
- from fides.api.models.detection_discovery import MonitorConfig, StagedResource
19
+ from fides.api.models.detection_discovery.core import MonitorConfig, StagedResource
20
+ from fides.api.models.detection_discovery.monitor_task import (
21
+ MonitorTask,
22
+ MonitorTaskExecutionLog,
23
+ )
20
24
  from fides.api.models.experience_notices import ExperienceNotices
21
25
  from fides.api.models.fides_cloud import FidesCloud
22
26
  from fides.api.models.fides_user import FidesUser
@@ -27,8 +31,6 @@ from fides.api.models.fides_user_respondent_email_verification import (
27
31
  )
28
32
  from fides.api.models.identity_salt import IdentitySalt
29
33
  from fides.api.models.location_regulation_selections import LocationRegulationSelections
30
- from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
31
- from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
32
34
  from fides.api.models.manual_webhook import AccessManualWebhook
33
35
  from fides.api.models.messaging import MessagingConfig
34
36
  from fides.api.models.messaging_template import MessagingTemplate
fides/api/main.py CHANGED
@@ -244,7 +244,6 @@ async def log_request(request: Request, call_next: Callable) -> Response:
244
244
  status_code=response.status_code,
245
245
  handler_time=f"{total_time}ms",
246
246
  path=request.url.path,
247
- fides_client=request.headers.get("Fides-Client", "unknown"),
248
247
  ).info("Request received")
249
248
  return response
250
249
 
@@ -1,6 +1,5 @@
1
1
  import os
2
2
  from enum import Enum as EnumType
3
- from io import BytesIO
4
3
  from typing import IO, TYPE_CHECKING, Any, Tuple
5
4
 
6
5
  from fideslang.validation import AnyHttpUrlString
@@ -20,6 +19,7 @@ from fides.api.service.storage.s3 import (
20
19
  generic_upload_to_s3,
21
20
  )
22
21
  from fides.api.service.storage.util import AllowedFileType, get_local_filename
22
+ from fides.config import CONFIG
23
23
 
24
24
  if TYPE_CHECKING:
25
25
  from fides.api.models.comment import Comment
@@ -28,11 +28,6 @@ if TYPE_CHECKING:
28
28
  from fides.api.models.storage import StorageConfig
29
29
 
30
30
 
31
- # This is 7 days in seconds and is currently the max allowed
32
- # configurable expiration time for presigned URLs for both s3 and gcs.
33
- MAX_TTL_SECONDS = 604800
34
-
35
-
36
31
  class AttachmentType(str, EnumType):
37
32
  """
38
33
  Enum for attachment types. Indicates attachment usage.
@@ -127,11 +122,6 @@ class Attachment(Base):
127
122
  """Returns the content type of the attachment."""
128
123
  return AllowedFileType[self.file_name.split(".")[-1]].value
129
124
 
130
- @property
131
- def file_key(self) -> str:
132
- """Returns the file key of the attachment."""
133
- return f"{self.id}/{self.file_name}"
134
-
135
125
  def upload(self, attachment: IO[bytes]) -> None:
136
126
  """Uploads an attachment to S3, GCS, or local storage."""
137
127
  if self.config.type == StorageType.s3:
@@ -140,7 +130,7 @@ class Attachment(Base):
140
130
  generic_upload_to_s3(
141
131
  storage_secrets=self.config.secrets,
142
132
  bucket_name=bucket_name,
143
- file_key=self.file_key,
133
+ file_key=f"{self.id}/{self.file_name}",
144
134
  document=attachment,
145
135
  auth_method=auth_method,
146
136
  )
@@ -152,7 +142,7 @@ class Attachment(Base):
152
142
  auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
153
143
  storage_client = get_gcs_client(auth_method, self.config.secrets)
154
144
  bucket = storage_client.bucket(bucket_name)
155
- blob = bucket.blob(self.file_key)
145
+ blob = bucket.blob(f"{self.id}/{self.file_name}")
156
146
 
157
147
  # Reset the file pointer to the beginning
158
148
  try:
@@ -165,7 +155,7 @@ class Attachment(Base):
165
155
  return
166
156
 
167
157
  if self.config.type == StorageType.local:
168
- filename = get_local_filename(self.file_key)
158
+ filename = get_local_filename(f"{self.id}/{self.file_name}")
169
159
 
170
160
  # Validate that attachment is a file-like object
171
161
  if not hasattr(attachment, "read"):
@@ -215,10 +205,9 @@ class Attachment(Base):
215
205
  size, url = generic_retrieve_from_s3(
216
206
  storage_secrets=self.config.secrets,
217
207
  bucket_name=bucket_name,
218
- file_key=self.file_key,
208
+ file_key=f"{self.id}/{self.file_name}",
219
209
  auth_method=auth_method,
220
210
  get_content=False,
221
- ttl_seconds=MAX_TTL_SECONDS,
222
211
  )
223
212
  return size, url
224
213
 
@@ -227,20 +216,19 @@ class Attachment(Base):
227
216
  auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
228
217
  storage_client = get_gcs_client(auth_method, self.config.secrets)
229
218
  bucket = storage_client.bucket(bucket_name)
230
- blob = bucket.blob(self.file_key)
219
+ blob = bucket.blob(f"{self.id}/{self.file_name}")
231
220
 
232
221
  # Ensure we have the blob metadata
233
222
  blob.reload()
234
- # Expiration is set to 7 days
235
223
  url = blob.generate_signed_url(
236
224
  version="v4",
237
- expiration=MAX_TTL_SECONDS,
225
+ expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
238
226
  method="GET",
239
227
  )
240
228
  return blob.size, url
241
229
 
242
230
  if self.config.type == StorageType.local:
243
- filename = get_local_filename(self.file_key)
231
+ filename = get_local_filename(f"{self.id}/{self.file_name}")
244
232
  size = os.path.getsize(filename)
245
233
  return size, filename
246
234
 
@@ -248,7 +236,7 @@ class Attachment(Base):
248
236
 
249
237
  def retrieve_attachment_content(
250
238
  self,
251
- ) -> Tuple[int, IO[bytes]]:
239
+ ) -> Tuple[int, bytes]:
252
240
  """
253
241
  Retrieves the size of the attachment and its actual content.
254
242
  - For s3:
@@ -264,33 +252,30 @@ class Attachment(Base):
264
252
  if self.config.type == StorageType.s3:
265
253
  bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
266
254
  auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
267
- size, fileobj = generic_retrieve_from_s3(
255
+ size, content = generic_retrieve_from_s3(
268
256
  storage_secrets=self.config.secrets,
269
257
  bucket_name=bucket_name,
270
- file_key=self.file_key,
258
+ file_key=f"{self.id}/{self.file_name}",
271
259
  auth_method=auth_method,
272
260
  get_content=True,
273
261
  )
274
- return size, fileobj
262
+ return size, content
275
263
 
276
264
  if self.config.type == StorageType.gcs:
277
265
  bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
278
266
  auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
279
267
  storage_client = get_gcs_client(auth_method, self.config.secrets)
280
268
  bucket = storage_client.bucket(bucket_name)
281
- blob = bucket.blob(self.file_key)
269
+ blob = bucket.blob(f"{self.id}/{self.file_name}")
282
270
 
283
- fileobj = BytesIO()
284
- blob.download_to_file(fileobj)
285
- fileobj.seek(0) # Reset pointer to beginning after download
286
- return blob.size, fileobj
271
+ content = blob.download_as_bytes()
272
+ return len(content), content
287
273
 
288
274
  if self.config.type == StorageType.local:
289
- filename = get_local_filename(self.file_key)
290
- size = os.path.getsize(filename)
291
- with open(filename, "rb") as fileobj:
292
- content = BytesIO(fileobj.read())
293
- content.seek(0) # Reset pointer to beginning
275
+ filename = get_local_filename(f"{self.id}/{self.file_name}")
276
+ with open(filename, "rb") as file:
277
+ content = file.read()
278
+ size = len(content)
294
279
  return size, content
295
280
 
296
281
  raise ValueError(f"Unsupported storage type: {self.config.type}")
@@ -303,7 +288,7 @@ class Attachment(Base):
303
288
  generic_delete_from_s3(
304
289
  storage_secrets=self.config.secrets,
305
290
  bucket_name=bucket_name,
306
- file_key=self.file_key,
291
+ file_key=f"{self.id}/{self.file_name}",
307
292
  auth_method=auth_method,
308
293
  )
309
294
  return
@@ -322,7 +307,9 @@ class Attachment(Base):
322
307
  return
323
308
 
324
309
  if self.config.type == StorageType.local:
325
- folder_path = os.path.dirname(get_local_filename(self.file_key))
310
+ folder_path = os.path.dirname(
311
+ get_local_filename(f"{self.id}/{self.file_name}")
312
+ )
326
313
  if os.path.exists(folder_path):
327
314
  import shutil
328
315
 
@@ -23,7 +23,7 @@ from fides.api.schemas.saas.saas_config import SaaSConfig
23
23
  from fides.config import CONFIG
24
24
 
25
25
  if TYPE_CHECKING:
26
- from fides.api.models.detection_discovery import MonitorConfig
26
+ from fides.api.models.detection_discovery.core import MonitorConfig
27
27
  from fides.api.schemas.connection_configuration.enums.system_type import SystemType
28
28
 
29
29
 
@@ -0,0 +1,35 @@
1
+ from .core import (
2
+ DiffStatus,
3
+ MonitorConfig,
4
+ MonitorExecution,
5
+ MonitorFrequency,
6
+ SharedMonitorConfig,
7
+ StagedResource,
8
+ StagedResourceAncestor,
9
+ fetch_staged_resources_by_type_query,
10
+ )
11
+ from .monitor_task import (
12
+ MonitorTask,
13
+ MonitorTaskExecutionLog,
14
+ MonitorTaskType,
15
+ TaskRunType,
16
+ create_monitor_task_with_execution_log,
17
+ update_monitor_task_with_execution_log,
18
+ )
19
+
20
+ __all__ = [
21
+ "DiffStatus",
22
+ "MonitorConfig",
23
+ "MonitorExecution",
24
+ "MonitorFrequency",
25
+ "SharedMonitorConfig",
26
+ "StagedResource",
27
+ "StagedResourceAncestor",
28
+ "fetch_staged_resources_by_type_query",
29
+ "MonitorTask",
30
+ "MonitorTaskExecutionLog",
31
+ "MonitorTaskType",
32
+ "TaskRunType",
33
+ "create_monitor_task_with_execution_log",
34
+ "update_monitor_task_with_execution_log",
35
+ ]
@@ -0,0 +1,161 @@
1
+ from __future__ import annotations
2
+
3
+ from enum import Enum
4
+ from typing import List, Optional
5
+
6
+ from sqlalchemy import ARRAY, Column
7
+ from sqlalchemy import Enum as SQLAlchemyEnum
8
+ from sqlalchemy import ForeignKey, String
9
+ from sqlalchemy.dialects.postgresql import JSONB
10
+ from sqlalchemy.orm import Session, relationship
11
+
12
+ from fides.api.db.base_class import Base, FidesBase # type: ignore[attr-defined]
13
+ from fides.api.models.detection_discovery.core import MonitorConfig
14
+ from fides.api.models.worker_task import (
15
+ ExecutionLogStatus,
16
+ TaskExecutionLog,
17
+ WorkerTask,
18
+ )
19
+
20
+
21
+ class MonitorTaskType(Enum):
22
+ """
23
+ Types of tasks that can be executed by a worker.
24
+ """
25
+
26
+ DETECTION = "detection"
27
+ CLASSIFICATION = "classification"
28
+ PROMOTION = "promotion"
29
+
30
+
31
+ class MonitorTask(WorkerTask, Base):
32
+ """
33
+ A monitor task executed by a worker.
34
+ """
35
+
36
+ # celery_id is used to track task executions. While MonitorTask.id remains constant,
37
+ # celery_id changes with each execution or retry of the task, allowing us to track
38
+ # the current execution state while maintaining a stable reference to the original task.
39
+ celery_id = Column(
40
+ String(255), unique=True, nullable=False, default=FidesBase.generate_uuid
41
+ )
42
+ task_arguments = Column(JSONB, nullable=True) # To be able to rerun the task
43
+ # Contains info, warning, or error messages
44
+ message = Column(String)
45
+ monitor_config_id = Column(
46
+ String,
47
+ ForeignKey(MonitorConfig.id_field_path, ondelete="CASCADE"),
48
+ index=True,
49
+ nullable=False,
50
+ )
51
+ staged_resource_urns = Column(ARRAY(String), nullable=True)
52
+ child_resource_urns = Column(ARRAY(String), nullable=True)
53
+
54
+ monitor_config = relationship(MonitorConfig, cascade="all, delete")
55
+ execution_logs = relationship(
56
+ "MonitorTaskExecutionLog", back_populates="monitor_task", cascade="all, delete"
57
+ )
58
+
59
+ @classmethod
60
+ def allowed_action_types(cls) -> List[str]:
61
+ return [e.value for e in MonitorTaskType]
62
+
63
+
64
+ class TaskRunType(Enum):
65
+ """
66
+ Type of task run.
67
+ """
68
+
69
+ MANUAL = "manual"
70
+ SYSTEM = "system"
71
+
72
+
73
+ class MonitorTaskExecutionLog(TaskExecutionLog, Base):
74
+ """
75
+ Stores the individual execution logs associated with a MonitorTask.
76
+ """
77
+
78
+ # This celery_id preserves the specific execution ID for historical tracking,
79
+ # unlike MonitorTask.celery_id which is updated with each execution.
80
+ # This allows us to maintain a complete history of all task execution attempts.
81
+ celery_id = Column(String(255), nullable=False)
82
+ monitor_task_id = Column(
83
+ String,
84
+ ForeignKey(MonitorTask.id_field_path, ondelete="CASCADE"),
85
+ index=True,
86
+ nullable=False,
87
+ )
88
+ run_type = Column(
89
+ SQLAlchemyEnum(TaskRunType), nullable=False, default=TaskRunType.SYSTEM
90
+ )
91
+
92
+ monitor_task = relationship("MonitorTask", back_populates="execution_logs")
93
+
94
+
95
+ def create_monitor_task_with_execution_log(
96
+ db: Session, monitor_task_data: dict
97
+ ) -> MonitorTask:
98
+ """
99
+ Creates a monitor task with an execution log.
100
+ The default status is pending for the task and pending for the execution log.
101
+ """
102
+ status = ExecutionLogStatus.pending
103
+ task_record = MonitorTask( # type: ignore
104
+ status=status.value,
105
+ **monitor_task_data,
106
+ )
107
+ db.add(task_record)
108
+ db.flush()
109
+
110
+ execution_log = MonitorTaskExecutionLog( # type: ignore
111
+ monitor_task=task_record, celery_id=task_record.celery_id, status=status
112
+ )
113
+ db.add(execution_log)
114
+
115
+ db.commit()
116
+ db.refresh(task_record)
117
+ return task_record
118
+
119
+
120
+ def update_monitor_task_with_execution_log(
121
+ db: Session,
122
+ status: ExecutionLogStatus,
123
+ task_record: Optional[MonitorTask] = None,
124
+ celery_id: Optional[str] = None,
125
+ message: Optional[str] = None,
126
+ run_type: TaskRunType = TaskRunType.SYSTEM,
127
+ ) -> MonitorTask:
128
+ """
129
+ Updates a monitor task with an execution log.
130
+
131
+ It must be either celery_id or task_record. If it doesn't receive a celery_id, it's assumed a new one needs to be created because a new run is about to be performed.
132
+ If it receives a celery_id, it means it only needs to update the status of an existing run. It can receive task_record to avoid querying the database again to get it.
133
+ """
134
+ if not celery_id and not task_record:
135
+ raise ValueError("Either celery_id or task_record must be provided")
136
+
137
+ if celery_id and not task_record:
138
+ task_record = MonitorTask.get_by(db=db, field="celery_id", value=celery_id)
139
+ if not task_record:
140
+ raise ValueError(f"Could not find MonitorTask with celery_id {celery_id}")
141
+
142
+ assert task_record is not None # help type checker understand the control flow
143
+
144
+ if not celery_id:
145
+ celery_id = task_record.generate_uuid()
146
+ task_record.celery_id = celery_id
147
+
148
+ task_record.status = status.value # type: ignore
149
+ task_record.message = message
150
+
151
+ MonitorTaskExecutionLog( # type: ignore
152
+ monitor_task=task_record,
153
+ status=status,
154
+ message=message,
155
+ celery_id=celery_id,
156
+ run_type=run_type,
157
+ )
158
+
159
+ db.commit()
160
+ db.refresh(task_record)
161
+ return task_record
@@ -0,0 +1,5 @@
1
+ from .encrypted_large_data import EncryptedLargeDataDescriptor
2
+
3
+ __all__ = [
4
+ "EncryptedLargeDataDescriptor",
5
+ ]