ethyca-fides 2.63.0rc3__py2.py3-none-any.whl → 2.63.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/RECORD +129 -110
  3. fides/_version.py +3 -3
  4. fides/api/alembic/migrations/versions/29e56fa1fdb3_add_monitor_tasks.py +147 -0
  5. fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +160 -0
  6. fides/api/api/v1/endpoints/privacy_request_endpoints.py +4 -4
  7. fides/api/db/base.py +7 -1
  8. fides/api/models/connectionconfig.py +1 -1
  9. fides/api/models/detection_discovery/__init__.py +35 -0
  10. fides/api/models/detection_discovery/monitor_task.py +162 -0
  11. fides/api/models/field_types/__init__.py +5 -0
  12. fides/api/models/field_types/encrypted_large_data.py +151 -0
  13. fides/api/models/manual_tasks/__init__.py +8 -0
  14. fides/api/models/manual_tasks/manual_task.py +110 -0
  15. fides/api/models/manual_tasks/manual_task_log.py +100 -0
  16. fides/api/models/privacy_preference.py +1 -1
  17. fides/api/models/privacy_request/execution_log.py +3 -31
  18. fides/api/models/privacy_request/privacy_request.py +16 -3
  19. fides/api/models/privacy_request/request_task.py +36 -25
  20. fides/api/models/worker_task.py +96 -0
  21. fides/api/schemas/external_storage.py +22 -0
  22. fides/api/schemas/manual_tasks/__init__.py +0 -0
  23. fides/api/schemas/manual_tasks/manual_task_schemas.py +79 -0
  24. fides/api/schemas/manual_tasks/manual_task_status.py +151 -0
  25. fides/api/schemas/privacy_request.py +1 -12
  26. fides/api/service/connectors/base_erasure_email_connector.py +1 -1
  27. fides/api/service/connectors/consent_email_connector.py +2 -1
  28. fides/api/service/connectors/dynamic_erasure_email_connector.py +2 -1
  29. fides/api/service/connectors/erasure_email_connector.py +1 -1
  30. fides/api/service/external_data_storage.py +371 -0
  31. fides/api/service/privacy_request/request_runner_service.py +5 -5
  32. fides/api/service/privacy_request/request_service.py +1 -1
  33. fides/api/task/create_request_tasks.py +1 -1
  34. fides/api/task/execute_request_tasks.py +9 -8
  35. fides/api/task/graph_task.py +22 -10
  36. fides/api/util/consent_util.py +1 -1
  37. fides/api/util/data_size.py +102 -0
  38. fides/api/util/encryption/aes_gcm_encryption_util.py +271 -0
  39. fides/service/manual_tasks/__init__.py +0 -0
  40. fides/service/manual_tasks/manual_task_service.py +150 -0
  41. fides/service/privacy_request/privacy_request_service.py +1 -1
  42. fides/ui-build/static/admin/404.html +1 -1
  43. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  44. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  45. fides/ui-build/static/admin/add-systems.html +1 -1
  46. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  47. fides/ui-build/static/admin/consent/configure.html +1 -1
  48. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  49. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  50. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  51. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  52. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  53. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  54. fides/ui-build/static/admin/consent/properties.html +1 -1
  55. fides/ui-build/static/admin/consent/reporting.html +1 -1
  56. fides/ui-build/static/admin/consent.html +1 -1
  57. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  58. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  59. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  60. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  61. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  62. fides/ui-build/static/admin/data-catalog.html +1 -1
  63. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  64. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  65. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  66. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  67. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  68. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  69. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  70. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  71. fides/ui-build/static/admin/datamap.html +1 -1
  72. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  73. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  74. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  75. fides/ui-build/static/admin/dataset/new.html +1 -1
  76. fides/ui-build/static/admin/dataset.html +1 -1
  77. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  78. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  79. fides/ui-build/static/admin/datastore-connection.html +1 -1
  80. fides/ui-build/static/admin/index.html +1 -1
  81. fides/ui-build/static/admin/integrations/[id].html +1 -1
  82. fides/ui-build/static/admin/integrations.html +1 -1
  83. fides/ui-build/static/admin/login/[provider].html +1 -1
  84. fides/ui-build/static/admin/login.html +1 -1
  85. fides/ui-build/static/admin/messaging/[id].html +1 -1
  86. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  87. fides/ui-build/static/admin/messaging.html +1 -1
  88. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  89. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  90. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  91. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  92. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  93. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  94. fides/ui-build/static/admin/poc/forms.html +1 -1
  95. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  96. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  97. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  98. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  99. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  100. fides/ui-build/static/admin/privacy-requests.html +1 -1
  101. fides/ui-build/static/admin/properties/[id].html +1 -1
  102. fides/ui-build/static/admin/properties/add-property.html +1 -1
  103. fides/ui-build/static/admin/properties.html +1 -1
  104. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  105. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  106. fides/ui-build/static/admin/settings/about.html +1 -1
  107. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  108. fides/ui-build/static/admin/settings/consent.html +1 -1
  109. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  110. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  111. fides/ui-build/static/admin/settings/domains.html +1 -1
  112. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  113. fides/ui-build/static/admin/settings/locations.html +1 -1
  114. fides/ui-build/static/admin/settings/organization.html +1 -1
  115. fides/ui-build/static/admin/settings/regulations.html +1 -1
  116. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  117. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  118. fides/ui-build/static/admin/systems.html +1 -1
  119. fides/ui-build/static/admin/taxonomy.html +1 -1
  120. fides/ui-build/static/admin/user-management/new.html +1 -1
  121. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  122. fides/ui-build/static/admin/user-management.html +1 -1
  123. {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/WHEEL +0 -0
  124. {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/entry_points.txt +0 -0
  125. {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/licenses/LICENSE +0 -0
  126. {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/top_level.txt +0 -0
  127. /fides/api/models/{detection_discovery.py → detection_discovery/core.py} +0 -0
  128. /fides/ui-build/static/admin/_next/static/{XobHpfndIH7IpV30u2vGV → SZn_Fpr_qG1COMjkdloep}/_buildManifest.js +0 -0
  129. /fides/ui-build/static/admin/_next/static/{XobHpfndIH7IpV30u2vGV → SZn_Fpr_qG1COMjkdloep}/_ssgManifest.js +0 -0
@@ -0,0 +1,160 @@
1
+ """add manual task tables
2
+
3
+ Revision ID: 5efcdf18438e
4
+ Revises: c586a56c25e7
5
+ Create Date: 2025-06-04 17:24:00.300170
6
+
7
+ """
8
+
9
+ import sqlalchemy as sa
10
+ from alembic import op
11
+ from sqlalchemy.dialects import postgresql
12
+ from sqlalchemy.sql import func
13
+
14
+ # revision identifiers, used by Alembic.
15
+ revision = "5efcdf18438e"
16
+ down_revision = "c586a56c25e7"
17
+ branch_labels = None
18
+ depends_on = None
19
+
20
+
21
+ def upgrade():
22
+ # ### commands auto generated by Alembic - please adjust! ###
23
+ op.create_table(
24
+ "manual_task",
25
+ sa.Column("id", sa.String(), nullable=False),
26
+ sa.Column(
27
+ "created_at",
28
+ sa.DateTime(timezone=True),
29
+ nullable=False,
30
+ server_default=func.now(),
31
+ ),
32
+ sa.Column(
33
+ "updated_at",
34
+ sa.DateTime(timezone=True),
35
+ nullable=False,
36
+ server_default=func.now(),
37
+ ),
38
+ sa.Column(
39
+ "task_type", sa.String(), nullable=False, server_default="privacy_request"
40
+ ),
41
+ sa.Column("parent_entity_id", sa.String(), nullable=False),
42
+ sa.Column("parent_entity_type", sa.String(), nullable=False),
43
+ sa.Column("due_date", sa.DateTime(timezone=True), nullable=True),
44
+ sa.PrimaryKeyConstraint("id"),
45
+ sa.UniqueConstraint(
46
+ "parent_entity_id",
47
+ "parent_entity_type",
48
+ name="uq_manual_task_parent_entity",
49
+ ),
50
+ )
51
+
52
+ op.create_table(
53
+ "manual_task_reference",
54
+ sa.Column("id", sa.String(), nullable=False),
55
+ sa.Column(
56
+ "created_at",
57
+ sa.DateTime(timezone=True),
58
+ nullable=False,
59
+ server_default=func.now(),
60
+ ),
61
+ sa.Column(
62
+ "updated_at",
63
+ sa.DateTime(timezone=True),
64
+ nullable=False,
65
+ server_default=func.now(),
66
+ ),
67
+ sa.Column("task_id", sa.String(), nullable=False),
68
+ sa.Column("reference_id", sa.String(), nullable=False),
69
+ sa.Column("reference_type", sa.String(), nullable=False),
70
+ sa.ForeignKeyConstraint(
71
+ ["task_id"],
72
+ ["manual_task.id"],
73
+ ondelete="CASCADE",
74
+ ),
75
+ sa.PrimaryKeyConstraint("id"),
76
+ )
77
+
78
+ op.create_table(
79
+ "manual_task_log",
80
+ sa.Column("id", sa.String(), nullable=False),
81
+ sa.Column(
82
+ "created_at",
83
+ sa.DateTime(timezone=True),
84
+ nullable=False,
85
+ server_default=func.now(),
86
+ ),
87
+ sa.Column(
88
+ "updated_at",
89
+ sa.DateTime(timezone=True),
90
+ nullable=False,
91
+ server_default=func.now(),
92
+ ),
93
+ sa.Column("task_id", sa.String(), nullable=False),
94
+ sa.Column("config_id", sa.String(), nullable=True),
95
+ sa.Column("instance_id", sa.String(), nullable=True),
96
+ sa.Column("status", sa.String(), nullable=False),
97
+ sa.Column("message", sa.String(), nullable=True),
98
+ sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
99
+ sa.ForeignKeyConstraint(
100
+ ["task_id"],
101
+ ["manual_task.id"],
102
+ ondelete="CASCADE",
103
+ ),
104
+ sa.PrimaryKeyConstraint("id"),
105
+ )
106
+
107
+ # Create indexes for manual_task
108
+ op.create_index("ix_manual_task_task_type", "manual_task", ["task_type"])
109
+ op.create_index(
110
+ "ix_manual_task_parent_entity",
111
+ "manual_task",
112
+ ["parent_entity_type", "parent_entity_id"],
113
+ )
114
+ op.create_index("ix_manual_task_due_date", "manual_task", ["due_date"])
115
+
116
+ # Create indexes for manual_task_reference
117
+ op.create_index(
118
+ "ix_manual_task_reference_task_id", "manual_task_reference", ["task_id"]
119
+ )
120
+ op.create_index(
121
+ "ix_manual_task_reference_reference",
122
+ "manual_task_reference",
123
+ ["reference_id", "reference_type"],
124
+ )
125
+
126
+ # Create indexes for manual_task_log
127
+ op.create_index("ix_manual_task_log_task_id", "manual_task_log", ["task_id"])
128
+ op.create_index("ix_manual_task_log_config_id", "manual_task_log", ["config_id"])
129
+ op.create_index(
130
+ "ix_manual_task_log_instance_id", "manual_task_log", ["instance_id"]
131
+ )
132
+ op.create_index("ix_manual_task_log_status", "manual_task_log", ["status"])
133
+ op.create_index("ix_manual_task_log_created_at", "manual_task_log", ["created_at"])
134
+
135
+ # ### end Alembic commands ###
136
+
137
+
138
+ def downgrade():
139
+ # ### commands auto generated by Alembic - please adjust! ###
140
+ # Drop indexes first
141
+ op.drop_index("ix_manual_task_log_created_at", table_name="manual_task_log")
142
+ op.drop_index("ix_manual_task_log_status", table_name="manual_task_log")
143
+ op.drop_index("ix_manual_task_log_instance_id", table_name="manual_task_log")
144
+ op.drop_index("ix_manual_task_log_config_id", table_name="manual_task_log")
145
+ op.drop_index("ix_manual_task_log_task_id", table_name="manual_task_log")
146
+ op.drop_index(
147
+ "ix_manual_task_reference_reference", table_name="manual_task_reference"
148
+ )
149
+ op.drop_index(
150
+ "ix_manual_task_reference_task_id", table_name="manual_task_reference"
151
+ )
152
+ op.drop_index("ix_manual_task_due_date", table_name="manual_task")
153
+ op.drop_index("ix_manual_task_parent_entity", table_name="manual_task")
154
+ op.drop_index("ix_manual_task_task_type", table_name="manual_task")
155
+
156
+ # Then drop tables
157
+ op.drop_table("manual_task_log")
158
+ op.drop_table("manual_task_reference")
159
+ op.drop_table("manual_task")
160
+ # ### end Alembic commands ###
@@ -75,6 +75,7 @@ from fides.api.models.privacy_request import (
75
75
  ProvidedIdentity,
76
76
  RequestTask,
77
77
  )
78
+ from fides.api.models.worker_task import ExecutionLogStatus
78
79
  from fides.api.oauth.utils import (
79
80
  verify_callback_oauth_policy_pre_webhook,
80
81
  verify_callback_oauth_pre_approval_webhook,
@@ -91,7 +92,6 @@ from fides.api.schemas.privacy_request import (
91
92
  CheckpointActionRequired,
92
93
  DenyPrivacyRequests,
93
94
  ExecutionLogDetailResponse,
94
- ExecutionLogStatus,
95
95
  FilteredPrivacyRequestResults,
96
96
  LogEntry,
97
97
  ManualWebhookData,
@@ -1940,16 +1940,16 @@ def request_task_async_callback(
1940
1940
  ]:
1941
1941
  raise HTTPException(
1942
1942
  status_code=HTTP_400_BAD_REQUEST,
1943
- detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
1943
+ detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
1944
1944
  )
1945
1945
  if request_task.status != ExecutionLogStatus.awaiting_processing:
1946
1946
  raise HTTPException(
1947
1947
  status_code=HTTP_400_BAD_REQUEST,
1948
- detail=f"Callback failed. Cannot queue {request_task.action_type.value} task '{request_task.id}' with request task status '{request_task.status.value}'",
1948
+ detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with request task status '{request_task.status.value}'",
1949
1949
  )
1950
1950
  logger.info(
1951
1951
  "Callback received for {} task {} {}",
1952
- request_task.action_type.value,
1952
+ request_task.action_type,
1953
1953
  request_task.collection_address,
1954
1954
  request_task.id,
1955
1955
  )
fides/api/db/base.py CHANGED
@@ -16,7 +16,11 @@ from fides.api.models.custom_connector_template import CustomConnectorTemplate
16
16
  from fides.api.models.custom_report import CustomReport
17
17
  from fides.api.models.datasetconfig import DatasetConfig
18
18
  from fides.api.models.db_cache import DBCache
19
- from fides.api.models.detection_discovery import MonitorConfig, StagedResource
19
+ from fides.api.models.detection_discovery.core import MonitorConfig, StagedResource
20
+ from fides.api.models.detection_discovery.monitor_task import (
21
+ MonitorTask,
22
+ MonitorTaskExecutionLog,
23
+ )
20
24
  from fides.api.models.experience_notices import ExperienceNotices
21
25
  from fides.api.models.fides_cloud import FidesCloud
22
26
  from fides.api.models.fides_user import FidesUser
@@ -27,6 +31,8 @@ from fides.api.models.fides_user_respondent_email_verification import (
27
31
  )
28
32
  from fides.api.models.identity_salt import IdentitySalt
29
33
  from fides.api.models.location_regulation_selections import LocationRegulationSelections
34
+ from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
35
+ from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
30
36
  from fides.api.models.manual_webhook import AccessManualWebhook
31
37
  from fides.api.models.messaging import MessagingConfig
32
38
  from fides.api.models.messaging_template import MessagingTemplate
@@ -23,7 +23,7 @@ from fides.api.schemas.saas.saas_config import SaaSConfig
23
23
  from fides.config import CONFIG
24
24
 
25
25
  if TYPE_CHECKING:
26
- from fides.api.models.detection_discovery import MonitorConfig
26
+ from fides.api.models.detection_discovery.core import MonitorConfig
27
27
  from fides.api.schemas.connection_configuration.enums.system_type import SystemType
28
28
 
29
29
 
@@ -0,0 +1,35 @@
1
+ from .core import (
2
+ DiffStatus,
3
+ MonitorConfig,
4
+ MonitorExecution,
5
+ MonitorFrequency,
6
+ SharedMonitorConfig,
7
+ StagedResource,
8
+ StagedResourceAncestor,
9
+ fetch_staged_resources_by_type_query,
10
+ )
11
+ from .monitor_task import (
12
+ MonitorTask,
13
+ MonitorTaskExecutionLog,
14
+ MonitorTaskType,
15
+ TaskRunType,
16
+ create_monitor_task_with_execution_log,
17
+ update_monitor_task_with_execution_log,
18
+ )
19
+
20
+ __all__ = [
21
+ "DiffStatus",
22
+ "MonitorConfig",
23
+ "MonitorExecution",
24
+ "MonitorFrequency",
25
+ "SharedMonitorConfig",
26
+ "StagedResource",
27
+ "StagedResourceAncestor",
28
+ "fetch_staged_resources_by_type_query",
29
+ "MonitorTask",
30
+ "MonitorTaskExecutionLog",
31
+ "MonitorTaskType",
32
+ "TaskRunType",
33
+ "create_monitor_task_with_execution_log",
34
+ "update_monitor_task_with_execution_log",
35
+ ]
@@ -0,0 +1,162 @@
1
+ from __future__ import annotations
2
+
3
+ from enum import Enum
4
+ from typing import List, Optional
5
+
6
+ from sqlalchemy import ARRAY, Column
7
+ from sqlalchemy import Enum as SQLAlchemyEnum
8
+ from sqlalchemy import ForeignKey, String
9
+ from sqlalchemy.dialects.postgresql import JSONB
10
+ from sqlalchemy.orm import Session, relationship
11
+
12
+ from fides.api.db.base_class import Base, FidesBase # type: ignore[attr-defined]
13
+ from fides.api.models.detection_discovery.core import MonitorConfig
14
+ from fides.api.models.worker_task import (
15
+ ExecutionLogStatus,
16
+ TaskExecutionLog,
17
+ WorkerTask,
18
+ )
19
+
20
+
21
+ class MonitorTaskType(Enum):
22
+ """
23
+ Types of tasks that can be executed by a worker.
24
+ """
25
+
26
+ DETECTION = "detection"
27
+ CLASSIFICATION = "classification"
28
+ PROMOTION = "promotion"
29
+ REMOVAL_PROMOTION = "removal_promotion"
30
+
31
+
32
+ class MonitorTask(WorkerTask, Base):
33
+ """
34
+ A monitor task executed by a worker.
35
+ """
36
+
37
+ # celery_id is used to track task executions. While MonitorTask.id remains constant,
38
+ # celery_id changes with each execution or retry of the task, allowing us to track
39
+ # the current execution state while maintaining a stable reference to the original task.
40
+ celery_id = Column(
41
+ String(255), unique=True, nullable=False, default=FidesBase.generate_uuid
42
+ )
43
+ task_arguments = Column(JSONB, nullable=True) # To be able to rerun the task
44
+ # Contains info, warning, or error messages
45
+ message = Column(String)
46
+ monitor_config_id = Column(
47
+ String,
48
+ ForeignKey(MonitorConfig.id_field_path, ondelete="CASCADE"),
49
+ index=True,
50
+ nullable=False,
51
+ )
52
+ staged_resource_urns = Column(ARRAY(String), nullable=True)
53
+ child_resource_urns = Column(ARRAY(String), nullable=True)
54
+
55
+ monitor_config = relationship(MonitorConfig, cascade="all, delete")
56
+ execution_logs = relationship(
57
+ "MonitorTaskExecutionLog", back_populates="monitor_task", cascade="all, delete"
58
+ )
59
+
60
+ @classmethod
61
+ def allowed_action_types(cls) -> List[str]:
62
+ return [e.value for e in MonitorTaskType]
63
+
64
+
65
+ class TaskRunType(Enum):
66
+ """
67
+ Type of task run.
68
+ """
69
+
70
+ MANUAL = "manual"
71
+ SYSTEM = "system"
72
+
73
+
74
+ class MonitorTaskExecutionLog(TaskExecutionLog, Base):
75
+ """
76
+ Stores the individual execution logs associated with a MonitorTask.
77
+ """
78
+
79
+ # This celery_id preserves the specific execution ID for historical tracking,
80
+ # unlike MonitorTask.celery_id which is updated with each execution.
81
+ # This allows us to maintain a complete history of all task execution attempts.
82
+ celery_id = Column(String(255), nullable=False)
83
+ monitor_task_id = Column(
84
+ String,
85
+ ForeignKey(MonitorTask.id_field_path, ondelete="CASCADE"),
86
+ index=True,
87
+ nullable=False,
88
+ )
89
+ run_type = Column(
90
+ SQLAlchemyEnum(TaskRunType), nullable=False, default=TaskRunType.SYSTEM
91
+ )
92
+
93
+ monitor_task = relationship("MonitorTask", back_populates="execution_logs")
94
+
95
+
96
+ def create_monitor_task_with_execution_log(
97
+ db: Session, monitor_task_data: dict
98
+ ) -> MonitorTask:
99
+ """
100
+ Creates a monitor task with an execution log.
101
+ The default status is pending for the task and pending for the execution log.
102
+ """
103
+ status = ExecutionLogStatus.pending
104
+ task_record = MonitorTask( # type: ignore
105
+ status=status.value,
106
+ **monitor_task_data,
107
+ )
108
+ db.add(task_record)
109
+ db.flush()
110
+
111
+ execution_log = MonitorTaskExecutionLog( # type: ignore
112
+ monitor_task=task_record, celery_id=task_record.celery_id, status=status
113
+ )
114
+ db.add(execution_log)
115
+
116
+ db.commit()
117
+ db.refresh(task_record)
118
+ return task_record
119
+
120
+
121
+ def update_monitor_task_with_execution_log(
122
+ db: Session,
123
+ status: ExecutionLogStatus,
124
+ task_record: Optional[MonitorTask] = None,
125
+ celery_id: Optional[str] = None,
126
+ message: Optional[str] = None,
127
+ run_type: TaskRunType = TaskRunType.SYSTEM,
128
+ ) -> MonitorTask:
129
+ """
130
+ Updates a monitor task with an execution log.
131
+
132
+ It must be either celery_id or task_record. If it doesn't receive a celery_id, it's assumed a new one needs to be created because a new run is about to be performed.
133
+ If it receives a celery_id, it means it only needs to update the status of an existing run. It can receive task_record to avoid querying the database again to get it.
134
+ """
135
+ if not celery_id and not task_record:
136
+ raise ValueError("Either celery_id or task_record must be provided")
137
+
138
+ if celery_id and not task_record:
139
+ task_record = MonitorTask.get_by(db=db, field="celery_id", value=celery_id)
140
+ if not task_record:
141
+ raise ValueError(f"Could not find MonitorTask with celery_id {celery_id}")
142
+
143
+ assert task_record is not None # help type checker understand the control flow
144
+
145
+ if not celery_id:
146
+ celery_id = task_record.generate_uuid()
147
+ task_record.celery_id = celery_id
148
+
149
+ task_record.status = status.value # type: ignore
150
+ task_record.message = message
151
+
152
+ MonitorTaskExecutionLog( # type: ignore
153
+ monitor_task=task_record,
154
+ status=status,
155
+ message=message,
156
+ celery_id=celery_id,
157
+ run_type=run_type,
158
+ )
159
+
160
+ db.commit()
161
+ db.refresh(task_record)
162
+ return task_record
@@ -0,0 +1,5 @@
1
+ from .encrypted_large_data import EncryptedLargeDataDescriptor
2
+
3
+ __all__ = [
4
+ "EncryptedLargeDataDescriptor",
5
+ ]
@@ -0,0 +1,151 @@
1
+ from datetime import datetime
2
+ from typing import Any, Optional, Type
3
+
4
+ from loguru import logger
5
+
6
+ from fides.api.api.deps import get_autoclose_db_session
7
+ from fides.api.schemas.external_storage import ExternalStorageMetadata
8
+ from fides.api.service.external_data_storage import (
9
+ ExternalDataStorageError,
10
+ ExternalDataStorageService,
11
+ )
12
+ from fides.api.util.data_size import LARGE_DATA_THRESHOLD_BYTES, calculate_data_size
13
+
14
+
15
+ class EncryptedLargeDataDescriptor:
16
+ """
17
+ A Python descriptor for database fields with encrypted external storage fallback.
18
+
19
+ See the original implementation for detailed docstrings.
20
+ """
21
+
22
+ def __init__(
23
+ self,
24
+ field_name: str,
25
+ empty_default: Optional[Any] = None,
26
+ threshold_bytes: Optional[int] = None,
27
+ ):
28
+ self.field_name = field_name
29
+ self.private_field = f"_{field_name}"
30
+ self.empty_default = empty_default if empty_default is not None else []
31
+ self.threshold_bytes = threshold_bytes or LARGE_DATA_THRESHOLD_BYTES
32
+ self.model_class: Optional[str] = None
33
+ self.name: Optional[str] = None
34
+
35
+ # Descriptor protocol helpers
36
+
37
+ def __set_name__(
38
+ self, owner: Type, name: str
39
+ ) -> None: # noqa: D401 (docstring in orig file)
40
+ self.name = name
41
+ self.model_class = owner.__name__
42
+
43
+ def _generate_storage_path(self, instance: Any) -> str:
44
+ instance_id = getattr(instance, "id", None)
45
+ if not instance_id:
46
+ raise ValueError(f"Instance {instance} must have an 'id' attribute")
47
+ timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S-%f")
48
+ return f"{self.model_class}/{instance_id}/{self.field_name}/{timestamp}.txt"
49
+
50
+ def __get__(self, instance: Any, owner: Type) -> Any: # noqa: D401
51
+ if instance is None:
52
+ return self
53
+ raw_data = getattr(instance, self.private_field)
54
+ if raw_data is None:
55
+ return None
56
+ if isinstance(raw_data, dict) and "storage_type" in raw_data:
57
+ logger.info(
58
+ f"Reading {self.model_class}.{self.field_name} from external storage "
59
+ f"({raw_data.get('storage_type')})"
60
+ )
61
+ try:
62
+ metadata = ExternalStorageMetadata.model_validate(raw_data)
63
+ data = self._retrieve_external_data(metadata)
64
+ record_count = len(data) if isinstance(data, list) else "N/A"
65
+ logger.info(
66
+ f"Successfully retrieved {self.model_class}.{self.field_name} "
67
+ f"from external storage (records: {record_count})"
68
+ )
69
+ return data if data is not None else self.empty_default
70
+ except Exception as e: # pylint: disable=broad-except
71
+ logger.error(
72
+ f"Failed to retrieve {self.model_class}.{self.field_name} "
73
+ f"from external storage: {str(e)}"
74
+ )
75
+ raise ExternalDataStorageError(
76
+ f"Failed to retrieve {self.field_name}: {str(e)}"
77
+ ) from e
78
+ else:
79
+ return raw_data
80
+
81
+ def __set__(self, instance: Any, value: Any) -> None: # noqa: D401
82
+ if not value:
83
+ self._cleanup_external_data(instance)
84
+ setattr(instance, self.private_field, self.empty_default)
85
+ return
86
+ try:
87
+ current_data = self.__get__(instance, type(instance))
88
+ if current_data == value:
89
+ return
90
+ except Exception: # pylint: disable=broad-except
91
+ pass
92
+
93
+ data_size = calculate_data_size(value)
94
+ if data_size > self.threshold_bytes:
95
+ logger.info(
96
+ f"{self.model_class}.{self.field_name}: Data size ({data_size:,} bytes) "
97
+ f"exceeds threshold ({self.threshold_bytes:,} bytes), storing externally"
98
+ )
99
+ self._cleanup_external_data(instance)
100
+ metadata = self._store_external_data(instance, value)
101
+ setattr(instance, self.private_field, metadata.model_dump())
102
+ else:
103
+ self._cleanup_external_data(instance)
104
+ setattr(instance, self.private_field, value)
105
+
106
+ # External storage helpers
107
+
108
+ def _store_external_data(self, instance: Any, data: Any) -> ExternalStorageMetadata:
109
+ storage_path = self._generate_storage_path(instance)
110
+ with get_autoclose_db_session() as session:
111
+ metadata = ExternalDataStorageService.store_data(
112
+ db=session,
113
+ storage_path=storage_path,
114
+ data=data,
115
+ )
116
+ logger.info(
117
+ f"Stored {self.model_class}.{self.field_name} to external storage: {storage_path}"
118
+ )
119
+ return metadata
120
+
121
+ @staticmethod
122
+ def _retrieve_external_data(metadata: ExternalStorageMetadata) -> Any: # noqa: D401
123
+ with get_autoclose_db_session() as session:
124
+ return ExternalDataStorageService.retrieve_data(
125
+ db=session,
126
+ metadata=metadata,
127
+ )
128
+
129
+ def _cleanup_external_data(self, instance: Any) -> None: # noqa: D401
130
+ raw_data = getattr(instance, self.private_field, None)
131
+ if isinstance(raw_data, dict) and "storage_type" in raw_data:
132
+ try:
133
+ metadata = ExternalStorageMetadata.model_validate(raw_data)
134
+ with get_autoclose_db_session() as session:
135
+ ExternalDataStorageService.delete_data(
136
+ db=session,
137
+ metadata=metadata,
138
+ )
139
+ logger.info(
140
+ f"Cleaned up external storage for {self.model_class}.{self.field_name}: "
141
+ f"{metadata.file_key}"
142
+ )
143
+ except Exception as e: # pylint: disable=broad-except
144
+ logger.warning(
145
+ f"Failed to cleanup external {self.field_name}: {str(e)}"
146
+ )
147
+
148
+ # Public helper
149
+
150
+ def cleanup(self, instance: Any) -> None: # noqa: D401
151
+ self._cleanup_external_data(instance)
@@ -0,0 +1,8 @@
1
+ from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
2
+ from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
3
+
4
+ __all__ = [
5
+ "ManualTask",
6
+ "ManualTaskReference",
7
+ "ManualTaskLog",
8
+ ]