ethyca-fides 2.63.0rc3__py2.py3-none-any.whl → 2.63.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/METADATA +1 -1
- {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/RECORD +129 -110
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/29e56fa1fdb3_add_monitor_tasks.py +147 -0
- fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +160 -0
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +4 -4
- fides/api/db/base.py +7 -1
- fides/api/models/connectionconfig.py +1 -1
- fides/api/models/detection_discovery/__init__.py +35 -0
- fides/api/models/detection_discovery/monitor_task.py +162 -0
- fides/api/models/field_types/__init__.py +5 -0
- fides/api/models/field_types/encrypted_large_data.py +151 -0
- fides/api/models/manual_tasks/__init__.py +8 -0
- fides/api/models/manual_tasks/manual_task.py +110 -0
- fides/api/models/manual_tasks/manual_task_log.py +100 -0
- fides/api/models/privacy_preference.py +1 -1
- fides/api/models/privacy_request/execution_log.py +3 -31
- fides/api/models/privacy_request/privacy_request.py +16 -3
- fides/api/models/privacy_request/request_task.py +36 -25
- fides/api/models/worker_task.py +96 -0
- fides/api/schemas/external_storage.py +22 -0
- fides/api/schemas/manual_tasks/__init__.py +0 -0
- fides/api/schemas/manual_tasks/manual_task_schemas.py +79 -0
- fides/api/schemas/manual_tasks/manual_task_status.py +151 -0
- fides/api/schemas/privacy_request.py +1 -12
- fides/api/service/connectors/base_erasure_email_connector.py +1 -1
- fides/api/service/connectors/consent_email_connector.py +2 -1
- fides/api/service/connectors/dynamic_erasure_email_connector.py +2 -1
- fides/api/service/connectors/erasure_email_connector.py +1 -1
- fides/api/service/external_data_storage.py +371 -0
- fides/api/service/privacy_request/request_runner_service.py +5 -5
- fides/api/service/privacy_request/request_service.py +1 -1
- fides/api/task/create_request_tasks.py +1 -1
- fides/api/task/execute_request_tasks.py +9 -8
- fides/api/task/graph_task.py +22 -10
- fides/api/util/consent_util.py +1 -1
- fides/api/util/data_size.py +102 -0
- fides/api/util/encryption/aes_gcm_encryption_util.py +271 -0
- fides/service/manual_tasks/__init__.py +0 -0
- fides/service/manual_tasks/manual_task_service.py +150 -0
- fides/service/privacy_request/privacy_request_service.py +1 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.63.0rc3.dist-info → ethyca_fides-2.63.1.dist-info}/top_level.txt +0 -0
- /fides/api/models/{detection_discovery.py → detection_discovery/core.py} +0 -0
- /fides/ui-build/static/admin/_next/static/{XobHpfndIH7IpV30u2vGV → SZn_Fpr_qG1COMjkdloep}/_buildManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/{XobHpfndIH7IpV30u2vGV → SZn_Fpr_qG1COMjkdloep}/_ssgManifest.js +0 -0
@@ -0,0 +1,160 @@
|
|
1
|
+
"""add manual task tables
|
2
|
+
|
3
|
+
Revision ID: 5efcdf18438e
|
4
|
+
Revises: c586a56c25e7
|
5
|
+
Create Date: 2025-06-04 17:24:00.300170
|
6
|
+
|
7
|
+
"""
|
8
|
+
|
9
|
+
import sqlalchemy as sa
|
10
|
+
from alembic import op
|
11
|
+
from sqlalchemy.dialects import postgresql
|
12
|
+
from sqlalchemy.sql import func
|
13
|
+
|
14
|
+
# revision identifiers, used by Alembic.
|
15
|
+
revision = "5efcdf18438e"
|
16
|
+
down_revision = "c586a56c25e7"
|
17
|
+
branch_labels = None
|
18
|
+
depends_on = None
|
19
|
+
|
20
|
+
|
21
|
+
def upgrade():
|
22
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
23
|
+
op.create_table(
|
24
|
+
"manual_task",
|
25
|
+
sa.Column("id", sa.String(), nullable=False),
|
26
|
+
sa.Column(
|
27
|
+
"created_at",
|
28
|
+
sa.DateTime(timezone=True),
|
29
|
+
nullable=False,
|
30
|
+
server_default=func.now(),
|
31
|
+
),
|
32
|
+
sa.Column(
|
33
|
+
"updated_at",
|
34
|
+
sa.DateTime(timezone=True),
|
35
|
+
nullable=False,
|
36
|
+
server_default=func.now(),
|
37
|
+
),
|
38
|
+
sa.Column(
|
39
|
+
"task_type", sa.String(), nullable=False, server_default="privacy_request"
|
40
|
+
),
|
41
|
+
sa.Column("parent_entity_id", sa.String(), nullable=False),
|
42
|
+
sa.Column("parent_entity_type", sa.String(), nullable=False),
|
43
|
+
sa.Column("due_date", sa.DateTime(timezone=True), nullable=True),
|
44
|
+
sa.PrimaryKeyConstraint("id"),
|
45
|
+
sa.UniqueConstraint(
|
46
|
+
"parent_entity_id",
|
47
|
+
"parent_entity_type",
|
48
|
+
name="uq_manual_task_parent_entity",
|
49
|
+
),
|
50
|
+
)
|
51
|
+
|
52
|
+
op.create_table(
|
53
|
+
"manual_task_reference",
|
54
|
+
sa.Column("id", sa.String(), nullable=False),
|
55
|
+
sa.Column(
|
56
|
+
"created_at",
|
57
|
+
sa.DateTime(timezone=True),
|
58
|
+
nullable=False,
|
59
|
+
server_default=func.now(),
|
60
|
+
),
|
61
|
+
sa.Column(
|
62
|
+
"updated_at",
|
63
|
+
sa.DateTime(timezone=True),
|
64
|
+
nullable=False,
|
65
|
+
server_default=func.now(),
|
66
|
+
),
|
67
|
+
sa.Column("task_id", sa.String(), nullable=False),
|
68
|
+
sa.Column("reference_id", sa.String(), nullable=False),
|
69
|
+
sa.Column("reference_type", sa.String(), nullable=False),
|
70
|
+
sa.ForeignKeyConstraint(
|
71
|
+
["task_id"],
|
72
|
+
["manual_task.id"],
|
73
|
+
ondelete="CASCADE",
|
74
|
+
),
|
75
|
+
sa.PrimaryKeyConstraint("id"),
|
76
|
+
)
|
77
|
+
|
78
|
+
op.create_table(
|
79
|
+
"manual_task_log",
|
80
|
+
sa.Column("id", sa.String(), nullable=False),
|
81
|
+
sa.Column(
|
82
|
+
"created_at",
|
83
|
+
sa.DateTime(timezone=True),
|
84
|
+
nullable=False,
|
85
|
+
server_default=func.now(),
|
86
|
+
),
|
87
|
+
sa.Column(
|
88
|
+
"updated_at",
|
89
|
+
sa.DateTime(timezone=True),
|
90
|
+
nullable=False,
|
91
|
+
server_default=func.now(),
|
92
|
+
),
|
93
|
+
sa.Column("task_id", sa.String(), nullable=False),
|
94
|
+
sa.Column("config_id", sa.String(), nullable=True),
|
95
|
+
sa.Column("instance_id", sa.String(), nullable=True),
|
96
|
+
sa.Column("status", sa.String(), nullable=False),
|
97
|
+
sa.Column("message", sa.String(), nullable=True),
|
98
|
+
sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
99
|
+
sa.ForeignKeyConstraint(
|
100
|
+
["task_id"],
|
101
|
+
["manual_task.id"],
|
102
|
+
ondelete="CASCADE",
|
103
|
+
),
|
104
|
+
sa.PrimaryKeyConstraint("id"),
|
105
|
+
)
|
106
|
+
|
107
|
+
# Create indexes for manual_task
|
108
|
+
op.create_index("ix_manual_task_task_type", "manual_task", ["task_type"])
|
109
|
+
op.create_index(
|
110
|
+
"ix_manual_task_parent_entity",
|
111
|
+
"manual_task",
|
112
|
+
["parent_entity_type", "parent_entity_id"],
|
113
|
+
)
|
114
|
+
op.create_index("ix_manual_task_due_date", "manual_task", ["due_date"])
|
115
|
+
|
116
|
+
# Create indexes for manual_task_reference
|
117
|
+
op.create_index(
|
118
|
+
"ix_manual_task_reference_task_id", "manual_task_reference", ["task_id"]
|
119
|
+
)
|
120
|
+
op.create_index(
|
121
|
+
"ix_manual_task_reference_reference",
|
122
|
+
"manual_task_reference",
|
123
|
+
["reference_id", "reference_type"],
|
124
|
+
)
|
125
|
+
|
126
|
+
# Create indexes for manual_task_log
|
127
|
+
op.create_index("ix_manual_task_log_task_id", "manual_task_log", ["task_id"])
|
128
|
+
op.create_index("ix_manual_task_log_config_id", "manual_task_log", ["config_id"])
|
129
|
+
op.create_index(
|
130
|
+
"ix_manual_task_log_instance_id", "manual_task_log", ["instance_id"]
|
131
|
+
)
|
132
|
+
op.create_index("ix_manual_task_log_status", "manual_task_log", ["status"])
|
133
|
+
op.create_index("ix_manual_task_log_created_at", "manual_task_log", ["created_at"])
|
134
|
+
|
135
|
+
# ### end Alembic commands ###
|
136
|
+
|
137
|
+
|
138
|
+
def downgrade():
|
139
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
140
|
+
# Drop indexes first
|
141
|
+
op.drop_index("ix_manual_task_log_created_at", table_name="manual_task_log")
|
142
|
+
op.drop_index("ix_manual_task_log_status", table_name="manual_task_log")
|
143
|
+
op.drop_index("ix_manual_task_log_instance_id", table_name="manual_task_log")
|
144
|
+
op.drop_index("ix_manual_task_log_config_id", table_name="manual_task_log")
|
145
|
+
op.drop_index("ix_manual_task_log_task_id", table_name="manual_task_log")
|
146
|
+
op.drop_index(
|
147
|
+
"ix_manual_task_reference_reference", table_name="manual_task_reference"
|
148
|
+
)
|
149
|
+
op.drop_index(
|
150
|
+
"ix_manual_task_reference_task_id", table_name="manual_task_reference"
|
151
|
+
)
|
152
|
+
op.drop_index("ix_manual_task_due_date", table_name="manual_task")
|
153
|
+
op.drop_index("ix_manual_task_parent_entity", table_name="manual_task")
|
154
|
+
op.drop_index("ix_manual_task_task_type", table_name="manual_task")
|
155
|
+
|
156
|
+
# Then drop tables
|
157
|
+
op.drop_table("manual_task_log")
|
158
|
+
op.drop_table("manual_task_reference")
|
159
|
+
op.drop_table("manual_task")
|
160
|
+
# ### end Alembic commands ###
|
@@ -75,6 +75,7 @@ from fides.api.models.privacy_request import (
|
|
75
75
|
ProvidedIdentity,
|
76
76
|
RequestTask,
|
77
77
|
)
|
78
|
+
from fides.api.models.worker_task import ExecutionLogStatus
|
78
79
|
from fides.api.oauth.utils import (
|
79
80
|
verify_callback_oauth_policy_pre_webhook,
|
80
81
|
verify_callback_oauth_pre_approval_webhook,
|
@@ -91,7 +92,6 @@ from fides.api.schemas.privacy_request import (
|
|
91
92
|
CheckpointActionRequired,
|
92
93
|
DenyPrivacyRequests,
|
93
94
|
ExecutionLogDetailResponse,
|
94
|
-
ExecutionLogStatus,
|
95
95
|
FilteredPrivacyRequestResults,
|
96
96
|
LogEntry,
|
97
97
|
ManualWebhookData,
|
@@ -1940,16 +1940,16 @@ def request_task_async_callback(
|
|
1940
1940
|
]:
|
1941
1941
|
raise HTTPException(
|
1942
1942
|
status_code=HTTP_400_BAD_REQUEST,
|
1943
|
-
detail=f"Callback failed. Cannot queue {request_task.action_type
|
1943
|
+
detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
|
1944
1944
|
)
|
1945
1945
|
if request_task.status != ExecutionLogStatus.awaiting_processing:
|
1946
1946
|
raise HTTPException(
|
1947
1947
|
status_code=HTTP_400_BAD_REQUEST,
|
1948
|
-
detail=f"Callback failed. Cannot queue {request_task.action_type
|
1948
|
+
detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with request task status '{request_task.status.value}'",
|
1949
1949
|
)
|
1950
1950
|
logger.info(
|
1951
1951
|
"Callback received for {} task {} {}",
|
1952
|
-
request_task.action_type
|
1952
|
+
request_task.action_type,
|
1953
1953
|
request_task.collection_address,
|
1954
1954
|
request_task.id,
|
1955
1955
|
)
|
fides/api/db/base.py
CHANGED
@@ -16,7 +16,11 @@ from fides.api.models.custom_connector_template import CustomConnectorTemplate
|
|
16
16
|
from fides.api.models.custom_report import CustomReport
|
17
17
|
from fides.api.models.datasetconfig import DatasetConfig
|
18
18
|
from fides.api.models.db_cache import DBCache
|
19
|
-
from fides.api.models.detection_discovery import MonitorConfig, StagedResource
|
19
|
+
from fides.api.models.detection_discovery.core import MonitorConfig, StagedResource
|
20
|
+
from fides.api.models.detection_discovery.monitor_task import (
|
21
|
+
MonitorTask,
|
22
|
+
MonitorTaskExecutionLog,
|
23
|
+
)
|
20
24
|
from fides.api.models.experience_notices import ExperienceNotices
|
21
25
|
from fides.api.models.fides_cloud import FidesCloud
|
22
26
|
from fides.api.models.fides_user import FidesUser
|
@@ -27,6 +31,8 @@ from fides.api.models.fides_user_respondent_email_verification import (
|
|
27
31
|
)
|
28
32
|
from fides.api.models.identity_salt import IdentitySalt
|
29
33
|
from fides.api.models.location_regulation_selections import LocationRegulationSelections
|
34
|
+
from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
|
35
|
+
from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
|
30
36
|
from fides.api.models.manual_webhook import AccessManualWebhook
|
31
37
|
from fides.api.models.messaging import MessagingConfig
|
32
38
|
from fides.api.models.messaging_template import MessagingTemplate
|
@@ -23,7 +23,7 @@ from fides.api.schemas.saas.saas_config import SaaSConfig
|
|
23
23
|
from fides.config import CONFIG
|
24
24
|
|
25
25
|
if TYPE_CHECKING:
|
26
|
-
from fides.api.models.detection_discovery import MonitorConfig
|
26
|
+
from fides.api.models.detection_discovery.core import MonitorConfig
|
27
27
|
from fides.api.schemas.connection_configuration.enums.system_type import SystemType
|
28
28
|
|
29
29
|
|
@@ -0,0 +1,35 @@
|
|
1
|
+
from .core import (
|
2
|
+
DiffStatus,
|
3
|
+
MonitorConfig,
|
4
|
+
MonitorExecution,
|
5
|
+
MonitorFrequency,
|
6
|
+
SharedMonitorConfig,
|
7
|
+
StagedResource,
|
8
|
+
StagedResourceAncestor,
|
9
|
+
fetch_staged_resources_by_type_query,
|
10
|
+
)
|
11
|
+
from .monitor_task import (
|
12
|
+
MonitorTask,
|
13
|
+
MonitorTaskExecutionLog,
|
14
|
+
MonitorTaskType,
|
15
|
+
TaskRunType,
|
16
|
+
create_monitor_task_with_execution_log,
|
17
|
+
update_monitor_task_with_execution_log,
|
18
|
+
)
|
19
|
+
|
20
|
+
__all__ = [
|
21
|
+
"DiffStatus",
|
22
|
+
"MonitorConfig",
|
23
|
+
"MonitorExecution",
|
24
|
+
"MonitorFrequency",
|
25
|
+
"SharedMonitorConfig",
|
26
|
+
"StagedResource",
|
27
|
+
"StagedResourceAncestor",
|
28
|
+
"fetch_staged_resources_by_type_query",
|
29
|
+
"MonitorTask",
|
30
|
+
"MonitorTaskExecutionLog",
|
31
|
+
"MonitorTaskType",
|
32
|
+
"TaskRunType",
|
33
|
+
"create_monitor_task_with_execution_log",
|
34
|
+
"update_monitor_task_with_execution_log",
|
35
|
+
]
|
@@ -0,0 +1,162 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from enum import Enum
|
4
|
+
from typing import List, Optional
|
5
|
+
|
6
|
+
from sqlalchemy import ARRAY, Column
|
7
|
+
from sqlalchemy import Enum as SQLAlchemyEnum
|
8
|
+
from sqlalchemy import ForeignKey, String
|
9
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
10
|
+
from sqlalchemy.orm import Session, relationship
|
11
|
+
|
12
|
+
from fides.api.db.base_class import Base, FidesBase # type: ignore[attr-defined]
|
13
|
+
from fides.api.models.detection_discovery.core import MonitorConfig
|
14
|
+
from fides.api.models.worker_task import (
|
15
|
+
ExecutionLogStatus,
|
16
|
+
TaskExecutionLog,
|
17
|
+
WorkerTask,
|
18
|
+
)
|
19
|
+
|
20
|
+
|
21
|
+
class MonitorTaskType(Enum):
|
22
|
+
"""
|
23
|
+
Types of tasks that can be executed by a worker.
|
24
|
+
"""
|
25
|
+
|
26
|
+
DETECTION = "detection"
|
27
|
+
CLASSIFICATION = "classification"
|
28
|
+
PROMOTION = "promotion"
|
29
|
+
REMOVAL_PROMOTION = "removal_promotion"
|
30
|
+
|
31
|
+
|
32
|
+
class MonitorTask(WorkerTask, Base):
|
33
|
+
"""
|
34
|
+
A monitor task executed by a worker.
|
35
|
+
"""
|
36
|
+
|
37
|
+
# celery_id is used to track task executions. While MonitorTask.id remains constant,
|
38
|
+
# celery_id changes with each execution or retry of the task, allowing us to track
|
39
|
+
# the current execution state while maintaining a stable reference to the original task.
|
40
|
+
celery_id = Column(
|
41
|
+
String(255), unique=True, nullable=False, default=FidesBase.generate_uuid
|
42
|
+
)
|
43
|
+
task_arguments = Column(JSONB, nullable=True) # To be able to rerun the task
|
44
|
+
# Contains info, warning, or error messages
|
45
|
+
message = Column(String)
|
46
|
+
monitor_config_id = Column(
|
47
|
+
String,
|
48
|
+
ForeignKey(MonitorConfig.id_field_path, ondelete="CASCADE"),
|
49
|
+
index=True,
|
50
|
+
nullable=False,
|
51
|
+
)
|
52
|
+
staged_resource_urns = Column(ARRAY(String), nullable=True)
|
53
|
+
child_resource_urns = Column(ARRAY(String), nullable=True)
|
54
|
+
|
55
|
+
monitor_config = relationship(MonitorConfig, cascade="all, delete")
|
56
|
+
execution_logs = relationship(
|
57
|
+
"MonitorTaskExecutionLog", back_populates="monitor_task", cascade="all, delete"
|
58
|
+
)
|
59
|
+
|
60
|
+
@classmethod
|
61
|
+
def allowed_action_types(cls) -> List[str]:
|
62
|
+
return [e.value for e in MonitorTaskType]
|
63
|
+
|
64
|
+
|
65
|
+
class TaskRunType(Enum):
|
66
|
+
"""
|
67
|
+
Type of task run.
|
68
|
+
"""
|
69
|
+
|
70
|
+
MANUAL = "manual"
|
71
|
+
SYSTEM = "system"
|
72
|
+
|
73
|
+
|
74
|
+
class MonitorTaskExecutionLog(TaskExecutionLog, Base):
|
75
|
+
"""
|
76
|
+
Stores the individual execution logs associated with a MonitorTask.
|
77
|
+
"""
|
78
|
+
|
79
|
+
# This celery_id preserves the specific execution ID for historical tracking,
|
80
|
+
# unlike MonitorTask.celery_id which is updated with each execution.
|
81
|
+
# This allows us to maintain a complete history of all task execution attempts.
|
82
|
+
celery_id = Column(String(255), nullable=False)
|
83
|
+
monitor_task_id = Column(
|
84
|
+
String,
|
85
|
+
ForeignKey(MonitorTask.id_field_path, ondelete="CASCADE"),
|
86
|
+
index=True,
|
87
|
+
nullable=False,
|
88
|
+
)
|
89
|
+
run_type = Column(
|
90
|
+
SQLAlchemyEnum(TaskRunType), nullable=False, default=TaskRunType.SYSTEM
|
91
|
+
)
|
92
|
+
|
93
|
+
monitor_task = relationship("MonitorTask", back_populates="execution_logs")
|
94
|
+
|
95
|
+
|
96
|
+
def create_monitor_task_with_execution_log(
|
97
|
+
db: Session, monitor_task_data: dict
|
98
|
+
) -> MonitorTask:
|
99
|
+
"""
|
100
|
+
Creates a monitor task with an execution log.
|
101
|
+
The default status is pending for the task and pending for the execution log.
|
102
|
+
"""
|
103
|
+
status = ExecutionLogStatus.pending
|
104
|
+
task_record = MonitorTask( # type: ignore
|
105
|
+
status=status.value,
|
106
|
+
**monitor_task_data,
|
107
|
+
)
|
108
|
+
db.add(task_record)
|
109
|
+
db.flush()
|
110
|
+
|
111
|
+
execution_log = MonitorTaskExecutionLog( # type: ignore
|
112
|
+
monitor_task=task_record, celery_id=task_record.celery_id, status=status
|
113
|
+
)
|
114
|
+
db.add(execution_log)
|
115
|
+
|
116
|
+
db.commit()
|
117
|
+
db.refresh(task_record)
|
118
|
+
return task_record
|
119
|
+
|
120
|
+
|
121
|
+
def update_monitor_task_with_execution_log(
|
122
|
+
db: Session,
|
123
|
+
status: ExecutionLogStatus,
|
124
|
+
task_record: Optional[MonitorTask] = None,
|
125
|
+
celery_id: Optional[str] = None,
|
126
|
+
message: Optional[str] = None,
|
127
|
+
run_type: TaskRunType = TaskRunType.SYSTEM,
|
128
|
+
) -> MonitorTask:
|
129
|
+
"""
|
130
|
+
Updates a monitor task with an execution log.
|
131
|
+
|
132
|
+
It must be either celery_id or task_record. If it doesn't receive a celery_id, it's assumed a new one needs to be created because a new run is about to be performed.
|
133
|
+
If it receives a celery_id, it means it only needs to update the status of an existing run. It can receive task_record to avoid querying the database again to get it.
|
134
|
+
"""
|
135
|
+
if not celery_id and not task_record:
|
136
|
+
raise ValueError("Either celery_id or task_record must be provided")
|
137
|
+
|
138
|
+
if celery_id and not task_record:
|
139
|
+
task_record = MonitorTask.get_by(db=db, field="celery_id", value=celery_id)
|
140
|
+
if not task_record:
|
141
|
+
raise ValueError(f"Could not find MonitorTask with celery_id {celery_id}")
|
142
|
+
|
143
|
+
assert task_record is not None # help type checker understand the control flow
|
144
|
+
|
145
|
+
if not celery_id:
|
146
|
+
celery_id = task_record.generate_uuid()
|
147
|
+
task_record.celery_id = celery_id
|
148
|
+
|
149
|
+
task_record.status = status.value # type: ignore
|
150
|
+
task_record.message = message
|
151
|
+
|
152
|
+
MonitorTaskExecutionLog( # type: ignore
|
153
|
+
monitor_task=task_record,
|
154
|
+
status=status,
|
155
|
+
message=message,
|
156
|
+
celery_id=celery_id,
|
157
|
+
run_type=run_type,
|
158
|
+
)
|
159
|
+
|
160
|
+
db.commit()
|
161
|
+
db.refresh(task_record)
|
162
|
+
return task_record
|
@@ -0,0 +1,151 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from typing import Any, Optional, Type
|
3
|
+
|
4
|
+
from loguru import logger
|
5
|
+
|
6
|
+
from fides.api.api.deps import get_autoclose_db_session
|
7
|
+
from fides.api.schemas.external_storage import ExternalStorageMetadata
|
8
|
+
from fides.api.service.external_data_storage import (
|
9
|
+
ExternalDataStorageError,
|
10
|
+
ExternalDataStorageService,
|
11
|
+
)
|
12
|
+
from fides.api.util.data_size import LARGE_DATA_THRESHOLD_BYTES, calculate_data_size
|
13
|
+
|
14
|
+
|
15
|
+
class EncryptedLargeDataDescriptor:
|
16
|
+
"""
|
17
|
+
A Python descriptor for database fields with encrypted external storage fallback.
|
18
|
+
|
19
|
+
See the original implementation for detailed docstrings.
|
20
|
+
"""
|
21
|
+
|
22
|
+
def __init__(
|
23
|
+
self,
|
24
|
+
field_name: str,
|
25
|
+
empty_default: Optional[Any] = None,
|
26
|
+
threshold_bytes: Optional[int] = None,
|
27
|
+
):
|
28
|
+
self.field_name = field_name
|
29
|
+
self.private_field = f"_{field_name}"
|
30
|
+
self.empty_default = empty_default if empty_default is not None else []
|
31
|
+
self.threshold_bytes = threshold_bytes or LARGE_DATA_THRESHOLD_BYTES
|
32
|
+
self.model_class: Optional[str] = None
|
33
|
+
self.name: Optional[str] = None
|
34
|
+
|
35
|
+
# Descriptor protocol helpers
|
36
|
+
|
37
|
+
def __set_name__(
|
38
|
+
self, owner: Type, name: str
|
39
|
+
) -> None: # noqa: D401 (docstring in orig file)
|
40
|
+
self.name = name
|
41
|
+
self.model_class = owner.__name__
|
42
|
+
|
43
|
+
def _generate_storage_path(self, instance: Any) -> str:
|
44
|
+
instance_id = getattr(instance, "id", None)
|
45
|
+
if not instance_id:
|
46
|
+
raise ValueError(f"Instance {instance} must have an 'id' attribute")
|
47
|
+
timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S-%f")
|
48
|
+
return f"{self.model_class}/{instance_id}/{self.field_name}/{timestamp}.txt"
|
49
|
+
|
50
|
+
def __get__(self, instance: Any, owner: Type) -> Any: # noqa: D401
|
51
|
+
if instance is None:
|
52
|
+
return self
|
53
|
+
raw_data = getattr(instance, self.private_field)
|
54
|
+
if raw_data is None:
|
55
|
+
return None
|
56
|
+
if isinstance(raw_data, dict) and "storage_type" in raw_data:
|
57
|
+
logger.info(
|
58
|
+
f"Reading {self.model_class}.{self.field_name} from external storage "
|
59
|
+
f"({raw_data.get('storage_type')})"
|
60
|
+
)
|
61
|
+
try:
|
62
|
+
metadata = ExternalStorageMetadata.model_validate(raw_data)
|
63
|
+
data = self._retrieve_external_data(metadata)
|
64
|
+
record_count = len(data) if isinstance(data, list) else "N/A"
|
65
|
+
logger.info(
|
66
|
+
f"Successfully retrieved {self.model_class}.{self.field_name} "
|
67
|
+
f"from external storage (records: {record_count})"
|
68
|
+
)
|
69
|
+
return data if data is not None else self.empty_default
|
70
|
+
except Exception as e: # pylint: disable=broad-except
|
71
|
+
logger.error(
|
72
|
+
f"Failed to retrieve {self.model_class}.{self.field_name} "
|
73
|
+
f"from external storage: {str(e)}"
|
74
|
+
)
|
75
|
+
raise ExternalDataStorageError(
|
76
|
+
f"Failed to retrieve {self.field_name}: {str(e)}"
|
77
|
+
) from e
|
78
|
+
else:
|
79
|
+
return raw_data
|
80
|
+
|
81
|
+
def __set__(self, instance: Any, value: Any) -> None: # noqa: D401
|
82
|
+
if not value:
|
83
|
+
self._cleanup_external_data(instance)
|
84
|
+
setattr(instance, self.private_field, self.empty_default)
|
85
|
+
return
|
86
|
+
try:
|
87
|
+
current_data = self.__get__(instance, type(instance))
|
88
|
+
if current_data == value:
|
89
|
+
return
|
90
|
+
except Exception: # pylint: disable=broad-except
|
91
|
+
pass
|
92
|
+
|
93
|
+
data_size = calculate_data_size(value)
|
94
|
+
if data_size > self.threshold_bytes:
|
95
|
+
logger.info(
|
96
|
+
f"{self.model_class}.{self.field_name}: Data size ({data_size:,} bytes) "
|
97
|
+
f"exceeds threshold ({self.threshold_bytes:,} bytes), storing externally"
|
98
|
+
)
|
99
|
+
self._cleanup_external_data(instance)
|
100
|
+
metadata = self._store_external_data(instance, value)
|
101
|
+
setattr(instance, self.private_field, metadata.model_dump())
|
102
|
+
else:
|
103
|
+
self._cleanup_external_data(instance)
|
104
|
+
setattr(instance, self.private_field, value)
|
105
|
+
|
106
|
+
# External storage helpers
|
107
|
+
|
108
|
+
def _store_external_data(self, instance: Any, data: Any) -> ExternalStorageMetadata:
|
109
|
+
storage_path = self._generate_storage_path(instance)
|
110
|
+
with get_autoclose_db_session() as session:
|
111
|
+
metadata = ExternalDataStorageService.store_data(
|
112
|
+
db=session,
|
113
|
+
storage_path=storage_path,
|
114
|
+
data=data,
|
115
|
+
)
|
116
|
+
logger.info(
|
117
|
+
f"Stored {self.model_class}.{self.field_name} to external storage: {storage_path}"
|
118
|
+
)
|
119
|
+
return metadata
|
120
|
+
|
121
|
+
@staticmethod
|
122
|
+
def _retrieve_external_data(metadata: ExternalStorageMetadata) -> Any: # noqa: D401
|
123
|
+
with get_autoclose_db_session() as session:
|
124
|
+
return ExternalDataStorageService.retrieve_data(
|
125
|
+
db=session,
|
126
|
+
metadata=metadata,
|
127
|
+
)
|
128
|
+
|
129
|
+
def _cleanup_external_data(self, instance: Any) -> None: # noqa: D401
|
130
|
+
raw_data = getattr(instance, self.private_field, None)
|
131
|
+
if isinstance(raw_data, dict) and "storage_type" in raw_data:
|
132
|
+
try:
|
133
|
+
metadata = ExternalStorageMetadata.model_validate(raw_data)
|
134
|
+
with get_autoclose_db_session() as session:
|
135
|
+
ExternalDataStorageService.delete_data(
|
136
|
+
db=session,
|
137
|
+
metadata=metadata,
|
138
|
+
)
|
139
|
+
logger.info(
|
140
|
+
f"Cleaned up external storage for {self.model_class}.{self.field_name}: "
|
141
|
+
f"{metadata.file_key}"
|
142
|
+
)
|
143
|
+
except Exception as e: # pylint: disable=broad-except
|
144
|
+
logger.warning(
|
145
|
+
f"Failed to cleanup external {self.field_name}: {str(e)}"
|
146
|
+
)
|
147
|
+
|
148
|
+
# Public helper
|
149
|
+
|
150
|
+
def cleanup(self, instance: Any) -> None: # noqa: D401
|
151
|
+
self._cleanup_external_data(instance)
|