ethyca-fides 2.63.1b3__py2.py3-none-any.whl → 2.63.1rc0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/RECORD +139 -141
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/29e56fa1fdb3_add_monitor_tasks.py +147 -0
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +4 -4
- fides/api/db/base.py +5 -3
- fides/api/main.py +0 -1
- fides/api/models/attachment.py +23 -36
- fides/api/models/connectionconfig.py +1 -1
- fides/api/models/detection_discovery/__init__.py +35 -0
- fides/api/models/detection_discovery/monitor_task.py +161 -0
- fides/api/models/field_types/__init__.py +5 -0
- fides/api/models/field_types/encrypted_large_data.py +151 -0
- fides/api/models/privacy_preference.py +1 -1
- fides/api/models/privacy_request/execution_log.py +3 -31
- fides/api/models/privacy_request/privacy_request.py +16 -3
- fides/api/models/privacy_request/request_task.py +36 -25
- fides/api/models/worker_task.py +96 -0
- fides/api/schemas/external_storage.py +22 -0
- fides/api/schemas/privacy_request.py +1 -12
- fides/api/service/connectors/base_erasure_email_connector.py +1 -1
- fides/api/service/connectors/consent_email_connector.py +2 -1
- fides/api/service/connectors/dynamic_erasure_email_connector.py +2 -1
- fides/api/service/connectors/erasure_email_connector.py +1 -1
- fides/api/service/external_data_storage.py +371 -0
- fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +46 -264
- fides/api/service/privacy_request/dsr_package/templates/collection_index.html +9 -34
- fides/api/service/privacy_request/dsr_package/templates/item.html +37 -0
- fides/api/service/privacy_request/dsr_package/templates/main.css +2 -45
- fides/api/service/privacy_request/dsr_package/templates/welcome.html +8 -12
- fides/api/service/privacy_request/request_runner_service.py +139 -258
- fides/api/service/privacy_request/request_service.py +1 -1
- fides/api/service/storage/gcs.py +3 -15
- fides/api/service/storage/s3.py +14 -28
- fides/api/service/storage/util.py +7 -45
- fides/api/task/create_request_tasks.py +1 -1
- fides/api/task/execute_request_tasks.py +9 -8
- fides/api/task/graph_task.py +22 -10
- fides/api/tasks/storage.py +91 -85
- fides/api/util/cache.py +1 -77
- fides/api/util/consent_util.py +1 -1
- fides/api/util/data_size.py +102 -0
- fides/api/util/encryption/aes_gcm_encryption_util.py +271 -0
- fides/config/redis_settings.py +8 -99
- fides/service/messaging/aws_ses_service.py +1 -5
- fides/service/privacy_request/privacy_request_service.py +1 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +0 -160
- fides/api/models/manual_tasks/__init__.py +0 -8
- fides/api/models/manual_tasks/manual_task.py +0 -110
- fides/api/models/manual_tasks/manual_task_log.py +0 -100
- fides/api/schemas/manual_tasks/__init__.py +0 -0
- fides/api/schemas/manual_tasks/manual_task_schemas.py +0 -79
- fides/api/schemas/manual_tasks/manual_task_status.py +0 -151
- fides/api/service/privacy_request/attachment_handling.py +0 -132
- fides/api/service/privacy_request/dsr_package/templates/attachments_index.html +0 -33
- fides/api/tasks/csv_utils.py +0 -170
- fides/api/tasks/encryption_utils.py +0 -42
- fides/service/manual_tasks/__init__.py +0 -0
- fides/service/manual_tasks/manual_task_service.py +0 -150
- {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.63.1b3.dist-info → ethyca_fides-2.63.1rc0.dist-info}/top_level.txt +0 -0
- /fides/api/models/{detection_discovery.py → detection_discovery/core.py} +0 -0
- /fides/ui-build/static/admin/_next/static/{ycPcko8qnif6BlkQ6MN4D → PEElhfUdgE5bJjiyu5QCD}/_buildManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/{ycPcko8qnif6BlkQ6MN4D → PEElhfUdgE5bJjiyu5QCD}/_ssgManifest.js +0 -0
@@ -0,0 +1,147 @@
|
|
1
|
+
"""add_monitor_tasks
|
2
|
+
|
3
|
+
Revision ID: 29e56fa1fdb3
|
4
|
+
Revises: 5efcdf18438e
|
5
|
+
Create Date: 2025-06-11 14:40:08.384571
|
6
|
+
|
7
|
+
"""
|
8
|
+
|
9
|
+
import sqlalchemy as sa
|
10
|
+
from alembic import op
|
11
|
+
from sqlalchemy.dialects import postgresql
|
12
|
+
|
13
|
+
# revision identifiers, used by Alembic.
|
14
|
+
revision = "29e56fa1fdb3"
|
15
|
+
down_revision = "5efcdf18438e"
|
16
|
+
branch_labels = None
|
17
|
+
depends_on = None
|
18
|
+
|
19
|
+
|
20
|
+
def upgrade():
|
21
|
+
op.create_table(
|
22
|
+
"monitortask",
|
23
|
+
sa.Column("id", sa.String(length=255), nullable=False),
|
24
|
+
sa.Column(
|
25
|
+
"created_at",
|
26
|
+
sa.DateTime(timezone=True),
|
27
|
+
server_default=sa.text("now()"),
|
28
|
+
nullable=True,
|
29
|
+
),
|
30
|
+
sa.Column(
|
31
|
+
"updated_at",
|
32
|
+
sa.DateTime(timezone=True),
|
33
|
+
server_default=sa.text("now()"),
|
34
|
+
nullable=True,
|
35
|
+
),
|
36
|
+
sa.Column("action_type", sa.String(), nullable=False),
|
37
|
+
sa.Column(
|
38
|
+
"status",
|
39
|
+
sa.Enum(
|
40
|
+
"in_processing",
|
41
|
+
"pending",
|
42
|
+
"complete",
|
43
|
+
"error",
|
44
|
+
"paused",
|
45
|
+
"retrying",
|
46
|
+
"skipped",
|
47
|
+
name="executionlogstatus",
|
48
|
+
native_enum=False,
|
49
|
+
),
|
50
|
+
nullable=False,
|
51
|
+
),
|
52
|
+
sa.Column("celery_id", sa.String(length=255), nullable=False),
|
53
|
+
sa.Column(
|
54
|
+
"task_arguments", postgresql.JSONB(astext_type=sa.Text()), nullable=True
|
55
|
+
),
|
56
|
+
sa.Column("message", sa.String(), nullable=True),
|
57
|
+
sa.Column("monitor_config_id", sa.String(), nullable=False),
|
58
|
+
sa.Column("staged_resource_urns", sa.ARRAY(sa.String()), nullable=True),
|
59
|
+
sa.Column("child_resource_urns", sa.ARRAY(sa.String()), nullable=True),
|
60
|
+
sa.ForeignKeyConstraint(
|
61
|
+
["monitor_config_id"], ["monitorconfig.id"], ondelete="CASCADE"
|
62
|
+
),
|
63
|
+
sa.PrimaryKeyConstraint("id"),
|
64
|
+
sa.UniqueConstraint("celery_id"),
|
65
|
+
)
|
66
|
+
op.create_index(
|
67
|
+
op.f("ix_monitortask_action_type"), "monitortask", ["action_type"], unique=False
|
68
|
+
)
|
69
|
+
op.create_index(op.f("ix_monitortask_id"), "monitortask", ["id"], unique=False)
|
70
|
+
op.create_index(
|
71
|
+
op.f("ix_monitortask_monitor_config_id"),
|
72
|
+
"monitortask",
|
73
|
+
["monitor_config_id"],
|
74
|
+
unique=False,
|
75
|
+
)
|
76
|
+
op.create_index(
|
77
|
+
op.f("ix_monitortask_status"), "monitortask", ["status"], unique=False
|
78
|
+
)
|
79
|
+
op.create_table(
|
80
|
+
"monitortaskexecutionlog",
|
81
|
+
sa.Column("id", sa.String(length=255), nullable=False),
|
82
|
+
sa.Column(
|
83
|
+
"status",
|
84
|
+
postgresql.ENUM(name="executionlogstatus", create_type=False),
|
85
|
+
nullable=False,
|
86
|
+
),
|
87
|
+
sa.Column("message", sa.String(), nullable=True),
|
88
|
+
sa.Column(
|
89
|
+
"created_at",
|
90
|
+
sa.DateTime(timezone=True),
|
91
|
+
server_default=sa.text("clock_timestamp()"),
|
92
|
+
nullable=True,
|
93
|
+
),
|
94
|
+
sa.Column(
|
95
|
+
"updated_at",
|
96
|
+
sa.DateTime(timezone=True),
|
97
|
+
server_default=sa.text("clock_timestamp()"),
|
98
|
+
nullable=True,
|
99
|
+
),
|
100
|
+
sa.Column("celery_id", sa.String(length=255), nullable=False),
|
101
|
+
sa.Column("monitor_task_id", sa.String(), nullable=False),
|
102
|
+
sa.Column(
|
103
|
+
"run_type", sa.Enum("MANUAL", "SYSTEM", name="taskruntype"), nullable=False
|
104
|
+
),
|
105
|
+
sa.ForeignKeyConstraint(
|
106
|
+
["monitor_task_id"], ["monitortask.id"], ondelete="CASCADE"
|
107
|
+
),
|
108
|
+
sa.PrimaryKeyConstraint("id"),
|
109
|
+
)
|
110
|
+
op.create_index(
|
111
|
+
op.f("ix_monitortaskexecutionlog_id"),
|
112
|
+
"monitortaskexecutionlog",
|
113
|
+
["id"],
|
114
|
+
unique=False,
|
115
|
+
)
|
116
|
+
op.create_index(
|
117
|
+
op.f("ix_monitortaskexecutionlog_monitor_task_id"),
|
118
|
+
"monitortaskexecutionlog",
|
119
|
+
["monitor_task_id"],
|
120
|
+
unique=False,
|
121
|
+
)
|
122
|
+
op.create_index(
|
123
|
+
op.f("ix_monitortaskexecutionlog_status"),
|
124
|
+
"monitortaskexecutionlog",
|
125
|
+
["status"],
|
126
|
+
unique=False,
|
127
|
+
)
|
128
|
+
|
129
|
+
|
130
|
+
def downgrade():
|
131
|
+
op.drop_index(
|
132
|
+
op.f("ix_monitortaskexecutionlog_status"), table_name="monitortaskexecutionlog"
|
133
|
+
)
|
134
|
+
op.drop_index(
|
135
|
+
op.f("ix_monitortaskexecutionlog_monitor_task_id"),
|
136
|
+
table_name="monitortaskexecutionlog",
|
137
|
+
)
|
138
|
+
op.drop_index(
|
139
|
+
op.f("ix_monitortaskexecutionlog_id"), table_name="monitortaskexecutionlog"
|
140
|
+
)
|
141
|
+
op.drop_table("monitortaskexecutionlog")
|
142
|
+
op.drop_index(op.f("ix_monitortask_status"), table_name="monitortask")
|
143
|
+
op.drop_index(op.f("ix_monitortask_monitor_config_id"), table_name="monitortask")
|
144
|
+
op.drop_index(op.f("ix_monitortask_id"), table_name="monitortask")
|
145
|
+
op.drop_index(op.f("ix_monitortask_action_type"), table_name="monitortask")
|
146
|
+
op.drop_table("monitortask")
|
147
|
+
op.execute("DROP TYPE IF EXISTS taskruntype")
|
@@ -75,6 +75,7 @@ from fides.api.models.privacy_request import (
|
|
75
75
|
ProvidedIdentity,
|
76
76
|
RequestTask,
|
77
77
|
)
|
78
|
+
from fides.api.models.worker_task import ExecutionLogStatus
|
78
79
|
from fides.api.oauth.utils import (
|
79
80
|
verify_callback_oauth_policy_pre_webhook,
|
80
81
|
verify_callback_oauth_pre_approval_webhook,
|
@@ -91,7 +92,6 @@ from fides.api.schemas.privacy_request import (
|
|
91
92
|
CheckpointActionRequired,
|
92
93
|
DenyPrivacyRequests,
|
93
94
|
ExecutionLogDetailResponse,
|
94
|
-
ExecutionLogStatus,
|
95
95
|
FilteredPrivacyRequestResults,
|
96
96
|
LogEntry,
|
97
97
|
ManualWebhookData,
|
@@ -1940,16 +1940,16 @@ def request_task_async_callback(
|
|
1940
1940
|
]:
|
1941
1941
|
raise HTTPException(
|
1942
1942
|
status_code=HTTP_400_BAD_REQUEST,
|
1943
|
-
detail=f"Callback failed. Cannot queue {request_task.action_type
|
1943
|
+
detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with privacy request status '{privacy_request.status.value}'",
|
1944
1944
|
)
|
1945
1945
|
if request_task.status != ExecutionLogStatus.awaiting_processing:
|
1946
1946
|
raise HTTPException(
|
1947
1947
|
status_code=HTTP_400_BAD_REQUEST,
|
1948
|
-
detail=f"Callback failed. Cannot queue {request_task.action_type
|
1948
|
+
detail=f"Callback failed. Cannot queue {request_task.action_type} task '{request_task.id}' with request task status '{request_task.status.value}'",
|
1949
1949
|
)
|
1950
1950
|
logger.info(
|
1951
1951
|
"Callback received for {} task {} {}",
|
1952
|
-
request_task.action_type
|
1952
|
+
request_task.action_type,
|
1953
1953
|
request_task.collection_address,
|
1954
1954
|
request_task.id,
|
1955
1955
|
)
|
fides/api/db/base.py
CHANGED
@@ -16,7 +16,11 @@ from fides.api.models.custom_connector_template import CustomConnectorTemplate
|
|
16
16
|
from fides.api.models.custom_report import CustomReport
|
17
17
|
from fides.api.models.datasetconfig import DatasetConfig
|
18
18
|
from fides.api.models.db_cache import DBCache
|
19
|
-
from fides.api.models.detection_discovery import MonitorConfig, StagedResource
|
19
|
+
from fides.api.models.detection_discovery.core import MonitorConfig, StagedResource
|
20
|
+
from fides.api.models.detection_discovery.monitor_task import (
|
21
|
+
MonitorTask,
|
22
|
+
MonitorTaskExecutionLog,
|
23
|
+
)
|
20
24
|
from fides.api.models.experience_notices import ExperienceNotices
|
21
25
|
from fides.api.models.fides_cloud import FidesCloud
|
22
26
|
from fides.api.models.fides_user import FidesUser
|
@@ -27,8 +31,6 @@ from fides.api.models.fides_user_respondent_email_verification import (
|
|
27
31
|
)
|
28
32
|
from fides.api.models.identity_salt import IdentitySalt
|
29
33
|
from fides.api.models.location_regulation_selections import LocationRegulationSelections
|
30
|
-
from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
|
31
|
-
from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
|
32
34
|
from fides.api.models.manual_webhook import AccessManualWebhook
|
33
35
|
from fides.api.models.messaging import MessagingConfig
|
34
36
|
from fides.api.models.messaging_template import MessagingTemplate
|
fides/api/main.py
CHANGED
@@ -244,7 +244,6 @@ async def log_request(request: Request, call_next: Callable) -> Response:
|
|
244
244
|
status_code=response.status_code,
|
245
245
|
handler_time=f"{total_time}ms",
|
246
246
|
path=request.url.path,
|
247
|
-
fides_client=request.headers.get("Fides-Client", "unknown"),
|
248
247
|
).info("Request received")
|
249
248
|
return response
|
250
249
|
|
fides/api/models/attachment.py
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
import os
|
2
2
|
from enum import Enum as EnumType
|
3
|
-
from io import BytesIO
|
4
3
|
from typing import IO, TYPE_CHECKING, Any, Tuple
|
5
4
|
|
6
5
|
from fideslang.validation import AnyHttpUrlString
|
@@ -20,6 +19,7 @@ from fides.api.service.storage.s3 import (
|
|
20
19
|
generic_upload_to_s3,
|
21
20
|
)
|
22
21
|
from fides.api.service.storage.util import AllowedFileType, get_local_filename
|
22
|
+
from fides.config import CONFIG
|
23
23
|
|
24
24
|
if TYPE_CHECKING:
|
25
25
|
from fides.api.models.comment import Comment
|
@@ -28,11 +28,6 @@ if TYPE_CHECKING:
|
|
28
28
|
from fides.api.models.storage import StorageConfig
|
29
29
|
|
30
30
|
|
31
|
-
# This is 7 days in seconds and is currently the max allowed
|
32
|
-
# configurable expiration time for presigned URLs for both s3 and gcs.
|
33
|
-
MAX_TTL_SECONDS = 604800
|
34
|
-
|
35
|
-
|
36
31
|
class AttachmentType(str, EnumType):
|
37
32
|
"""
|
38
33
|
Enum for attachment types. Indicates attachment usage.
|
@@ -127,11 +122,6 @@ class Attachment(Base):
|
|
127
122
|
"""Returns the content type of the attachment."""
|
128
123
|
return AllowedFileType[self.file_name.split(".")[-1]].value
|
129
124
|
|
130
|
-
@property
|
131
|
-
def file_key(self) -> str:
|
132
|
-
"""Returns the file key of the attachment."""
|
133
|
-
return f"{self.id}/{self.file_name}"
|
134
|
-
|
135
125
|
def upload(self, attachment: IO[bytes]) -> None:
|
136
126
|
"""Uploads an attachment to S3, GCS, or local storage."""
|
137
127
|
if self.config.type == StorageType.s3:
|
@@ -140,7 +130,7 @@ class Attachment(Base):
|
|
140
130
|
generic_upload_to_s3(
|
141
131
|
storage_secrets=self.config.secrets,
|
142
132
|
bucket_name=bucket_name,
|
143
|
-
file_key=self.
|
133
|
+
file_key=f"{self.id}/{self.file_name}",
|
144
134
|
document=attachment,
|
145
135
|
auth_method=auth_method,
|
146
136
|
)
|
@@ -152,7 +142,7 @@ class Attachment(Base):
|
|
152
142
|
auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
|
153
143
|
storage_client = get_gcs_client(auth_method, self.config.secrets)
|
154
144
|
bucket = storage_client.bucket(bucket_name)
|
155
|
-
blob = bucket.blob(self.
|
145
|
+
blob = bucket.blob(f"{self.id}/{self.file_name}")
|
156
146
|
|
157
147
|
# Reset the file pointer to the beginning
|
158
148
|
try:
|
@@ -165,7 +155,7 @@ class Attachment(Base):
|
|
165
155
|
return
|
166
156
|
|
167
157
|
if self.config.type == StorageType.local:
|
168
|
-
filename = get_local_filename(self.
|
158
|
+
filename = get_local_filename(f"{self.id}/{self.file_name}")
|
169
159
|
|
170
160
|
# Validate that attachment is a file-like object
|
171
161
|
if not hasattr(attachment, "read"):
|
@@ -215,10 +205,9 @@ class Attachment(Base):
|
|
215
205
|
size, url = generic_retrieve_from_s3(
|
216
206
|
storage_secrets=self.config.secrets,
|
217
207
|
bucket_name=bucket_name,
|
218
|
-
file_key=self.
|
208
|
+
file_key=f"{self.id}/{self.file_name}",
|
219
209
|
auth_method=auth_method,
|
220
210
|
get_content=False,
|
221
|
-
ttl_seconds=MAX_TTL_SECONDS,
|
222
211
|
)
|
223
212
|
return size, url
|
224
213
|
|
@@ -227,20 +216,19 @@ class Attachment(Base):
|
|
227
216
|
auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
|
228
217
|
storage_client = get_gcs_client(auth_method, self.config.secrets)
|
229
218
|
bucket = storage_client.bucket(bucket_name)
|
230
|
-
blob = bucket.blob(self.
|
219
|
+
blob = bucket.blob(f"{self.id}/{self.file_name}")
|
231
220
|
|
232
221
|
# Ensure we have the blob metadata
|
233
222
|
blob.reload()
|
234
|
-
# Expiration is set to 7 days
|
235
223
|
url = blob.generate_signed_url(
|
236
224
|
version="v4",
|
237
|
-
expiration=
|
225
|
+
expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
|
238
226
|
method="GET",
|
239
227
|
)
|
240
228
|
return blob.size, url
|
241
229
|
|
242
230
|
if self.config.type == StorageType.local:
|
243
|
-
filename = get_local_filename(self.
|
231
|
+
filename = get_local_filename(f"{self.id}/{self.file_name}")
|
244
232
|
size = os.path.getsize(filename)
|
245
233
|
return size, filename
|
246
234
|
|
@@ -248,7 +236,7 @@ class Attachment(Base):
|
|
248
236
|
|
249
237
|
def retrieve_attachment_content(
|
250
238
|
self,
|
251
|
-
) -> Tuple[int,
|
239
|
+
) -> Tuple[int, bytes]:
|
252
240
|
"""
|
253
241
|
Retrieves the size of the attachment and its actual content.
|
254
242
|
- For s3:
|
@@ -264,33 +252,30 @@ class Attachment(Base):
|
|
264
252
|
if self.config.type == StorageType.s3:
|
265
253
|
bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
|
266
254
|
auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
|
267
|
-
size,
|
255
|
+
size, content = generic_retrieve_from_s3(
|
268
256
|
storage_secrets=self.config.secrets,
|
269
257
|
bucket_name=bucket_name,
|
270
|
-
file_key=self.
|
258
|
+
file_key=f"{self.id}/{self.file_name}",
|
271
259
|
auth_method=auth_method,
|
272
260
|
get_content=True,
|
273
261
|
)
|
274
|
-
return size,
|
262
|
+
return size, content
|
275
263
|
|
276
264
|
if self.config.type == StorageType.gcs:
|
277
265
|
bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
|
278
266
|
auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
|
279
267
|
storage_client = get_gcs_client(auth_method, self.config.secrets)
|
280
268
|
bucket = storage_client.bucket(bucket_name)
|
281
|
-
blob = bucket.blob(self.
|
269
|
+
blob = bucket.blob(f"{self.id}/{self.file_name}")
|
282
270
|
|
283
|
-
|
284
|
-
|
285
|
-
fileobj.seek(0) # Reset pointer to beginning after download
|
286
|
-
return blob.size, fileobj
|
271
|
+
content = blob.download_as_bytes()
|
272
|
+
return len(content), content
|
287
273
|
|
288
274
|
if self.config.type == StorageType.local:
|
289
|
-
filename = get_local_filename(self.
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
content.seek(0) # Reset pointer to beginning
|
275
|
+
filename = get_local_filename(f"{self.id}/{self.file_name}")
|
276
|
+
with open(filename, "rb") as file:
|
277
|
+
content = file.read()
|
278
|
+
size = len(content)
|
294
279
|
return size, content
|
295
280
|
|
296
281
|
raise ValueError(f"Unsupported storage type: {self.config.type}")
|
@@ -303,7 +288,7 @@ class Attachment(Base):
|
|
303
288
|
generic_delete_from_s3(
|
304
289
|
storage_secrets=self.config.secrets,
|
305
290
|
bucket_name=bucket_name,
|
306
|
-
file_key=self.
|
291
|
+
file_key=f"{self.id}/{self.file_name}",
|
307
292
|
auth_method=auth_method,
|
308
293
|
)
|
309
294
|
return
|
@@ -322,7 +307,9 @@ class Attachment(Base):
|
|
322
307
|
return
|
323
308
|
|
324
309
|
if self.config.type == StorageType.local:
|
325
|
-
folder_path = os.path.dirname(
|
310
|
+
folder_path = os.path.dirname(
|
311
|
+
get_local_filename(f"{self.id}/{self.file_name}")
|
312
|
+
)
|
326
313
|
if os.path.exists(folder_path):
|
327
314
|
import shutil
|
328
315
|
|
@@ -23,7 +23,7 @@ from fides.api.schemas.saas.saas_config import SaaSConfig
|
|
23
23
|
from fides.config import CONFIG
|
24
24
|
|
25
25
|
if TYPE_CHECKING:
|
26
|
-
from fides.api.models.detection_discovery import MonitorConfig
|
26
|
+
from fides.api.models.detection_discovery.core import MonitorConfig
|
27
27
|
from fides.api.schemas.connection_configuration.enums.system_type import SystemType
|
28
28
|
|
29
29
|
|
@@ -0,0 +1,35 @@
|
|
1
|
+
from .core import (
|
2
|
+
DiffStatus,
|
3
|
+
MonitorConfig,
|
4
|
+
MonitorExecution,
|
5
|
+
MonitorFrequency,
|
6
|
+
SharedMonitorConfig,
|
7
|
+
StagedResource,
|
8
|
+
StagedResourceAncestor,
|
9
|
+
fetch_staged_resources_by_type_query,
|
10
|
+
)
|
11
|
+
from .monitor_task import (
|
12
|
+
MonitorTask,
|
13
|
+
MonitorTaskExecutionLog,
|
14
|
+
MonitorTaskType,
|
15
|
+
TaskRunType,
|
16
|
+
create_monitor_task_with_execution_log,
|
17
|
+
update_monitor_task_with_execution_log,
|
18
|
+
)
|
19
|
+
|
20
|
+
__all__ = [
|
21
|
+
"DiffStatus",
|
22
|
+
"MonitorConfig",
|
23
|
+
"MonitorExecution",
|
24
|
+
"MonitorFrequency",
|
25
|
+
"SharedMonitorConfig",
|
26
|
+
"StagedResource",
|
27
|
+
"StagedResourceAncestor",
|
28
|
+
"fetch_staged_resources_by_type_query",
|
29
|
+
"MonitorTask",
|
30
|
+
"MonitorTaskExecutionLog",
|
31
|
+
"MonitorTaskType",
|
32
|
+
"TaskRunType",
|
33
|
+
"create_monitor_task_with_execution_log",
|
34
|
+
"update_monitor_task_with_execution_log",
|
35
|
+
]
|
@@ -0,0 +1,161 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from enum import Enum
|
4
|
+
from typing import List, Optional
|
5
|
+
|
6
|
+
from sqlalchemy import ARRAY, Column
|
7
|
+
from sqlalchemy import Enum as SQLAlchemyEnum
|
8
|
+
from sqlalchemy import ForeignKey, String
|
9
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
10
|
+
from sqlalchemy.orm import Session, relationship
|
11
|
+
|
12
|
+
from fides.api.db.base_class import Base, FidesBase # type: ignore[attr-defined]
|
13
|
+
from fides.api.models.detection_discovery.core import MonitorConfig
|
14
|
+
from fides.api.models.worker_task import (
|
15
|
+
ExecutionLogStatus,
|
16
|
+
TaskExecutionLog,
|
17
|
+
WorkerTask,
|
18
|
+
)
|
19
|
+
|
20
|
+
|
21
|
+
class MonitorTaskType(Enum):
|
22
|
+
"""
|
23
|
+
Types of tasks that can be executed by a worker.
|
24
|
+
"""
|
25
|
+
|
26
|
+
DETECTION = "detection"
|
27
|
+
CLASSIFICATION = "classification"
|
28
|
+
PROMOTION = "promotion"
|
29
|
+
|
30
|
+
|
31
|
+
class MonitorTask(WorkerTask, Base):
|
32
|
+
"""
|
33
|
+
A monitor task executed by a worker.
|
34
|
+
"""
|
35
|
+
|
36
|
+
# celery_id is used to track task executions. While MonitorTask.id remains constant,
|
37
|
+
# celery_id changes with each execution or retry of the task, allowing us to track
|
38
|
+
# the current execution state while maintaining a stable reference to the original task.
|
39
|
+
celery_id = Column(
|
40
|
+
String(255), unique=True, nullable=False, default=FidesBase.generate_uuid
|
41
|
+
)
|
42
|
+
task_arguments = Column(JSONB, nullable=True) # To be able to rerun the task
|
43
|
+
# Contains info, warning, or error messages
|
44
|
+
message = Column(String)
|
45
|
+
monitor_config_id = Column(
|
46
|
+
String,
|
47
|
+
ForeignKey(MonitorConfig.id_field_path, ondelete="CASCADE"),
|
48
|
+
index=True,
|
49
|
+
nullable=False,
|
50
|
+
)
|
51
|
+
staged_resource_urns = Column(ARRAY(String), nullable=True)
|
52
|
+
child_resource_urns = Column(ARRAY(String), nullable=True)
|
53
|
+
|
54
|
+
monitor_config = relationship(MonitorConfig, cascade="all, delete")
|
55
|
+
execution_logs = relationship(
|
56
|
+
"MonitorTaskExecutionLog", back_populates="monitor_task", cascade="all, delete"
|
57
|
+
)
|
58
|
+
|
59
|
+
@classmethod
|
60
|
+
def allowed_action_types(cls) -> List[str]:
|
61
|
+
return [e.value for e in MonitorTaskType]
|
62
|
+
|
63
|
+
|
64
|
+
class TaskRunType(Enum):
|
65
|
+
"""
|
66
|
+
Type of task run.
|
67
|
+
"""
|
68
|
+
|
69
|
+
MANUAL = "manual"
|
70
|
+
SYSTEM = "system"
|
71
|
+
|
72
|
+
|
73
|
+
class MonitorTaskExecutionLog(TaskExecutionLog, Base):
|
74
|
+
"""
|
75
|
+
Stores the individual execution logs associated with a MonitorTask.
|
76
|
+
"""
|
77
|
+
|
78
|
+
# This celery_id preserves the specific execution ID for historical tracking,
|
79
|
+
# unlike MonitorTask.celery_id which is updated with each execution.
|
80
|
+
# This allows us to maintain a complete history of all task execution attempts.
|
81
|
+
celery_id = Column(String(255), nullable=False)
|
82
|
+
monitor_task_id = Column(
|
83
|
+
String,
|
84
|
+
ForeignKey(MonitorTask.id_field_path, ondelete="CASCADE"),
|
85
|
+
index=True,
|
86
|
+
nullable=False,
|
87
|
+
)
|
88
|
+
run_type = Column(
|
89
|
+
SQLAlchemyEnum(TaskRunType), nullable=False, default=TaskRunType.SYSTEM
|
90
|
+
)
|
91
|
+
|
92
|
+
monitor_task = relationship("MonitorTask", back_populates="execution_logs")
|
93
|
+
|
94
|
+
|
95
|
+
def create_monitor_task_with_execution_log(
|
96
|
+
db: Session, monitor_task_data: dict
|
97
|
+
) -> MonitorTask:
|
98
|
+
"""
|
99
|
+
Creates a monitor task with an execution log.
|
100
|
+
The default status is pending for the task and pending for the execution log.
|
101
|
+
"""
|
102
|
+
status = ExecutionLogStatus.pending
|
103
|
+
task_record = MonitorTask( # type: ignore
|
104
|
+
status=status.value,
|
105
|
+
**monitor_task_data,
|
106
|
+
)
|
107
|
+
db.add(task_record)
|
108
|
+
db.flush()
|
109
|
+
|
110
|
+
execution_log = MonitorTaskExecutionLog( # type: ignore
|
111
|
+
monitor_task=task_record, celery_id=task_record.celery_id, status=status
|
112
|
+
)
|
113
|
+
db.add(execution_log)
|
114
|
+
|
115
|
+
db.commit()
|
116
|
+
db.refresh(task_record)
|
117
|
+
return task_record
|
118
|
+
|
119
|
+
|
120
|
+
def update_monitor_task_with_execution_log(
|
121
|
+
db: Session,
|
122
|
+
status: ExecutionLogStatus,
|
123
|
+
task_record: Optional[MonitorTask] = None,
|
124
|
+
celery_id: Optional[str] = None,
|
125
|
+
message: Optional[str] = None,
|
126
|
+
run_type: TaskRunType = TaskRunType.SYSTEM,
|
127
|
+
) -> MonitorTask:
|
128
|
+
"""
|
129
|
+
Updates a monitor task with an execution log.
|
130
|
+
|
131
|
+
It must be either celery_id or task_record. If it doesn't receive a celery_id, it's assumed a new one needs to be created because a new run is about to be performed.
|
132
|
+
If it receives a celery_id, it means it only needs to update the status of an existing run. It can receive task_record to avoid querying the database again to get it.
|
133
|
+
"""
|
134
|
+
if not celery_id and not task_record:
|
135
|
+
raise ValueError("Either celery_id or task_record must be provided")
|
136
|
+
|
137
|
+
if celery_id and not task_record:
|
138
|
+
task_record = MonitorTask.get_by(db=db, field="celery_id", value=celery_id)
|
139
|
+
if not task_record:
|
140
|
+
raise ValueError(f"Could not find MonitorTask with celery_id {celery_id}")
|
141
|
+
|
142
|
+
assert task_record is not None # help type checker understand the control flow
|
143
|
+
|
144
|
+
if not celery_id:
|
145
|
+
celery_id = task_record.generate_uuid()
|
146
|
+
task_record.celery_id = celery_id
|
147
|
+
|
148
|
+
task_record.status = status.value # type: ignore
|
149
|
+
task_record.message = message
|
150
|
+
|
151
|
+
MonitorTaskExecutionLog( # type: ignore
|
152
|
+
monitor_task=task_record,
|
153
|
+
status=status,
|
154
|
+
message=message,
|
155
|
+
celery_id=celery_id,
|
156
|
+
run_type=run_type,
|
157
|
+
)
|
158
|
+
|
159
|
+
db.commit()
|
160
|
+
db.refresh(task_record)
|
161
|
+
return task_record
|