ethyca-fides 2.68.0rc3__py2.py3-none-any.whl → 2.68.1b1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.68.0rc3.dist-info → ethyca_fides-2.68.1b1.dist-info}/METADATA +1 -1
- {ethyca_fides-2.68.0rc3.dist-info → ethyca_fides-2.68.1b1.dist-info}/RECORD +100 -98
- fides/_version.py +3 -3
- fides/api/schemas/application_config.py +2 -1
- fides/api/schemas/privacy_center_config.py +15 -0
- fides/api/task/conditional_dependencies/evaluator.py +192 -45
- fides/api/task/conditional_dependencies/logging_utils.py +196 -0
- fides/api/task/conditional_dependencies/operators.py +8 -2
- fides/api/task/conditional_dependencies/schemas.py +25 -1
- fides/api/task/graph_task.py +9 -2
- fides/api/task/manual/manual_task_conditional_evaluation.py +193 -0
- fides/api/task/manual/manual_task_graph_task.py +213 -119
- fides/api/task/manual/manual_task_utils.py +0 -4
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- {ethyca_fides-2.68.0rc3.dist-info → ethyca_fides-2.68.1b1.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.68.0rc3.dist-info → ethyca_fides-2.68.1b1.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.68.0rc3.dist-info → ethyca_fides-2.68.1b1.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.68.0rc3.dist-info → ethyca_fides-2.68.1b1.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{5XFHjjKVYqngLW-SDXDX4 → tzF4yti8NslASlGnxnZ8m}/_buildManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/{5XFHjjKVYqngLW-SDXDX4 → tzF4yti8NslASlGnxnZ8m}/_ssgManifest.js +0 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
from typing import Any, Optional
|
|
2
|
+
|
|
3
|
+
from pydantic.v1.utils import deep_update
|
|
4
|
+
from sqlalchemy.orm import Session
|
|
5
|
+
|
|
6
|
+
from fides.api.graph.config import CollectionAddress, FieldAddress
|
|
7
|
+
from fides.api.models.manual_task import ManualTask
|
|
8
|
+
from fides.api.models.manual_task.conditional_dependency import (
|
|
9
|
+
ManualTaskConditionalDependency,
|
|
10
|
+
)
|
|
11
|
+
from fides.api.task.conditional_dependencies.evaluator import ConditionEvaluator
|
|
12
|
+
from fides.api.task.conditional_dependencies.schemas import EvaluationResult
|
|
13
|
+
from fides.api.util.collection_util import Row
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def extract_conditional_dependency_data_from_inputs(
|
|
17
|
+
*inputs: list[Row], manual_task: ManualTask, input_keys: list[CollectionAddress]
|
|
18
|
+
) -> dict[str, Any]:
|
|
19
|
+
"""
|
|
20
|
+
Extract data for conditional dependency field addresses from input data.
|
|
21
|
+
|
|
22
|
+
This method processes data from upstream regular tasks that provide fields
|
|
23
|
+
referenced in manual task conditional dependencies. It extracts the relevant
|
|
24
|
+
field values and makes them available for conditional dependency evaluation.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
*inputs: Input data from upstream nodes (regular tasks)
|
|
28
|
+
manual_task: Manual task to extract conditional dependencies from
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
Dictionary mapping field addresses to their values from input data
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
conditional_data: dict[str, Any] = {}
|
|
35
|
+
|
|
36
|
+
# Get all conditional dependencies field addresses
|
|
37
|
+
field_addresses = [
|
|
38
|
+
dependency.field_address
|
|
39
|
+
for dependency in manual_task.conditional_dependencies
|
|
40
|
+
if dependency.field_address
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
# if no field addresses, return empty conditional data
|
|
44
|
+
# This will allow the manual task to be executed if there are no conditional dependencies
|
|
45
|
+
if not field_addresses:
|
|
46
|
+
return conditional_data
|
|
47
|
+
|
|
48
|
+
# For manual tasks, we need to preserve the original field names from conditional dependencies
|
|
49
|
+
# Instead of using pre_process_input_data which consolidates fields, we'll extract directly
|
|
50
|
+
# from the raw input data based on the execution node's input_keys
|
|
51
|
+
|
|
52
|
+
# Create a mapping between collections and their input data
|
|
53
|
+
# Convert CollectionAddress objects to strings for consistent key types
|
|
54
|
+
collection_data_map = {
|
|
55
|
+
str(collection_key): input_data
|
|
56
|
+
for collection_key, input_data in zip(input_keys, inputs)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
# Extract data for each conditional dependency field address
|
|
60
|
+
for field_address in field_addresses:
|
|
61
|
+
source_collection_key, field_path = parse_field_address(field_address)
|
|
62
|
+
|
|
63
|
+
# Find the input data for this collection
|
|
64
|
+
field_value = None
|
|
65
|
+
input_data = collection_data_map.get(source_collection_key)
|
|
66
|
+
if input_data:
|
|
67
|
+
|
|
68
|
+
# Look for the field in the input data
|
|
69
|
+
for row in input_data:
|
|
70
|
+
|
|
71
|
+
# Traverse the nested field path to get the actual value
|
|
72
|
+
field_value = extract_nested_field_value(row, field_path)
|
|
73
|
+
|
|
74
|
+
if field_value is not None:
|
|
75
|
+
break
|
|
76
|
+
# Found the field value, break out of the inner loop (over rows)
|
|
77
|
+
# but continue with the outer loop to process this field
|
|
78
|
+
|
|
79
|
+
# Always include the field in conditional_data, even if value is None
|
|
80
|
+
# This allows conditional dependencies to evaluate existence, non-existence, and falsy values
|
|
81
|
+
nested_data = set_nested_value(field_address, field_value)
|
|
82
|
+
conditional_data = deep_update(conditional_data, nested_data)
|
|
83
|
+
|
|
84
|
+
return conditional_data
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def parse_field_address(field_address: str) -> tuple[str, list[str]]:
|
|
88
|
+
"""
|
|
89
|
+
Parse a field address into dataset, collection, and field path.
|
|
90
|
+
|
|
91
|
+
For complex addresses with > 2 colons, uses manual string parsing.
|
|
92
|
+
For simple addresses with ≤ 2 colons, uses FieldAddress.from_string().
|
|
93
|
+
"""
|
|
94
|
+
if field_address.count(":") > 2:
|
|
95
|
+
# Parse manually: dataset:collection:field:subfield -> dataset, collection, [field, subfield]
|
|
96
|
+
dataset, collection, *field_path = field_address.split(":")
|
|
97
|
+
source_collection_key = f"{dataset}:{collection}"
|
|
98
|
+
else:
|
|
99
|
+
# Use standard FieldAddress parsing for simple cases
|
|
100
|
+
field_address_obj = FieldAddress.from_string(field_address)
|
|
101
|
+
source_collection_key = str(field_address_obj.collection_address())
|
|
102
|
+
field_path = list(field_address_obj.field_path.levels)
|
|
103
|
+
return source_collection_key, field_path
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def extract_nested_field_value(data: Any, field_path: list[str]) -> Any:
|
|
107
|
+
"""
|
|
108
|
+
Extract a nested field value by traversing the field path.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
data: The data to extract from (usually a dict)
|
|
112
|
+
field_path: List of field names to traverse (e.g., ["profile", "preferences", "theme"])
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
The value at the end of the field path, or None if not found
|
|
116
|
+
"""
|
|
117
|
+
if not field_path:
|
|
118
|
+
return data
|
|
119
|
+
|
|
120
|
+
current = data
|
|
121
|
+
for field_name in field_path:
|
|
122
|
+
if isinstance(current, dict) and field_name in current:
|
|
123
|
+
current = current[field_name]
|
|
124
|
+
else:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
return current
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def set_nested_value(field_address: str, value: Any) -> dict[str, Any]:
|
|
131
|
+
"""
|
|
132
|
+
Set a field value in the conditional data structure.
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
field_address: Colon-separated field address (e.g., "dataset:collection:field" or "dataset:collection:nested:field")
|
|
136
|
+
value: The value to set
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
Dictionary with the field value set at the specified path
|
|
140
|
+
"""
|
|
141
|
+
# For conditional dependencies, we want to set the field value directly
|
|
142
|
+
# The field_address format is "dataset:collection:field" or "dataset:collection:nested:field"
|
|
143
|
+
# We want to create: {dataset: {collection: {field: value}}} or {dataset: {collection: {nested: {field: value}}}}
|
|
144
|
+
parts = field_address.split(":")
|
|
145
|
+
|
|
146
|
+
if len(parts) >= 3:
|
|
147
|
+
dataset, collection = parts[0], parts[1]
|
|
148
|
+
# Handle nested field paths beyond the first 3 parts
|
|
149
|
+
if len(parts) == 3:
|
|
150
|
+
# Simple case: dataset:collection:field
|
|
151
|
+
field = parts[2]
|
|
152
|
+
return {dataset: {collection: {field: value}}}
|
|
153
|
+
|
|
154
|
+
# Nested case: dataset:collection:nested:field
|
|
155
|
+
# Build the nested structure from the remaining parts
|
|
156
|
+
nested_structure = value
|
|
157
|
+
for part in reversed(parts[2:]):
|
|
158
|
+
nested_structure = {part: nested_structure}
|
|
159
|
+
return {dataset: {collection: nested_structure}}
|
|
160
|
+
|
|
161
|
+
# Fallback for unexpected formats
|
|
162
|
+
return {field_address: value}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def evaluate_conditional_dependencies(
|
|
166
|
+
db: Session, manual_task: ManualTask, conditional_data: dict[str, Any]
|
|
167
|
+
) -> Optional[EvaluationResult]:
|
|
168
|
+
"""
|
|
169
|
+
Evaluate conditional dependencies for a manual task using data from regular tasks.
|
|
170
|
+
|
|
171
|
+
This method evaluates whether a manual task should be executed based on its
|
|
172
|
+
conditional dependencies and the data received from upstream regular tasks.
|
|
173
|
+
|
|
174
|
+
Args:
|
|
175
|
+
manual_task: The manual task to evaluate
|
|
176
|
+
conditional_data: Data from regular tasks for conditional dependency fields
|
|
177
|
+
|
|
178
|
+
Returns:
|
|
179
|
+
EvaluationResult object containing detailed information about which conditions
|
|
180
|
+
were met or not met, or None if no conditional dependencies exist
|
|
181
|
+
"""
|
|
182
|
+
# Get the root condition for this manual task
|
|
183
|
+
root_condition = ManualTaskConditionalDependency.get_root_condition(
|
|
184
|
+
db, manual_task.id
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
if not root_condition:
|
|
188
|
+
# No conditional dependencies - always execute
|
|
189
|
+
return None
|
|
190
|
+
|
|
191
|
+
# Evaluate the condition using the data from regular tasks
|
|
192
|
+
evaluator = ConditionEvaluator(db)
|
|
193
|
+
return evaluator.evaluate_rule(root_condition, conditional_data)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from typing import Any, Optional
|
|
2
2
|
|
|
3
3
|
from loguru import logger
|
|
4
|
-
from
|
|
4
|
+
from pydantic.v1.utils import deep_update
|
|
5
5
|
|
|
6
6
|
from fides.api.common_exceptions import AwaitingAsyncTaskCallback
|
|
7
7
|
from fides.api.models.attachment import AttachmentType
|
|
@@ -15,13 +15,23 @@ from fides.api.models.manual_task import (
|
|
|
15
15
|
StatusType,
|
|
16
16
|
)
|
|
17
17
|
from fides.api.models.privacy_request import PrivacyRequest
|
|
18
|
+
from fides.api.models.worker_task import ExecutionLogStatus
|
|
18
19
|
from fides.api.schemas.policy import ActionType
|
|
19
20
|
from fides.api.schemas.privacy_request import PrivacyRequestStatus
|
|
21
|
+
from fides.api.task.conditional_dependencies.logging_utils import (
|
|
22
|
+
format_evaluation_failure_message,
|
|
23
|
+
format_evaluation_success_message,
|
|
24
|
+
)
|
|
20
25
|
from fides.api.task.graph_task import GraphTask, retry
|
|
21
26
|
from fides.api.task.manual.manual_task_address import ManualTaskAddress
|
|
27
|
+
from fides.api.task.manual.manual_task_conditional_evaluation import (
|
|
28
|
+
evaluate_conditional_dependencies,
|
|
29
|
+
extract_conditional_dependency_data_from_inputs,
|
|
30
|
+
)
|
|
22
31
|
from fides.api.task.manual.manual_task_utils import (
|
|
23
32
|
get_manual_task_for_connection_config,
|
|
24
33
|
)
|
|
34
|
+
from fides.api.task.task_resources import TaskResources
|
|
25
35
|
from fides.api.util.collection_util import Row
|
|
26
36
|
from fides.api.util.storage_util import format_size
|
|
27
37
|
|
|
@@ -29,88 +39,225 @@ from fides.api.util.storage_util import format_size
|
|
|
29
39
|
class ManualTaskGraphTask(GraphTask):
|
|
30
40
|
"""GraphTask implementation for ManualTask execution"""
|
|
31
41
|
|
|
42
|
+
# class level constants
|
|
43
|
+
DRY_RUN_PLACEHOLDER_VALUE = 1
|
|
44
|
+
|
|
45
|
+
def __init__(self, resources: TaskResources) -> None:
|
|
46
|
+
super().__init__(resources)
|
|
47
|
+
self.connection_key = ManualTaskAddress.get_connection_key(
|
|
48
|
+
self.execution_node.address
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# ------------------------------------------------------------------------------------------------
|
|
52
|
+
# Public methods
|
|
53
|
+
# ------------------------------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
def dry_run_task(self) -> int:
|
|
56
|
+
"""Return estimated row count for dry run - manual tasks don't have predictable counts"""
|
|
57
|
+
return self.DRY_RUN_PLACEHOLDER_VALUE
|
|
58
|
+
|
|
32
59
|
@retry(action_type=ActionType.access, default_return=[])
|
|
33
60
|
def access_request(self, *inputs: list[Row]) -> list[Row]:
|
|
61
|
+
"""
|
|
62
|
+
Execute manual task logic following the standard GraphTask pattern.
|
|
63
|
+
Calls _run_request with ACCESS configs.
|
|
64
|
+
Returns data if submitted, raise AwaitingAsyncTaskCallback if not
|
|
65
|
+
"""
|
|
66
|
+
result = self._run_request(
|
|
67
|
+
ManualTaskConfigurationType.access_privacy_request,
|
|
68
|
+
ActionType.access,
|
|
69
|
+
*inputs,
|
|
70
|
+
)
|
|
71
|
+
if result is None:
|
|
72
|
+
# Conditional skip or not applicable already logged upstream; do not mark complete here
|
|
73
|
+
return []
|
|
74
|
+
|
|
75
|
+
# We are picking up after awaiting input and have provided data – mark complete with record count
|
|
76
|
+
self.log_end(ActionType.access, record_count=len(result))
|
|
77
|
+
return result
|
|
78
|
+
|
|
79
|
+
# Provide erasure support for manual tasks
|
|
80
|
+
@retry(action_type=ActionType.erasure, default_return=0)
|
|
81
|
+
def erasure_request(
|
|
82
|
+
self,
|
|
83
|
+
retrieved_data: list[Row], # This is not used for manual tasks.
|
|
84
|
+
*erasure_prereqs: int, # noqa: D401, pylint: disable=unused-argument # TODO Remove when we stop support for DSR 2.0
|
|
85
|
+
inputs: Optional[list[list[Row]]] = None,
|
|
86
|
+
) -> int:
|
|
87
|
+
"""Execute manual-task-driven erasure logic.
|
|
88
|
+
Calls _run_request with ERASURE configs.
|
|
89
|
+
|
|
90
|
+
Mirrors access_request behaviour but returns the number of rows masked (always 0)
|
|
91
|
+
once all required manual task submissions are present. If submissions are
|
|
92
|
+
incomplete the privacy request is paused awaiting user input.
|
|
93
|
+
Returns the number of rows masked (always 0)
|
|
94
|
+
Raises AwaitingAsyncTaskCallback if data is not submitted
|
|
95
|
+
"""
|
|
96
|
+
if not inputs:
|
|
97
|
+
inputs = []
|
|
98
|
+
result = self._run_request(
|
|
99
|
+
ManualTaskConfigurationType.erasure_privacy_request,
|
|
100
|
+
ActionType.erasure,
|
|
101
|
+
*inputs,
|
|
102
|
+
)
|
|
103
|
+
if result is None:
|
|
104
|
+
# Conditional skip or not applicable already logged upstream; do not mark complete here
|
|
105
|
+
return 0
|
|
106
|
+
|
|
107
|
+
# Mark rows_masked = 0 (manual tasks do not mask data directly)
|
|
108
|
+
if self.request_task.id:
|
|
109
|
+
# Storing result for DSR 3.0; SQLAlchemy column typing triggers mypy warning
|
|
110
|
+
self.request_task.rows_masked = 0 # type: ignore[assignment]
|
|
111
|
+
|
|
112
|
+
# Picking up after awaiting input, mark erasure node complete with rows masked count (always 0)
|
|
113
|
+
self.log_end(ActionType.erasure, record_count=0)
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
# ------------------------------------------------------------------------------------------------
|
|
117
|
+
# Private methods
|
|
118
|
+
# ------------------------------------------------------------------------------------------------
|
|
119
|
+
|
|
120
|
+
def _run_request(
|
|
121
|
+
self,
|
|
122
|
+
config_type: ManualTaskConfigurationType,
|
|
123
|
+
action_type: ActionType,
|
|
124
|
+
*inputs: list[Row],
|
|
125
|
+
) -> Optional[list[Row]]:
|
|
34
126
|
"""
|
|
35
127
|
Execute manual task logic following the standard GraphTask pattern:
|
|
36
128
|
1. Create ManualTaskInstances if they don't exist
|
|
37
|
-
2. Check
|
|
129
|
+
2. Check if all required submissions are present
|
|
38
130
|
3. Return data if submitted, raise AwaitingAsyncTaskCallback if not
|
|
39
131
|
"""
|
|
40
|
-
|
|
41
|
-
|
|
132
|
+
manual_task = self._get_manual_task_or_none()
|
|
133
|
+
if manual_task is None:
|
|
134
|
+
return None
|
|
42
135
|
|
|
43
|
-
#
|
|
44
|
-
|
|
45
|
-
raise ValueError(f"Invalid manual task address: {collection_address}")
|
|
136
|
+
# Complete a series of checks to determine if the manual task should be executed
|
|
137
|
+
# If any of these checks fail, complete immediately or mark as skipped
|
|
46
138
|
|
|
47
|
-
|
|
139
|
+
# Check if any eligible manual tasks have applicable configs
|
|
140
|
+
if not self._check_manual_task_configs(manual_task, config_type, action_type):
|
|
141
|
+
return None
|
|
48
142
|
|
|
49
|
-
#
|
|
50
|
-
|
|
143
|
+
# Check if there are any rules for this action type
|
|
144
|
+
if not self.resources.request.policy.get_rules_for_action(
|
|
145
|
+
action_type=action_type
|
|
146
|
+
):
|
|
147
|
+
return None
|
|
51
148
|
|
|
52
|
-
|
|
53
|
-
|
|
149
|
+
# Extract conditional dependency data from inputs
|
|
150
|
+
conditional_data = extract_conditional_dependency_data_from_inputs(
|
|
151
|
+
*inputs, manual_task=manual_task, input_keys=self.execution_node.input_keys
|
|
152
|
+
)
|
|
153
|
+
# Evaluate conditional dependencies
|
|
154
|
+
evaluation_result = evaluate_conditional_dependencies(
|
|
155
|
+
self.resources.session, manual_task, conditional_data=conditional_data
|
|
156
|
+
)
|
|
157
|
+
detailed_message: Optional[str] = None
|
|
158
|
+
# if there were conditional dependencies and they were not met,
|
|
159
|
+
# clean up any existing ManualTaskInstances and return None to cause a skip
|
|
160
|
+
if evaluation_result is not None and not evaluation_result.result:
|
|
161
|
+
self._cleanup_manual_task_instances(manual_task, self.resources.request)
|
|
162
|
+
detailed_message = format_evaluation_failure_message(evaluation_result)
|
|
163
|
+
self.update_status(
|
|
164
|
+
f"Manual task conditional dependencies not met. {detailed_message}",
|
|
165
|
+
[],
|
|
166
|
+
ActionType(self.resources.privacy_request_task.action_type),
|
|
167
|
+
ExecutionLogStatus.skipped,
|
|
168
|
+
)
|
|
169
|
+
return None
|
|
54
170
|
|
|
55
|
-
# Check
|
|
56
|
-
|
|
171
|
+
# Check/Create manual task instances for applicable configs only
|
|
172
|
+
self._ensure_manual_task_instances(
|
|
173
|
+
manual_task,
|
|
174
|
+
self.resources.request,
|
|
175
|
+
config_type,
|
|
176
|
+
)
|
|
57
177
|
|
|
178
|
+
# Check if all manual task instances have submissions for applicable configs only
|
|
179
|
+
# No separate pending log; include details in the awaiting-processing log
|
|
180
|
+
if evaluation_result:
|
|
181
|
+
detailed_message = format_evaluation_success_message(evaluation_result)
|
|
182
|
+
result = self._set_submitted_data_or_raise_awaiting_async_task_callback(
|
|
183
|
+
manual_task,
|
|
184
|
+
config_type,
|
|
185
|
+
action_type,
|
|
186
|
+
conditional_data=conditional_data,
|
|
187
|
+
awaiting_detail_message=detailed_message,
|
|
188
|
+
)
|
|
189
|
+
return result
|
|
190
|
+
|
|
191
|
+
def _check_manual_task_configs(
|
|
192
|
+
self,
|
|
193
|
+
manual_task: ManualTask,
|
|
194
|
+
config_type: ManualTaskConfigurationType,
|
|
195
|
+
action_type: ActionType,
|
|
196
|
+
) -> bool:
|
|
58
197
|
has_access_configs = [
|
|
59
198
|
config
|
|
60
199
|
for config in manual_task.configs
|
|
61
|
-
if config.is_current
|
|
62
|
-
and config.config_type == ManualTaskConfigurationType.access_privacy_request
|
|
200
|
+
if config.is_current and config.config_type == config_type
|
|
63
201
|
]
|
|
64
202
|
|
|
65
203
|
if not has_access_configs:
|
|
66
204
|
# No access configs - complete immediately
|
|
67
|
-
self.log_end(
|
|
68
|
-
return
|
|
205
|
+
self.log_end(action_type)
|
|
206
|
+
return False
|
|
69
207
|
|
|
70
|
-
|
|
71
|
-
action_type=ActionType.access
|
|
72
|
-
):
|
|
73
|
-
# TODO: This will be changed with Manual Task Dependencies Implementation.
|
|
74
|
-
self.log_end(ActionType.access)
|
|
75
|
-
return []
|
|
208
|
+
return True
|
|
76
209
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
210
|
+
def _get_manual_task_or_none(self) -> Optional[ManualTask]:
|
|
211
|
+
# Verify this is a manual task address
|
|
212
|
+
if not ManualTaskAddress.is_manual_task_address(self.execution_node.address):
|
|
213
|
+
raise ValueError(
|
|
214
|
+
f"Invalid manual task address: {self.execution_node.address}"
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
# Get the manual task for this connection config (1:1 relationship)
|
|
218
|
+
manual_task = get_manual_task_for_connection_config(
|
|
219
|
+
self.resources.session, self.connection_key
|
|
83
220
|
)
|
|
221
|
+
return manual_task
|
|
84
222
|
|
|
223
|
+
def _set_submitted_data_or_raise_awaiting_async_task_callback(
|
|
224
|
+
self,
|
|
225
|
+
manual_task: ManualTask,
|
|
226
|
+
config_type: ManualTaskConfigurationType,
|
|
227
|
+
action_type: ActionType,
|
|
228
|
+
conditional_data: Optional[dict[str, Any]] = None,
|
|
229
|
+
awaiting_detail_message: Optional[str] = None,
|
|
230
|
+
) -> Optional[list[Row]]:
|
|
231
|
+
"""
|
|
232
|
+
Set submitted data for a manual task and raise AwaitingAsyncTaskCallback if all instances are not completed
|
|
233
|
+
"""
|
|
85
234
|
# Check if all manual task instances have submissions for ACCESS configs only
|
|
86
235
|
submitted_data = self._get_submitted_data(
|
|
87
|
-
db,
|
|
88
236
|
manual_task,
|
|
89
237
|
self.resources.request,
|
|
90
|
-
|
|
238
|
+
config_type,
|
|
239
|
+
conditional_data=conditional_data,
|
|
91
240
|
)
|
|
92
241
|
|
|
93
242
|
if submitted_data is not None:
|
|
94
243
|
result: list[Row] = [submitted_data] if submitted_data else []
|
|
95
244
|
self.request_task.access_data = result
|
|
96
245
|
|
|
97
|
-
# Mark request task as complete and write execution log
|
|
98
|
-
self.log_end(ActionType.access)
|
|
99
246
|
return result
|
|
100
247
|
|
|
101
248
|
# Set privacy request status to requires_input if not already set
|
|
102
249
|
if self.resources.request.status != PrivacyRequestStatus.requires_input:
|
|
103
250
|
self.resources.request.status = PrivacyRequestStatus.requires_input
|
|
104
|
-
self.resources.request.save(
|
|
251
|
+
self.resources.request.save(self.resources.session)
|
|
105
252
|
|
|
106
|
-
# This
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
253
|
+
# This will trigger log_awaiting_processing via the @retry decorator; include conditional details
|
|
254
|
+
base_msg = f"Manual task for {self.connection_key} requires user input"
|
|
255
|
+
if awaiting_detail_message:
|
|
256
|
+
base_msg = f"{base_msg}. {awaiting_detail_message}"
|
|
257
|
+
raise AwaitingAsyncTaskCallback(base_msg)
|
|
110
258
|
|
|
111
259
|
def _ensure_manual_task_instances(
|
|
112
260
|
self,
|
|
113
|
-
db: Session,
|
|
114
261
|
manual_task: ManualTask,
|
|
115
262
|
privacy_request: PrivacyRequest,
|
|
116
263
|
allowed_config_type: "ManualTaskConfigurationType",
|
|
@@ -139,6 +286,7 @@ class ManualTaskGraphTask(GraphTask):
|
|
|
139
286
|
|
|
140
287
|
# If no existing instances, create a new one for the current config
|
|
141
288
|
# There will only be one config of each type per manual task
|
|
289
|
+
# Sort by version descending to get the latest version first
|
|
142
290
|
config = next(
|
|
143
291
|
(
|
|
144
292
|
config
|
|
@@ -154,7 +302,7 @@ class ManualTaskGraphTask(GraphTask):
|
|
|
154
302
|
|
|
155
303
|
if config:
|
|
156
304
|
ManualTaskInstance.create(
|
|
157
|
-
db=
|
|
305
|
+
db=self.resources.session,
|
|
158
306
|
data={
|
|
159
307
|
"task_id": manual_task.id,
|
|
160
308
|
"config_id": config.id,
|
|
@@ -166,10 +314,10 @@ class ManualTaskGraphTask(GraphTask):
|
|
|
166
314
|
|
|
167
315
|
def _get_submitted_data(
|
|
168
316
|
self,
|
|
169
|
-
db: Session,
|
|
170
317
|
manual_task: ManualTask,
|
|
171
318
|
privacy_request: PrivacyRequest,
|
|
172
319
|
allowed_config_type: "ManualTaskConfigurationType",
|
|
320
|
+
conditional_data: Optional[dict[str, Any]] = None,
|
|
173
321
|
) -> Optional[dict[str, Any]]:
|
|
174
322
|
"""
|
|
175
323
|
Check if all manual task instances have submissions for ALL fields and return aggregated data
|
|
@@ -193,10 +341,15 @@ class ManualTaskGraphTask(GraphTask):
|
|
|
193
341
|
# Update status if needed
|
|
194
342
|
if inst.status != StatusType.completed:
|
|
195
343
|
inst.status = StatusType.completed
|
|
196
|
-
inst.save(
|
|
344
|
+
inst.save(self.resources.session)
|
|
197
345
|
|
|
198
346
|
# Aggregate submission data from all instances
|
|
199
347
|
aggregated_data = self._aggregate_submission_data(candidate_instances)
|
|
348
|
+
|
|
349
|
+
# Merge conditional data with aggregated submission data
|
|
350
|
+
if conditional_data:
|
|
351
|
+
aggregated_data = deep_update(aggregated_data, conditional_data)
|
|
352
|
+
|
|
200
353
|
return aggregated_data or None
|
|
201
354
|
|
|
202
355
|
def _aggregate_submission_data(
|
|
@@ -254,84 +407,25 @@ class ManualTaskGraphTask(GraphTask):
|
|
|
254
407
|
)
|
|
255
408
|
return attachment_map or None
|
|
256
409
|
|
|
257
|
-
def
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
# Provide erasure support for manual tasks
|
|
262
|
-
@retry(action_type=ActionType.erasure, default_return=0)
|
|
263
|
-
def erasure_request(
|
|
264
|
-
self,
|
|
265
|
-
retrieved_data: list[Row],
|
|
266
|
-
*erasure_prereqs: int, # noqa: D401, pylint: disable=unused-argument
|
|
267
|
-
inputs: Optional[list[list[Row]]] = None,
|
|
268
|
-
) -> int:
|
|
269
|
-
"""Execute manual-task-driven erasure logic.
|
|
270
|
-
|
|
271
|
-
Mirrors access_request behaviour but returns the number of rows masked (always 0)
|
|
272
|
-
once all required manual task submissions are present. If submissions are
|
|
273
|
-
incomplete the privacy request is paused awaiting user input.
|
|
410
|
+
def _cleanup_manual_task_instances(
|
|
411
|
+
self, manual_task: ManualTask, privacy_request: PrivacyRequest
|
|
412
|
+
) -> None:
|
|
274
413
|
"""
|
|
275
|
-
|
|
276
|
-
collection_address = self.execution_node.address
|
|
414
|
+
Clean up ManualTaskInstances for a manual task when conditional dependencies are not met.
|
|
277
415
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
if not manual_task:
|
|
287
|
-
# No manual tasks defined – nothing to erase
|
|
288
|
-
self.log_end(ActionType.erasure)
|
|
289
|
-
return 0
|
|
290
|
-
|
|
291
|
-
# Check if any manual tasks have ERASURE configs
|
|
292
|
-
has_erasure_configs = [
|
|
293
|
-
config
|
|
294
|
-
for config in manual_task.configs
|
|
295
|
-
if config.is_current
|
|
296
|
-
and config.config_type
|
|
297
|
-
== ManualTaskConfigurationType.erasure_privacy_request
|
|
416
|
+
This method removes any existing instances that were created before the conditional
|
|
417
|
+
dependency evaluation determined the task should not execute.
|
|
418
|
+
"""
|
|
419
|
+
# Find all instances for this manual task and privacy request
|
|
420
|
+
instances_to_remove = [
|
|
421
|
+
instance
|
|
422
|
+
for instance in privacy_request.manual_task_instances
|
|
423
|
+
if instance.task_id == manual_task.id
|
|
298
424
|
]
|
|
299
425
|
|
|
300
|
-
if
|
|
301
|
-
# No erasure configs - complete immediately
|
|
302
|
-
self.log_end(ActionType.erasure)
|
|
303
|
-
return 0
|
|
304
|
-
|
|
305
|
-
# Create ManualTaskInstances for ERASURE configs only
|
|
306
|
-
self._ensure_manual_task_instances(
|
|
307
|
-
db,
|
|
308
|
-
manual_task,
|
|
309
|
-
self.resources.request,
|
|
310
|
-
ManualTaskConfigurationType.erasure_privacy_request,
|
|
311
|
-
)
|
|
312
|
-
|
|
313
|
-
# Check for full submissions – reuse helper used by access flow, filtering ERASURE configs
|
|
314
|
-
submissions_complete = self._get_submitted_data(
|
|
315
|
-
db,
|
|
316
|
-
manual_task,
|
|
317
|
-
self.resources.request,
|
|
318
|
-
ManualTaskConfigurationType.erasure_privacy_request,
|
|
319
|
-
)
|
|
426
|
+
if instances_to_remove:
|
|
320
427
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
self.resources.
|
|
325
|
-
self.resources.request.save(db)
|
|
326
|
-
raise AwaitingAsyncTaskCallback(
|
|
327
|
-
f"Manual erasure task for {connection_key} requires user input"
|
|
328
|
-
)
|
|
329
|
-
|
|
330
|
-
# Mark rows_masked = 0 (manual tasks do not mask data directly)
|
|
331
|
-
if self.request_task.id:
|
|
332
|
-
# Storing result for DSR 3.0; SQLAlchemy column typing triggers mypy warning
|
|
333
|
-
self.request_task.rows_masked = 0 # type: ignore[assignment]
|
|
334
|
-
|
|
335
|
-
# Mark successful completion
|
|
336
|
-
self.log_end(ActionType.erasure)
|
|
337
|
-
return 0
|
|
428
|
+
# Remove instances from the database
|
|
429
|
+
for instance in instances_to_remove:
|
|
430
|
+
instance.delete(self.resources.session)
|
|
431
|
+
self.resources.session.commit()
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
from loguru import logger
|
|
4
3
|
from sqlalchemy.orm import Session
|
|
5
4
|
|
|
6
5
|
from fides.api.graph.config import (
|
|
@@ -110,9 +109,6 @@ def create_conditional_dependency_scalar_fields(
|
|
|
110
109
|
# Use the full field address as the field name to preserve collection context
|
|
111
110
|
# This allows the manual task to receive data from specific collections
|
|
112
111
|
# e.g., "user.name" or "customer.profile.email" instead of just "name" or "email"
|
|
113
|
-
logger.info(
|
|
114
|
-
f"Creating conditional dependency scalar field for field address: {field_address}"
|
|
115
|
-
)
|
|
116
112
|
field_address_obj = FieldAddress.from_string(field_address)
|
|
117
113
|
|
|
118
114
|
scalar_field = ScalarField(
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/e1628f15dd5f019b.css" as="style"/><link rel="stylesheet" href="/_next/static/css/e1628f15dd5f019b.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-69658aeaf6155d89.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-65723cd4b8fc36ac.js" defer=""></script><script src="/_next/static/chunks/pages/404-9174cdb70126c2c5.js" defer=""></script><script src="/_next/static/
|
|
1
|
+
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/e1628f15dd5f019b.css" as="style"/><link rel="stylesheet" href="/_next/static/css/e1628f15dd5f019b.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-69658aeaf6155d89.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-65723cd4b8fc36ac.js" defer=""></script><script src="/_next/static/chunks/pages/404-9174cdb70126c2c5.js" defer=""></script><script src="/_next/static/tzF4yti8NslASlGnxnZ8m/_buildManifest.js" defer=""></script><script src="/_next/static/tzF4yti8NslASlGnxnZ8m/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"tzF4yti8NslASlGnxnZ8m","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|