acryl-datahub-cloud 0.3.12rc1__py3-none-any.whl → 0.3.12rc4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub-cloud might be problematic. Click here for more details.
- acryl_datahub_cloud/_codegen_config.json +1 -1
- acryl_datahub_cloud/datahub_forms_notifications/forms_notifications_source.py +559 -0
- acryl_datahub_cloud/datahub_forms_notifications/get_search_results_total.gql +14 -0
- acryl_datahub_cloud/datahub_forms_notifications/query.py +17 -0
- acryl_datahub_cloud/datahub_forms_notifications/scroll_forms_for_notification.gql +29 -0
- acryl_datahub_cloud/datahub_forms_notifications/send_form_notification_request.gql +5 -0
- acryl_datahub_cloud/datahub_usage_reporting/query_builder.py +48 -8
- acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py +49 -40
- acryl_datahub_cloud/metadata/_urns/urn_defs.py +1842 -1786
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/application/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/form/__init__.py +4 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/notification/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +2 -0
- acryl_datahub_cloud/metadata/schema.avsc +24861 -24050
- acryl_datahub_cloud/metadata/schema_classes.py +1031 -631
- acryl_datahub_cloud/metadata/schemas/ApplicationKey.avsc +31 -0
- acryl_datahub_cloud/metadata/schemas/ApplicationProperties.avsc +72 -0
- acryl_datahub_cloud/metadata/schemas/Applications.avsc +38 -0
- acryl_datahub_cloud/metadata/schemas/AssertionAnalyticsRunEvent.avsc +40 -7
- acryl_datahub_cloud/metadata/schemas/AssertionInfo.avsc +27 -6
- acryl_datahub_cloud/metadata/schemas/AssertionRunEvent.avsc +31 -7
- acryl_datahub_cloud/metadata/schemas/AssertionsSummary.avsc +14 -0
- acryl_datahub_cloud/metadata/schemas/ChartKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/ConstraintInfo.avsc +12 -1
- acryl_datahub_cloud/metadata/schemas/ContainerKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/CorpGroupKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/CorpUserKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/DashboardKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataFlowKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
- acryl_datahub_cloud/metadata/schemas/DataJobKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataProductKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataProductProperties.avsc +1 -1
- acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/FormAssignmentStatus.avsc +36 -0
- acryl_datahub_cloud/metadata/schemas/FormInfo.avsc +6 -0
- acryl_datahub_cloud/metadata/schemas/FormKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/FormNotifications.avsc +69 -0
- acryl_datahub_cloud/metadata/schemas/FormSettings.avsc +3 -0
- acryl_datahub_cloud/metadata/schemas/GlobalSettingsInfo.avsc +22 -0
- acryl_datahub_cloud/metadata/schemas/GlossaryTermKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/MLFeatureKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/MLFeatureTableKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/MLModelGroupKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/MLModelKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/MLPrimaryKeyKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc +12 -1
- acryl_datahub_cloud/metadata/schemas/MonitorInfo.avsc +27 -6
- acryl_datahub_cloud/metadata/schemas/NotebookKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/NotificationRequest.avsc +1 -0
- acryl_datahub_cloud/notifications/__init__.py +0 -0
- acryl_datahub_cloud/notifications/notification_recipient_builder.py +399 -0
- acryl_datahub_cloud/sdk/__init__.py +29 -0
- acryl_datahub_cloud/{_sdk_extras → sdk}/assertion.py +501 -193
- acryl_datahub_cloud/sdk/assertion_input/__init__.py +0 -0
- acryl_datahub_cloud/{_sdk_extras → sdk/assertion_input}/assertion_input.py +733 -189
- acryl_datahub_cloud/sdk/assertion_input/freshness_assertion_input.py +261 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_column_metric_assertion_input.py +947 -0
- acryl_datahub_cloud/sdk/assertions_client.py +1639 -0
- acryl_datahub_cloud/sdk/entities/__init__.py +0 -0
- acryl_datahub_cloud/{_sdk_extras → sdk}/entities/assertion.py +5 -2
- acryl_datahub_cloud/{_sdk_extras → sdk}/subscription_client.py +146 -33
- {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/METADATA +48 -43
- {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/RECORD +72 -54
- {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/entry_points.txt +1 -0
- acryl_datahub_cloud/_sdk_extras/__init__.py +0 -19
- acryl_datahub_cloud/_sdk_extras/assertions_client.py +0 -717
- /acryl_datahub_cloud/{_sdk_extras/entities → datahub_forms_notifications}/__init__.py +0 -0
- /acryl_datahub_cloud/{_sdk_extras → sdk}/entities/monitor.py +0 -0
- /acryl_datahub_cloud/{_sdk_extras → sdk}/entities/subscription.py +0 -0
- /acryl_datahub_cloud/{_sdk_extras → sdk}/errors.py +0 -0
- /acryl_datahub_cloud/{_sdk_extras → sdk}/resolver_client.py +0 -0
- {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/WHEEL +0 -0
- {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/top_level.txt +0 -0
|
@@ -10,28 +10,32 @@ import logging
|
|
|
10
10
|
from abc import ABC, abstractmethod
|
|
11
11
|
from datetime import datetime
|
|
12
12
|
from enum import Enum
|
|
13
|
-
from typing import
|
|
13
|
+
from typing import Optional, Union
|
|
14
14
|
|
|
15
15
|
from typing_extensions import Self
|
|
16
16
|
|
|
17
|
-
from acryl_datahub_cloud.
|
|
17
|
+
from acryl_datahub_cloud.sdk.assertion_input.assertion_input import (
|
|
18
18
|
ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
|
|
19
19
|
DEFAULT_DETECTION_MECHANISM,
|
|
20
|
+
DEFAULT_SCHEDULE,
|
|
20
21
|
DEFAULT_SENSITIVITY,
|
|
21
22
|
AssertionIncidentBehavior,
|
|
22
23
|
DetectionMechanism,
|
|
23
24
|
ExclusionWindowTypes,
|
|
24
25
|
FixedRangeExclusionWindow,
|
|
25
26
|
InferenceSensitivity,
|
|
27
|
+
TimeWindowSizeInputTypes,
|
|
26
28
|
_DetectionMechanismTypes,
|
|
27
29
|
)
|
|
28
|
-
from acryl_datahub_cloud.
|
|
29
|
-
from acryl_datahub_cloud.
|
|
30
|
-
|
|
30
|
+
from acryl_datahub_cloud.sdk.entities.assertion import Assertion
|
|
31
|
+
from acryl_datahub_cloud.sdk.entities.monitor import (
|
|
32
|
+
Monitor,
|
|
33
|
+
_get_nested_field_for_entity_with_default,
|
|
34
|
+
)
|
|
35
|
+
from acryl_datahub_cloud.sdk.errors import SDKNotYetSupportedError
|
|
31
36
|
from datahub.emitter.mce_builder import parse_ts_millis
|
|
32
37
|
from datahub.metadata import schema_classes as models
|
|
33
38
|
from datahub.metadata.urns import AssertionUrn, CorpUserUrn, DatasetUrn, TagUrn
|
|
34
|
-
from datahub.sdk.entity import Entity
|
|
35
39
|
|
|
36
40
|
logger = logging.getLogger(__name__)
|
|
37
41
|
|
|
@@ -48,34 +52,33 @@ class AssertionMode(Enum):
|
|
|
48
52
|
# PASSIVE = "PASSIVE" # Not supported in the user facing interface.
|
|
49
53
|
|
|
50
54
|
|
|
51
|
-
|
|
52
|
-
entity: Entity,
|
|
53
|
-
field_path: str,
|
|
54
|
-
default: Any = None,
|
|
55
|
-
) -> Any:
|
|
55
|
+
class _HasSchedule:
|
|
56
56
|
"""
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
Args:
|
|
60
|
-
entity: The entity to get the nested field from.
|
|
61
|
-
field_path: The path to the nested field.
|
|
62
|
-
default: The default value to return if the field is not found.
|
|
57
|
+
Mixin class that provides schedule functionality for assertions.
|
|
63
58
|
"""
|
|
64
|
-
fields = field_path.split(".")
|
|
65
|
-
current = entity
|
|
66
|
-
last_valid_path = entity.entity_type_name()
|
|
67
|
-
|
|
68
|
-
for field in fields:
|
|
69
|
-
try:
|
|
70
|
-
current = getattr(current, field)
|
|
71
|
-
last_valid_path = f"{last_valid_path}.{field}"
|
|
72
|
-
except AttributeError:
|
|
73
|
-
logger.warning(
|
|
74
|
-
f"{entity.entity_type_name().capitalize()} {entity.urn} does not have an `{last_valid_path}` field, defaulting to {default}"
|
|
75
|
-
)
|
|
76
|
-
return default
|
|
77
59
|
|
|
78
|
-
|
|
60
|
+
def __init__(self, schedule: models.CronScheduleClass) -> None:
|
|
61
|
+
self._schedule = schedule
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def schedule(self) -> models.CronScheduleClass:
|
|
65
|
+
return self._schedule
|
|
66
|
+
|
|
67
|
+
@staticmethod
|
|
68
|
+
def _get_schedule(monitor: Monitor) -> models.CronScheduleClass:
|
|
69
|
+
"""Get the schedule from the monitor."""
|
|
70
|
+
assertion_evaluation_specs = _get_nested_field_for_entity_with_default(
|
|
71
|
+
monitor,
|
|
72
|
+
"info.assertionMonitor.assertions",
|
|
73
|
+
[],
|
|
74
|
+
)
|
|
75
|
+
if len(assertion_evaluation_specs) == 0:
|
|
76
|
+
return DEFAULT_SCHEDULE
|
|
77
|
+
assertion_evaluation_spec = assertion_evaluation_specs[0]
|
|
78
|
+
schedule = assertion_evaluation_spec.schedule
|
|
79
|
+
if schedule is None:
|
|
80
|
+
return DEFAULT_SCHEDULE
|
|
81
|
+
return schedule
|
|
79
82
|
|
|
80
83
|
|
|
81
84
|
class _HasSmartFunctionality:
|
|
@@ -89,10 +92,6 @@ class _HasSmartFunctionality:
|
|
|
89
92
|
sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
|
|
90
93
|
exclusion_windows: list[ExclusionWindowTypes],
|
|
91
94
|
training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
|
|
92
|
-
incident_behavior: list[AssertionIncidentBehavior],
|
|
93
|
-
detection_mechanism: Optional[
|
|
94
|
-
_DetectionMechanismTypes
|
|
95
|
-
] = DEFAULT_DETECTION_MECHANISM,
|
|
96
95
|
) -> None:
|
|
97
96
|
"""
|
|
98
97
|
Initialize the smart functionality mixin.
|
|
@@ -108,8 +107,6 @@ class _HasSmartFunctionality:
|
|
|
108
107
|
self._sensitivity = sensitivity
|
|
109
108
|
self._exclusion_windows = exclusion_windows
|
|
110
109
|
self._training_data_lookback_days = training_data_lookback_days
|
|
111
|
-
self._incident_behavior = incident_behavior
|
|
112
|
-
self._detection_mechanism = detection_mechanism
|
|
113
110
|
|
|
114
111
|
@property
|
|
115
112
|
def sensitivity(self) -> InferenceSensitivity:
|
|
@@ -123,14 +120,6 @@ class _HasSmartFunctionality:
|
|
|
123
120
|
def training_data_lookback_days(self) -> int:
|
|
124
121
|
return self._training_data_lookback_days
|
|
125
122
|
|
|
126
|
-
@property
|
|
127
|
-
def incident_behavior(self) -> list[AssertionIncidentBehavior]:
|
|
128
|
-
return self._incident_behavior
|
|
129
|
-
|
|
130
|
-
@property
|
|
131
|
-
def detection_mechanism(self) -> Optional[_DetectionMechanismTypes]:
|
|
132
|
-
return self._detection_mechanism
|
|
133
|
-
|
|
134
123
|
@staticmethod
|
|
135
124
|
def _get_sensitivity(monitor: Monitor) -> InferenceSensitivity:
|
|
136
125
|
# 1. Check if the monitor has a sensitivity field
|
|
@@ -179,6 +168,115 @@ class _HasSmartFunctionality:
|
|
|
179
168
|
assert isinstance(retrieved, int)
|
|
180
169
|
return retrieved
|
|
181
170
|
|
|
171
|
+
|
|
172
|
+
class _AssertionPublic(ABC):
|
|
173
|
+
"""
|
|
174
|
+
Abstract base class that represents a public facing assertion and contains the common properties of all assertions.
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
_SUPPORTED_WITH_FILTER_ASSERTION_TYPES = (
|
|
178
|
+
models.FreshnessAssertionInfoClass,
|
|
179
|
+
models.VolumeAssertionInfoClass,
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
def __init__(
|
|
183
|
+
self,
|
|
184
|
+
*,
|
|
185
|
+
urn: AssertionUrn,
|
|
186
|
+
dataset_urn: DatasetUrn,
|
|
187
|
+
display_name: str,
|
|
188
|
+
mode: AssertionMode,
|
|
189
|
+
tags: list[TagUrn],
|
|
190
|
+
incident_behavior: list[AssertionIncidentBehavior],
|
|
191
|
+
detection_mechanism: Optional[
|
|
192
|
+
_DetectionMechanismTypes
|
|
193
|
+
] = DEFAULT_DETECTION_MECHANISM,
|
|
194
|
+
created_by: Optional[CorpUserUrn] = None,
|
|
195
|
+
created_at: Union[datetime, None] = None,
|
|
196
|
+
updated_by: Optional[CorpUserUrn] = None,
|
|
197
|
+
updated_at: Optional[datetime] = None,
|
|
198
|
+
):
|
|
199
|
+
"""
|
|
200
|
+
Initialize the public facing assertion class.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
urn: The urn of the assertion.
|
|
204
|
+
dataset_urn: The urn of the dataset that the assertion is for.
|
|
205
|
+
display_name: The display name of the assertion.
|
|
206
|
+
mode: The mode of the assertion (active, inactive).
|
|
207
|
+
tags: The tags of the assertion.
|
|
208
|
+
created_by: The urn of the user that created the assertion.
|
|
209
|
+
created_at: The timestamp of when the assertion was created.
|
|
210
|
+
updated_by: The urn of the user that updated the assertion.
|
|
211
|
+
updated_at: The timestamp of when the assertion was updated.
|
|
212
|
+
"""
|
|
213
|
+
self._urn = urn
|
|
214
|
+
self._dataset_urn = dataset_urn
|
|
215
|
+
self._display_name = display_name
|
|
216
|
+
self._mode = mode
|
|
217
|
+
self._incident_behavior = incident_behavior
|
|
218
|
+
self._detection_mechanism = detection_mechanism
|
|
219
|
+
self._created_by = created_by
|
|
220
|
+
self._created_at = created_at
|
|
221
|
+
self._updated_by = updated_by
|
|
222
|
+
self._updated_at = updated_at
|
|
223
|
+
self._tags = tags
|
|
224
|
+
|
|
225
|
+
@property
|
|
226
|
+
def urn(self) -> AssertionUrn:
|
|
227
|
+
return self._urn
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def dataset_urn(self) -> DatasetUrn:
|
|
231
|
+
return self._dataset_urn
|
|
232
|
+
|
|
233
|
+
@property
|
|
234
|
+
def display_name(self) -> str:
|
|
235
|
+
return self._display_name
|
|
236
|
+
|
|
237
|
+
@property
|
|
238
|
+
def mode(self) -> AssertionMode:
|
|
239
|
+
return self._mode
|
|
240
|
+
|
|
241
|
+
@property
|
|
242
|
+
def incident_behavior(self) -> list[AssertionIncidentBehavior]:
|
|
243
|
+
return self._incident_behavior
|
|
244
|
+
|
|
245
|
+
@property
|
|
246
|
+
def detection_mechanism(self) -> Optional[_DetectionMechanismTypes]:
|
|
247
|
+
return self._detection_mechanism
|
|
248
|
+
|
|
249
|
+
@property
|
|
250
|
+
def created_by(self) -> Optional[CorpUserUrn]:
|
|
251
|
+
return self._created_by
|
|
252
|
+
|
|
253
|
+
@property
|
|
254
|
+
def created_at(self) -> Union[datetime, None]:
|
|
255
|
+
return self._created_at
|
|
256
|
+
|
|
257
|
+
@property
|
|
258
|
+
def updated_by(self) -> Optional[CorpUserUrn]:
|
|
259
|
+
return self._updated_by
|
|
260
|
+
|
|
261
|
+
@property
|
|
262
|
+
def updated_at(self) -> Union[datetime, None]:
|
|
263
|
+
return self._updated_at
|
|
264
|
+
|
|
265
|
+
@property
|
|
266
|
+
def tags(self) -> list[TagUrn]:
|
|
267
|
+
return self._tags
|
|
268
|
+
|
|
269
|
+
@staticmethod
|
|
270
|
+
def _get_incident_behavior(assertion: Assertion) -> list[AssertionIncidentBehavior]:
|
|
271
|
+
incident_behaviors = []
|
|
272
|
+
for action in assertion.on_failure + assertion.on_success:
|
|
273
|
+
if action.type == models.AssertionActionTypeClass.RAISE_INCIDENT:
|
|
274
|
+
incident_behaviors.append(AssertionIncidentBehavior.RAISE_ON_FAIL)
|
|
275
|
+
elif action.type == models.AssertionActionTypeClass.RESOLVE_INCIDENT:
|
|
276
|
+
incident_behaviors.append(AssertionIncidentBehavior.RESOLVE_ON_PASS)
|
|
277
|
+
|
|
278
|
+
return incident_behaviors
|
|
279
|
+
|
|
182
280
|
@staticmethod
|
|
183
281
|
def _get_detection_mechanism(
|
|
184
282
|
assertion: Assertion,
|
|
@@ -186,7 +284,7 @@ class _HasSmartFunctionality:
|
|
|
186
284
|
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
187
285
|
) -> Optional[_DetectionMechanismTypes]:
|
|
188
286
|
"""Get the detection mechanism from the monitor and assertion."""
|
|
189
|
-
if not
|
|
287
|
+
if not _AssertionPublic._has_valid_monitor_info(monitor):
|
|
190
288
|
return default
|
|
191
289
|
|
|
192
290
|
# 1. Check if the assertion has a parameters field
|
|
@@ -199,70 +297,35 @@ class _HasSmartFunctionality:
|
|
|
199
297
|
)
|
|
200
298
|
return default
|
|
201
299
|
|
|
202
|
-
parameters =
|
|
300
|
+
parameters = _AssertionPublic._get_assertion_parameters(monitor, default)
|
|
203
301
|
if parameters is None:
|
|
204
302
|
return _warn_and_return_default_detection_mechanism("parameters", default)
|
|
205
303
|
|
|
206
304
|
# 2. Convert the raw detection mechanism to the SDK detection mechanism
|
|
207
|
-
if
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
305
|
+
if parameters.type in [
|
|
306
|
+
models.AssertionEvaluationParametersTypeClass.DATASET_FRESHNESS,
|
|
307
|
+
models.AssertionEvaluationParametersTypeClass.DATASET_VOLUME,
|
|
308
|
+
]:
|
|
309
|
+
if assertion.info is None:
|
|
310
|
+
return _warn_and_return_default_detection_mechanism("info", default)
|
|
311
|
+
if isinstance(assertion.info, models.VolumeAssertionInfoClass):
|
|
312
|
+
return _AssertionPublic._get_volume_detection_mechanism(
|
|
313
|
+
assertion, parameters, default
|
|
314
|
+
)
|
|
315
|
+
elif isinstance(assertion.info, models.FreshnessAssertionInfoClass):
|
|
316
|
+
return _AssertionPublic._get_freshness_detection_mechanism(
|
|
317
|
+
assertion, parameters, default
|
|
318
|
+
)
|
|
319
|
+
# TODO: Consider moving the detection mechanism logic to the assertion classes themselves e.g. _get_assertion_specific_detection_mechanism as an abstract method
|
|
320
|
+
# TODO: Add support here for other detection mechanisms when other assertion types are supported
|
|
321
|
+
else:
|
|
322
|
+
raise SDKNotYetSupportedError(
|
|
323
|
+
f"AssertionType {type(assertion.info).__name__}"
|
|
324
|
+
)
|
|
215
325
|
else:
|
|
216
326
|
raise SDKNotYetSupportedError(
|
|
217
|
-
f"AssertionEvaluationParametersType {parameters.type}"
|
|
218
|
-
)
|
|
219
|
-
|
|
220
|
-
@staticmethod
|
|
221
|
-
def _has_valid_monitor_info(monitor: Monitor) -> bool:
|
|
222
|
-
"""Check if monitor has valid info and assertion monitor."""
|
|
223
|
-
|
|
224
|
-
def _warn_and_return_false(field_name: str) -> bool:
|
|
225
|
-
logger.warning(
|
|
226
|
-
f"Monitor {monitor.urn} does not have an `{field_name}` field, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
|
|
227
|
-
)
|
|
228
|
-
return False
|
|
229
|
-
|
|
230
|
-
if monitor.info is None:
|
|
231
|
-
return _warn_and_return_false("info")
|
|
232
|
-
if monitor.info.assertionMonitor is None:
|
|
233
|
-
return _warn_and_return_false("assertionMonitor")
|
|
234
|
-
if (
|
|
235
|
-
monitor.info.assertionMonitor.assertions is None
|
|
236
|
-
or len(monitor.info.assertionMonitor.assertions) == 0
|
|
237
|
-
):
|
|
238
|
-
return _warn_and_return_false("assertionMonitor.assertions")
|
|
239
|
-
|
|
240
|
-
return True
|
|
241
|
-
|
|
242
|
-
@staticmethod
|
|
243
|
-
def _get_assertion_parameters(
|
|
244
|
-
monitor: Monitor,
|
|
245
|
-
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
246
|
-
) -> Optional[models.AssertionEvaluationParametersClass]:
|
|
247
|
-
"""Get the assertion parameters from the monitor."""
|
|
248
|
-
# We know these are not None from _has_valid_monitor_info check
|
|
249
|
-
assert (
|
|
250
|
-
monitor is not None
|
|
251
|
-
and monitor.info is not None
|
|
252
|
-
and monitor.info.assertionMonitor is not None
|
|
253
|
-
)
|
|
254
|
-
assertion_monitor = monitor.info.assertionMonitor
|
|
255
|
-
assert (
|
|
256
|
-
assertion_monitor is not None and assertion_monitor.assertions is not None
|
|
257
|
-
)
|
|
258
|
-
assertions = assertion_monitor.assertions
|
|
259
|
-
|
|
260
|
-
if assertions[0].parameters is None:
|
|
261
|
-
logger.warning(
|
|
262
|
-
f"Monitor {monitor.urn} does not have a assertionMonitor.assertions[0].parameters, defaulting detection mechanism to {default}"
|
|
327
|
+
f"AssertionEvaluationParametersType {parameters.type} not supported"
|
|
263
328
|
)
|
|
264
|
-
return None
|
|
265
|
-
return assertions[0].parameters
|
|
266
329
|
|
|
267
330
|
@staticmethod
|
|
268
331
|
def _get_freshness_detection_mechanism(
|
|
@@ -283,7 +346,7 @@ class _HasSmartFunctionality:
|
|
|
283
346
|
elif source_type == models.DatasetFreshnessSourceTypeClass.AUDIT_LOG:
|
|
284
347
|
return DetectionMechanism.AUDIT_LOG
|
|
285
348
|
elif source_type == models.DatasetFreshnessSourceTypeClass.FIELD_VALUE:
|
|
286
|
-
return
|
|
349
|
+
return _AssertionPublic._get_field_value_detection_mechanism(
|
|
287
350
|
assertion, parameters
|
|
288
351
|
)
|
|
289
352
|
elif source_type == models.DatasetFreshnessSourceTypeClass.DATAHUB_OPERATION:
|
|
@@ -293,6 +356,33 @@ class _HasSmartFunctionality:
|
|
|
293
356
|
else:
|
|
294
357
|
raise SDKNotYetSupportedError(f"DatasetFreshnessSourceType {source_type}")
|
|
295
358
|
|
|
359
|
+
@staticmethod
|
|
360
|
+
def _get_volume_detection_mechanism(
|
|
361
|
+
assertion: Assertion,
|
|
362
|
+
parameters: models.AssertionEvaluationParametersClass,
|
|
363
|
+
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
364
|
+
) -> _DetectionMechanismTypes:
|
|
365
|
+
"""Get the detection mechanism for volume assertions."""
|
|
366
|
+
if parameters.datasetVolumeParameters is None:
|
|
367
|
+
logger.warning(
|
|
368
|
+
f"Monitor does not have datasetVolumeParameters, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
|
|
369
|
+
)
|
|
370
|
+
if default is None:
|
|
371
|
+
return DEFAULT_DETECTION_MECHANISM
|
|
372
|
+
else:
|
|
373
|
+
return default
|
|
374
|
+
|
|
375
|
+
source_type = parameters.datasetVolumeParameters.sourceType
|
|
376
|
+
if source_type == models.DatasetVolumeSourceTypeClass.INFORMATION_SCHEMA:
|
|
377
|
+
return DetectionMechanism.INFORMATION_SCHEMA
|
|
378
|
+
elif source_type == models.DatasetVolumeSourceTypeClass.QUERY:
|
|
379
|
+
additional_filter = _AssertionPublic._get_additional_filter(assertion)
|
|
380
|
+
return DetectionMechanism.QUERY(additional_filter=additional_filter)
|
|
381
|
+
elif source_type == models.DatasetVolumeSourceTypeClass.DATAHUB_DATASET_PROFILE:
|
|
382
|
+
return DetectionMechanism.DATASET_PROFILE
|
|
383
|
+
else:
|
|
384
|
+
raise SDKNotYetSupportedError(f"DatasetVolumeSourceType {source_type}")
|
|
385
|
+
|
|
296
386
|
@staticmethod
|
|
297
387
|
def _get_field_value_detection_mechanism(
|
|
298
388
|
assertion: Assertion,
|
|
@@ -310,7 +400,7 @@ class _HasSmartFunctionality:
|
|
|
310
400
|
return DEFAULT_DETECTION_MECHANISM
|
|
311
401
|
|
|
312
402
|
column_name = field.path
|
|
313
|
-
additional_filter =
|
|
403
|
+
additional_filter = _AssertionPublic._get_additional_filter(assertion)
|
|
314
404
|
|
|
315
405
|
if field.kind == models.FreshnessFieldKindClass.LAST_MODIFIED:
|
|
316
406
|
return DetectionMechanism.LAST_MODIFIED_COLUMN(
|
|
@@ -332,7 +422,10 @@ class _HasSmartFunctionality:
|
|
|
332
422
|
)
|
|
333
423
|
return None
|
|
334
424
|
if (
|
|
335
|
-
not isinstance(
|
|
425
|
+
not isinstance(
|
|
426
|
+
assertion.info,
|
|
427
|
+
_AssertionPublic._SUPPORTED_WITH_FILTER_ASSERTION_TYPES,
|
|
428
|
+
)
|
|
336
429
|
or assertion.info.filter is None
|
|
337
430
|
):
|
|
338
431
|
logger.warning(
|
|
@@ -345,95 +438,52 @@ class _HasSmartFunctionality:
|
|
|
345
438
|
)
|
|
346
439
|
return assertion.info.filter.sql
|
|
347
440
|
|
|
441
|
+
@staticmethod
|
|
442
|
+
def _has_valid_monitor_info(monitor: Monitor) -> bool:
|
|
443
|
+
"""Check if monitor has valid info and assertion monitor."""
|
|
348
444
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
445
|
+
def _warn_and_return_false(field_name: str) -> bool:
|
|
446
|
+
logger.warning(
|
|
447
|
+
f"Monitor {monitor.urn} does not have an `{field_name}` field, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
|
|
448
|
+
)
|
|
449
|
+
return False
|
|
353
450
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
created_at: Union[datetime, None] = None,
|
|
364
|
-
updated_by: Optional[CorpUserUrn] = None,
|
|
365
|
-
updated_at: Optional[datetime] = None,
|
|
366
|
-
):
|
|
367
|
-
"""
|
|
368
|
-
Initialize the public facing assertion class.
|
|
451
|
+
if monitor.info is None:
|
|
452
|
+
return _warn_and_return_false("info")
|
|
453
|
+
if monitor.info.assertionMonitor is None:
|
|
454
|
+
return _warn_and_return_false("assertionMonitor")
|
|
455
|
+
if (
|
|
456
|
+
monitor.info.assertionMonitor.assertions is None
|
|
457
|
+
or len(monitor.info.assertionMonitor.assertions) == 0
|
|
458
|
+
):
|
|
459
|
+
return _warn_and_return_false("assertionMonitor.assertions")
|
|
369
460
|
|
|
370
|
-
|
|
371
|
-
urn: The urn of the assertion.
|
|
372
|
-
dataset_urn: The urn of the dataset that the assertion is for.
|
|
373
|
-
display_name: The display name of the assertion.
|
|
374
|
-
mode: The mode of the assertion (active, inactive).
|
|
375
|
-
tags: The tags of the assertion.
|
|
376
|
-
created_by: The urn of the user that created the assertion.
|
|
377
|
-
created_at: The timestamp of when the assertion was created.
|
|
378
|
-
updated_by: The urn of the user that updated the assertion.
|
|
379
|
-
updated_at: The timestamp of when the assertion was updated.
|
|
380
|
-
"""
|
|
381
|
-
self._urn = urn
|
|
382
|
-
self._dataset_urn = dataset_urn
|
|
383
|
-
self._display_name = display_name
|
|
384
|
-
self._mode = mode
|
|
385
|
-
self._created_by = created_by
|
|
386
|
-
self._created_at = created_at
|
|
387
|
-
self._updated_by = updated_by
|
|
388
|
-
self._updated_at = updated_at
|
|
389
|
-
self._tags = tags
|
|
390
|
-
|
|
391
|
-
@property
|
|
392
|
-
def urn(self) -> AssertionUrn:
|
|
393
|
-
return self._urn
|
|
394
|
-
|
|
395
|
-
@property
|
|
396
|
-
def dataset_urn(self) -> DatasetUrn:
|
|
397
|
-
return self._dataset_urn
|
|
398
|
-
|
|
399
|
-
@property
|
|
400
|
-
def display_name(self) -> str:
|
|
401
|
-
return self._display_name
|
|
402
|
-
|
|
403
|
-
@property
|
|
404
|
-
def mode(self) -> AssertionMode:
|
|
405
|
-
return self._mode
|
|
406
|
-
|
|
407
|
-
@property
|
|
408
|
-
def created_by(self) -> Optional[CorpUserUrn]:
|
|
409
|
-
return self._created_by
|
|
410
|
-
|
|
411
|
-
@property
|
|
412
|
-
def created_at(self) -> Union[datetime, None]:
|
|
413
|
-
return self._created_at
|
|
414
|
-
|
|
415
|
-
@property
|
|
416
|
-
def updated_by(self) -> Optional[CorpUserUrn]:
|
|
417
|
-
return self._updated_by
|
|
418
|
-
|
|
419
|
-
@property
|
|
420
|
-
def updated_at(self) -> Union[datetime, None]:
|
|
421
|
-
return self._updated_at
|
|
422
|
-
|
|
423
|
-
@property
|
|
424
|
-
def tags(self) -> list[TagUrn]:
|
|
425
|
-
return self._tags
|
|
461
|
+
return True
|
|
426
462
|
|
|
427
463
|
@staticmethod
|
|
428
|
-
def
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
464
|
+
def _get_assertion_parameters(
|
|
465
|
+
monitor: Monitor,
|
|
466
|
+
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
467
|
+
) -> Optional[models.AssertionEvaluationParametersClass]:
|
|
468
|
+
"""Get the assertion parameters from the monitor."""
|
|
469
|
+
# We know these are not None from _has_valid_monitor_info check
|
|
470
|
+
assert (
|
|
471
|
+
monitor is not None
|
|
472
|
+
and monitor.info is not None
|
|
473
|
+
and monitor.info.assertionMonitor is not None
|
|
474
|
+
)
|
|
475
|
+
assertion_monitor = monitor.info.assertionMonitor
|
|
476
|
+
assert (
|
|
477
|
+
assertion_monitor is not None and assertion_monitor.assertions is not None
|
|
478
|
+
)
|
|
479
|
+
assertions = assertion_monitor.assertions
|
|
435
480
|
|
|
436
|
-
|
|
481
|
+
if assertions[0].parameters is None:
|
|
482
|
+
logger.warning(
|
|
483
|
+
f"Monitor {monitor.urn} does not have a assertionMonitor.assertions[0].parameters, defaulting detection mechanism to {default}"
|
|
484
|
+
)
|
|
485
|
+
return None
|
|
486
|
+
return assertions[0].parameters
|
|
437
487
|
|
|
438
488
|
@staticmethod
|
|
439
489
|
def _get_created_by(assertion: Assertion) -> Optional[CorpUserUrn]:
|
|
@@ -500,19 +550,22 @@ class _AssertionPublic(ABC):
|
|
|
500
550
|
return AssertionMode.INACTIVE
|
|
501
551
|
return AssertionMode(monitor.info.status.mode)
|
|
502
552
|
|
|
553
|
+
@classmethod
|
|
503
554
|
@abstractmethod
|
|
504
|
-
def
|
|
555
|
+
def _from_entities(
|
|
505
556
|
cls,
|
|
506
557
|
assertion: Assertion,
|
|
507
558
|
monitor: Monitor,
|
|
508
559
|
) -> Self:
|
|
509
560
|
"""
|
|
510
561
|
Create an assertion from the assertion and monitor entities.
|
|
562
|
+
|
|
563
|
+
Note: This is a private method since it is intended to be called internally by the client.
|
|
511
564
|
"""
|
|
512
565
|
pass
|
|
513
566
|
|
|
514
567
|
|
|
515
|
-
class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
568
|
+
class SmartFreshnessAssertion(_HasSchedule, _HasSmartFunctionality, _AssertionPublic):
|
|
516
569
|
"""
|
|
517
570
|
A class that represents a smart freshness assertion.
|
|
518
571
|
"""
|
|
@@ -524,6 +577,7 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
|
524
577
|
dataset_urn: DatasetUrn,
|
|
525
578
|
display_name: str,
|
|
526
579
|
mode: AssertionMode,
|
|
580
|
+
schedule: models.CronScheduleClass = DEFAULT_SCHEDULE,
|
|
527
581
|
sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
|
|
528
582
|
exclusion_windows: list[ExclusionWindowTypes],
|
|
529
583
|
training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
|
|
@@ -547,6 +601,7 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
|
547
601
|
dataset_urn: The urn of the dataset that the assertion is for.
|
|
548
602
|
display_name: The display name of the assertion.
|
|
549
603
|
mode: The mode of the assertion (active, inactive).
|
|
604
|
+
schedule: The schedule of the assertion.
|
|
550
605
|
sensitivity: The sensitivity of the assertion (low, medium, high).
|
|
551
606
|
exclusion_windows: The exclusion windows of the assertion.
|
|
552
607
|
training_data_lookback_days: The max number of days of data to use for training the assertion.
|
|
@@ -558,14 +613,111 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
|
558
613
|
updated_by: The urn of the user that updated the assertion.
|
|
559
614
|
updated_at: The timestamp of when the assertion was updated.
|
|
560
615
|
"""
|
|
561
|
-
# Initialize the
|
|
616
|
+
# Initialize the mixins first
|
|
617
|
+
_HasSchedule.__init__(self, schedule=schedule)
|
|
562
618
|
_HasSmartFunctionality.__init__(
|
|
563
619
|
self,
|
|
564
620
|
sensitivity=sensitivity,
|
|
565
621
|
exclusion_windows=exclusion_windows,
|
|
566
622
|
training_data_lookback_days=training_data_lookback_days,
|
|
623
|
+
)
|
|
624
|
+
# Then initialize the parent class
|
|
625
|
+
_AssertionPublic.__init__(
|
|
626
|
+
self,
|
|
627
|
+
urn=urn,
|
|
628
|
+
dataset_urn=dataset_urn,
|
|
629
|
+
display_name=display_name,
|
|
630
|
+
mode=mode,
|
|
567
631
|
incident_behavior=incident_behavior,
|
|
568
632
|
detection_mechanism=detection_mechanism,
|
|
633
|
+
created_by=created_by,
|
|
634
|
+
created_at=created_at,
|
|
635
|
+
updated_by=updated_by,
|
|
636
|
+
updated_at=updated_at,
|
|
637
|
+
tags=tags,
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
@classmethod
|
|
641
|
+
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
642
|
+
"""
|
|
643
|
+
Create a smart freshness assertion from the assertion and monitor entities.
|
|
644
|
+
|
|
645
|
+
Note: This is a private method since it is intended to be called internally by the client.
|
|
646
|
+
"""
|
|
647
|
+
return cls(
|
|
648
|
+
urn=assertion.urn,
|
|
649
|
+
dataset_urn=assertion.dataset,
|
|
650
|
+
display_name=assertion.description or "",
|
|
651
|
+
mode=cls._get_mode(monitor),
|
|
652
|
+
schedule=cls._get_schedule(monitor),
|
|
653
|
+
sensitivity=cls._get_sensitivity(monitor),
|
|
654
|
+
exclusion_windows=cls._get_exclusion_windows(monitor),
|
|
655
|
+
training_data_lookback_days=cls._get_training_data_lookback_days(monitor),
|
|
656
|
+
incident_behavior=cls._get_incident_behavior(assertion),
|
|
657
|
+
detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
|
|
658
|
+
created_by=cls._get_created_by(assertion),
|
|
659
|
+
created_at=cls._get_created_at(assertion),
|
|
660
|
+
updated_by=cls._get_updated_by(assertion),
|
|
661
|
+
updated_at=cls._get_updated_at(assertion),
|
|
662
|
+
tags=cls._get_tags(assertion),
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
class SmartVolumeAssertion(_HasSchedule, _HasSmartFunctionality, _AssertionPublic):
|
|
667
|
+
"""
|
|
668
|
+
A class that represents a smart volume assertion.
|
|
669
|
+
"""
|
|
670
|
+
|
|
671
|
+
def __init__(
|
|
672
|
+
self,
|
|
673
|
+
*,
|
|
674
|
+
urn: AssertionUrn,
|
|
675
|
+
dataset_urn: DatasetUrn,
|
|
676
|
+
display_name: str,
|
|
677
|
+
mode: AssertionMode,
|
|
678
|
+
schedule: models.CronScheduleClass,
|
|
679
|
+
sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
|
|
680
|
+
exclusion_windows: list[ExclusionWindowTypes],
|
|
681
|
+
training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
|
|
682
|
+
incident_behavior: list[AssertionIncidentBehavior],
|
|
683
|
+
detection_mechanism: Optional[
|
|
684
|
+
_DetectionMechanismTypes
|
|
685
|
+
] = DEFAULT_DETECTION_MECHANISM,
|
|
686
|
+
tags: list[TagUrn],
|
|
687
|
+
created_by: Optional[CorpUserUrn] = None,
|
|
688
|
+
created_at: Union[datetime, None] = None,
|
|
689
|
+
updated_by: Optional[CorpUserUrn] = None,
|
|
690
|
+
updated_at: Optional[datetime] = None,
|
|
691
|
+
):
|
|
692
|
+
"""
|
|
693
|
+
Initialize a smart volume assertion.
|
|
694
|
+
|
|
695
|
+
Note: Values can be accessed, but not set on the assertion object.
|
|
696
|
+
To update an assertion, use the `upsert_*` method.
|
|
697
|
+
Args:
|
|
698
|
+
urn: The urn of the assertion.
|
|
699
|
+
dataset_urn: The urn of the dataset that the assertion is for.
|
|
700
|
+
display_name: The display name of the assertion.
|
|
701
|
+
mode: The mode of the assertion (active, inactive).
|
|
702
|
+
schedule: The schedule of the assertion.
|
|
703
|
+
sensitivity: The sensitivity of the assertion (low, medium, high).
|
|
704
|
+
exclusion_windows: The exclusion windows of the assertion.
|
|
705
|
+
training_data_lookback_days: The max number of days of data to use for training the assertion.
|
|
706
|
+
incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
|
|
707
|
+
detection_mechanism: The detection mechanism of the assertion.
|
|
708
|
+
tags: The tags applied to the assertion.
|
|
709
|
+
created_by: The urn of the user that created the assertion.
|
|
710
|
+
created_at: The timestamp of when the assertion was created.
|
|
711
|
+
updated_by: The urn of the user that updated the assertion.
|
|
712
|
+
updated_at: The timestamp of when the assertion was updated.
|
|
713
|
+
"""
|
|
714
|
+
# Initialize the mixins first
|
|
715
|
+
_HasSchedule.__init__(self, schedule=schedule)
|
|
716
|
+
_HasSmartFunctionality.__init__(
|
|
717
|
+
self,
|
|
718
|
+
sensitivity=sensitivity,
|
|
719
|
+
exclusion_windows=exclusion_windows,
|
|
720
|
+
training_data_lookback_days=training_data_lookback_days,
|
|
569
721
|
)
|
|
570
722
|
# Then initialize the parent class
|
|
571
723
|
_AssertionPublic.__init__(
|
|
@@ -574,6 +726,8 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
|
574
726
|
dataset_urn=dataset_urn,
|
|
575
727
|
display_name=display_name,
|
|
576
728
|
mode=mode,
|
|
729
|
+
incident_behavior=incident_behavior,
|
|
730
|
+
detection_mechanism=detection_mechanism,
|
|
577
731
|
created_by=created_by,
|
|
578
732
|
created_at=created_at,
|
|
579
733
|
updated_by=updated_by,
|
|
@@ -582,15 +736,18 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
|
582
736
|
)
|
|
583
737
|
|
|
584
738
|
@classmethod
|
|
585
|
-
def
|
|
739
|
+
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
586
740
|
"""
|
|
587
741
|
Create a smart freshness assertion from the assertion and monitor entities.
|
|
742
|
+
|
|
743
|
+
Note: This is a private method since it is intended to be called internally by the client.
|
|
588
744
|
"""
|
|
589
745
|
return cls(
|
|
590
746
|
urn=assertion.urn,
|
|
591
747
|
dataset_urn=assertion.dataset,
|
|
592
748
|
display_name=assertion.description or "",
|
|
593
749
|
mode=cls._get_mode(monitor),
|
|
750
|
+
schedule=cls._get_schedule(monitor),
|
|
594
751
|
sensitivity=cls._get_sensitivity(monitor),
|
|
595
752
|
exclusion_windows=cls._get_exclusion_windows(monitor),
|
|
596
753
|
training_data_lookback_days=cls._get_training_data_lookback_days(monitor),
|
|
@@ -604,7 +761,158 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
|
604
761
|
)
|
|
605
762
|
|
|
606
763
|
|
|
764
|
+
class FreshnessAssertion(_HasSchedule, _AssertionPublic):
|
|
765
|
+
"""
|
|
766
|
+
A class that represents a freshness assertion.
|
|
767
|
+
"""
|
|
768
|
+
|
|
769
|
+
def __init__(
|
|
770
|
+
self,
|
|
771
|
+
*,
|
|
772
|
+
urn: AssertionUrn,
|
|
773
|
+
dataset_urn: DatasetUrn,
|
|
774
|
+
display_name: str,
|
|
775
|
+
mode: AssertionMode,
|
|
776
|
+
schedule: models.CronScheduleClass,
|
|
777
|
+
freshness_schedule_check_type: Union[
|
|
778
|
+
str, models.FreshnessAssertionScheduleTypeClass
|
|
779
|
+
],
|
|
780
|
+
lookback_window: Optional[TimeWindowSizeInputTypes],
|
|
781
|
+
tags: list[TagUrn],
|
|
782
|
+
incident_behavior: list[AssertionIncidentBehavior],
|
|
783
|
+
detection_mechanism: Optional[
|
|
784
|
+
_DetectionMechanismTypes
|
|
785
|
+
] = DEFAULT_DETECTION_MECHANISM,
|
|
786
|
+
created_by: Optional[CorpUserUrn] = None,
|
|
787
|
+
created_at: Union[datetime, None] = None,
|
|
788
|
+
updated_by: Optional[CorpUserUrn] = None,
|
|
789
|
+
updated_at: Optional[datetime] = None,
|
|
790
|
+
):
|
|
791
|
+
"""
|
|
792
|
+
Initialize a freshness assertion.
|
|
793
|
+
|
|
794
|
+
Note: Values can be accessed, but not set on the assertion object.
|
|
795
|
+
To update an assertion, use the `upsert_*` method.
|
|
796
|
+
Args:
|
|
797
|
+
urn: The urn of the assertion.
|
|
798
|
+
dataset_urn: The urn of the dataset that the assertion is for.
|
|
799
|
+
display_name: The display name of the assertion.
|
|
800
|
+
mode: The mode of the assertion (active, inactive).
|
|
801
|
+
schedule: The schedule of the assertion.
|
|
802
|
+
freshness_schedule_check_type: The type of freshness schedule check to be used for the assertion.
|
|
803
|
+
lookback_window: The lookback window to be used for the assertion.
|
|
804
|
+
tags: The tags applied to the assertion.
|
|
805
|
+
incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
|
|
806
|
+
detection_mechanism: The detection mechanism of the assertion.
|
|
807
|
+
created_by: The urn of the user that created the assertion.
|
|
808
|
+
created_at: The timestamp of when the assertion was created.
|
|
809
|
+
updated_by: The urn of the user that updated the assertion.
|
|
810
|
+
updated_at: The timestamp of when the assertion was updated.
|
|
811
|
+
"""
|
|
812
|
+
_HasSchedule.__init__(self, schedule=schedule)
|
|
813
|
+
_AssertionPublic.__init__(
|
|
814
|
+
self,
|
|
815
|
+
urn=urn,
|
|
816
|
+
dataset_urn=dataset_urn,
|
|
817
|
+
display_name=display_name,
|
|
818
|
+
mode=mode,
|
|
819
|
+
incident_behavior=incident_behavior,
|
|
820
|
+
detection_mechanism=detection_mechanism,
|
|
821
|
+
created_by=created_by,
|
|
822
|
+
created_at=created_at,
|
|
823
|
+
updated_by=updated_by,
|
|
824
|
+
updated_at=updated_at,
|
|
825
|
+
tags=tags,
|
|
826
|
+
)
|
|
827
|
+
self._freshness_schedule_check_type = freshness_schedule_check_type
|
|
828
|
+
self._lookback_window = lookback_window
|
|
829
|
+
|
|
830
|
+
@property
|
|
831
|
+
def freshness_schedule_check_type(
|
|
832
|
+
self,
|
|
833
|
+
) -> Union[str, models.FreshnessAssertionScheduleTypeClass]:
|
|
834
|
+
return self._freshness_schedule_check_type
|
|
835
|
+
|
|
836
|
+
@property
|
|
837
|
+
def lookback_window(self) -> Optional[TimeWindowSizeInputTypes]:
|
|
838
|
+
return self._lookback_window
|
|
839
|
+
|
|
840
|
+
@staticmethod
|
|
841
|
+
def _get_freshness_schedule_check_type(
|
|
842
|
+
assertion: Assertion,
|
|
843
|
+
) -> Union[str, models.FreshnessAssertionScheduleTypeClass]:
|
|
844
|
+
if assertion.info is None:
|
|
845
|
+
raise SDKNotYetSupportedError(
|
|
846
|
+
f"Assertion {assertion.urn} does not have a freshness assertion info, which is not supported"
|
|
847
|
+
)
|
|
848
|
+
if isinstance(assertion.info, models.FreshnessAssertionInfoClass):
|
|
849
|
+
if assertion.info.schedule is None:
|
|
850
|
+
raise SDKNotYetSupportedError(
|
|
851
|
+
f"Traditional freshness assertion {assertion.urn} does not have a schedule, which is not supported"
|
|
852
|
+
)
|
|
853
|
+
return assertion.info.schedule.type
|
|
854
|
+
else:
|
|
855
|
+
raise SDKNotYetSupportedError(
|
|
856
|
+
f"Assertion {assertion.urn} is not a freshness assertion"
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
@staticmethod
|
|
860
|
+
def _get_lookback_window(
|
|
861
|
+
assertion: Assertion,
|
|
862
|
+
) -> Optional[models.FixedIntervalScheduleClass]:
|
|
863
|
+
if assertion.info is None:
|
|
864
|
+
raise SDKNotYetSupportedError(
|
|
865
|
+
f"Assertion {assertion.urn} does not have a freshness assertion info, which is not supported"
|
|
866
|
+
)
|
|
867
|
+
if isinstance(assertion.info, models.FreshnessAssertionInfoClass):
|
|
868
|
+
if assertion.info.schedule is None:
|
|
869
|
+
raise SDKNotYetSupportedError(
|
|
870
|
+
f"Traditional freshness assertion {assertion.urn} does not have a schedule, which is not supported"
|
|
871
|
+
)
|
|
872
|
+
return assertion.info.schedule.fixedInterval
|
|
873
|
+
else:
|
|
874
|
+
raise SDKNotYetSupportedError(
|
|
875
|
+
f"Assertion {assertion.urn} is not a freshness assertion"
|
|
876
|
+
)
|
|
877
|
+
|
|
878
|
+
@classmethod
|
|
879
|
+
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
880
|
+
"""
|
|
881
|
+
Create a freshness assertion from the assertion and monitor entities.
|
|
882
|
+
"""
|
|
883
|
+
return cls(
|
|
884
|
+
urn=assertion.urn,
|
|
885
|
+
dataset_urn=assertion.dataset,
|
|
886
|
+
display_name=assertion.description or "",
|
|
887
|
+
mode=cls._get_mode(monitor),
|
|
888
|
+
schedule=cls._get_schedule(monitor),
|
|
889
|
+
freshness_schedule_check_type=cls._get_freshness_schedule_check_type(
|
|
890
|
+
assertion
|
|
891
|
+
),
|
|
892
|
+
lookback_window=cls._get_lookback_window(assertion),
|
|
893
|
+
incident_behavior=cls._get_incident_behavior(assertion),
|
|
894
|
+
detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
|
|
895
|
+
created_by=cls._get_created_by(assertion),
|
|
896
|
+
created_at=cls._get_created_at(assertion),
|
|
897
|
+
updated_by=cls._get_updated_by(assertion),
|
|
898
|
+
updated_at=cls._get_updated_at(assertion),
|
|
899
|
+
tags=cls._get_tags(assertion),
|
|
900
|
+
)
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
class SmartColumnMetricAssertion(_HasSmartFunctionality, _AssertionPublic):
|
|
904
|
+
"""
|
|
905
|
+
A class that represents a smart column metric assertion.
|
|
906
|
+
"""
|
|
907
|
+
|
|
908
|
+
def __init__(self) -> None:
|
|
909
|
+
raise NotImplementedError("SmartColumnMetricAssertion is not implemented yet")
|
|
910
|
+
|
|
911
|
+
|
|
607
912
|
AssertionTypes = Union[
|
|
608
913
|
SmartFreshnessAssertion,
|
|
914
|
+
SmartVolumeAssertion,
|
|
915
|
+
FreshnessAssertion,
|
|
916
|
+
SmartColumnMetricAssertion,
|
|
609
917
|
# TODO: Add other assertion types here as we add them.
|
|
610
918
|
]
|