acryl-datahub-cloud 0.3.12rc1__py3-none-any.whl → 0.3.12rc4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub-cloud might be problematic. Click here for more details.

Files changed (74) hide show
  1. acryl_datahub_cloud/_codegen_config.json +1 -1
  2. acryl_datahub_cloud/datahub_forms_notifications/forms_notifications_source.py +559 -0
  3. acryl_datahub_cloud/datahub_forms_notifications/get_search_results_total.gql +14 -0
  4. acryl_datahub_cloud/datahub_forms_notifications/query.py +17 -0
  5. acryl_datahub_cloud/datahub_forms_notifications/scroll_forms_for_notification.gql +29 -0
  6. acryl_datahub_cloud/datahub_forms_notifications/send_form_notification_request.gql +5 -0
  7. acryl_datahub_cloud/datahub_usage_reporting/query_builder.py +48 -8
  8. acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py +49 -40
  9. acryl_datahub_cloud/metadata/_urns/urn_defs.py +1842 -1786
  10. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/application/__init__.py +19 -0
  11. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/form/__init__.py +4 -0
  12. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/notification/__init__.py +19 -0
  13. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +2 -0
  14. acryl_datahub_cloud/metadata/schema.avsc +24861 -24050
  15. acryl_datahub_cloud/metadata/schema_classes.py +1031 -631
  16. acryl_datahub_cloud/metadata/schemas/ApplicationKey.avsc +31 -0
  17. acryl_datahub_cloud/metadata/schemas/ApplicationProperties.avsc +72 -0
  18. acryl_datahub_cloud/metadata/schemas/Applications.avsc +38 -0
  19. acryl_datahub_cloud/metadata/schemas/AssertionAnalyticsRunEvent.avsc +40 -7
  20. acryl_datahub_cloud/metadata/schemas/AssertionInfo.avsc +27 -6
  21. acryl_datahub_cloud/metadata/schemas/AssertionRunEvent.avsc +31 -7
  22. acryl_datahub_cloud/metadata/schemas/AssertionsSummary.avsc +14 -0
  23. acryl_datahub_cloud/metadata/schemas/ChartKey.avsc +1 -0
  24. acryl_datahub_cloud/metadata/schemas/ConstraintInfo.avsc +12 -1
  25. acryl_datahub_cloud/metadata/schemas/ContainerKey.avsc +1 -0
  26. acryl_datahub_cloud/metadata/schemas/CorpGroupKey.avsc +2 -1
  27. acryl_datahub_cloud/metadata/schemas/CorpUserKey.avsc +2 -1
  28. acryl_datahub_cloud/metadata/schemas/DashboardKey.avsc +1 -0
  29. acryl_datahub_cloud/metadata/schemas/DataFlowKey.avsc +1 -0
  30. acryl_datahub_cloud/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
  31. acryl_datahub_cloud/metadata/schemas/DataJobKey.avsc +1 -0
  32. acryl_datahub_cloud/metadata/schemas/DataProductKey.avsc +1 -0
  33. acryl_datahub_cloud/metadata/schemas/DataProductProperties.avsc +1 -1
  34. acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc +1 -0
  35. acryl_datahub_cloud/metadata/schemas/FormAssignmentStatus.avsc +36 -0
  36. acryl_datahub_cloud/metadata/schemas/FormInfo.avsc +6 -0
  37. acryl_datahub_cloud/metadata/schemas/FormKey.avsc +2 -1
  38. acryl_datahub_cloud/metadata/schemas/FormNotifications.avsc +69 -0
  39. acryl_datahub_cloud/metadata/schemas/FormSettings.avsc +3 -0
  40. acryl_datahub_cloud/metadata/schemas/GlobalSettingsInfo.avsc +22 -0
  41. acryl_datahub_cloud/metadata/schemas/GlossaryTermKey.avsc +1 -0
  42. acryl_datahub_cloud/metadata/schemas/MLFeatureKey.avsc +1 -0
  43. acryl_datahub_cloud/metadata/schemas/MLFeatureTableKey.avsc +1 -0
  44. acryl_datahub_cloud/metadata/schemas/MLModelGroupKey.avsc +1 -0
  45. acryl_datahub_cloud/metadata/schemas/MLModelKey.avsc +1 -0
  46. acryl_datahub_cloud/metadata/schemas/MLPrimaryKeyKey.avsc +1 -0
  47. acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc +12 -1
  48. acryl_datahub_cloud/metadata/schemas/MonitorInfo.avsc +27 -6
  49. acryl_datahub_cloud/metadata/schemas/NotebookKey.avsc +1 -0
  50. acryl_datahub_cloud/metadata/schemas/NotificationRequest.avsc +1 -0
  51. acryl_datahub_cloud/notifications/__init__.py +0 -0
  52. acryl_datahub_cloud/notifications/notification_recipient_builder.py +399 -0
  53. acryl_datahub_cloud/sdk/__init__.py +29 -0
  54. acryl_datahub_cloud/{_sdk_extras → sdk}/assertion.py +501 -193
  55. acryl_datahub_cloud/sdk/assertion_input/__init__.py +0 -0
  56. acryl_datahub_cloud/{_sdk_extras → sdk/assertion_input}/assertion_input.py +733 -189
  57. acryl_datahub_cloud/sdk/assertion_input/freshness_assertion_input.py +261 -0
  58. acryl_datahub_cloud/sdk/assertion_input/smart_column_metric_assertion_input.py +947 -0
  59. acryl_datahub_cloud/sdk/assertions_client.py +1639 -0
  60. acryl_datahub_cloud/sdk/entities/__init__.py +0 -0
  61. acryl_datahub_cloud/{_sdk_extras → sdk}/entities/assertion.py +5 -2
  62. acryl_datahub_cloud/{_sdk_extras → sdk}/subscription_client.py +146 -33
  63. {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/METADATA +48 -43
  64. {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/RECORD +72 -54
  65. {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/entry_points.txt +1 -0
  66. acryl_datahub_cloud/_sdk_extras/__init__.py +0 -19
  67. acryl_datahub_cloud/_sdk_extras/assertions_client.py +0 -717
  68. /acryl_datahub_cloud/{_sdk_extras/entities → datahub_forms_notifications}/__init__.py +0 -0
  69. /acryl_datahub_cloud/{_sdk_extras → sdk}/entities/monitor.py +0 -0
  70. /acryl_datahub_cloud/{_sdk_extras → sdk}/entities/subscription.py +0 -0
  71. /acryl_datahub_cloud/{_sdk_extras → sdk}/errors.py +0 -0
  72. /acryl_datahub_cloud/{_sdk_extras → sdk}/resolver_client.py +0 -0
  73. {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/WHEEL +0 -0
  74. {acryl_datahub_cloud-0.3.12rc1.dist-info → acryl_datahub_cloud-0.3.12rc4.dist-info}/top_level.txt +0 -0
@@ -10,28 +10,32 @@ import logging
10
10
  from abc import ABC, abstractmethod
11
11
  from datetime import datetime
12
12
  from enum import Enum
13
- from typing import Any, Optional, Union
13
+ from typing import Optional, Union
14
14
 
15
15
  from typing_extensions import Self
16
16
 
17
- from acryl_datahub_cloud._sdk_extras.assertion_input import (
17
+ from acryl_datahub_cloud.sdk.assertion_input.assertion_input import (
18
18
  ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
19
19
  DEFAULT_DETECTION_MECHANISM,
20
+ DEFAULT_SCHEDULE,
20
21
  DEFAULT_SENSITIVITY,
21
22
  AssertionIncidentBehavior,
22
23
  DetectionMechanism,
23
24
  ExclusionWindowTypes,
24
25
  FixedRangeExclusionWindow,
25
26
  InferenceSensitivity,
27
+ TimeWindowSizeInputTypes,
26
28
  _DetectionMechanismTypes,
27
29
  )
28
- from acryl_datahub_cloud._sdk_extras.entities.assertion import Assertion
29
- from acryl_datahub_cloud._sdk_extras.entities.monitor import Monitor
30
- from acryl_datahub_cloud._sdk_extras.errors import SDKNotYetSupportedError
30
+ from acryl_datahub_cloud.sdk.entities.assertion import Assertion
31
+ from acryl_datahub_cloud.sdk.entities.monitor import (
32
+ Monitor,
33
+ _get_nested_field_for_entity_with_default,
34
+ )
35
+ from acryl_datahub_cloud.sdk.errors import SDKNotYetSupportedError
31
36
  from datahub.emitter.mce_builder import parse_ts_millis
32
37
  from datahub.metadata import schema_classes as models
33
38
  from datahub.metadata.urns import AssertionUrn, CorpUserUrn, DatasetUrn, TagUrn
34
- from datahub.sdk.entity import Entity
35
39
 
36
40
  logger = logging.getLogger(__name__)
37
41
 
@@ -48,34 +52,33 @@ class AssertionMode(Enum):
48
52
  # PASSIVE = "PASSIVE" # Not supported in the user facing interface.
49
53
 
50
54
 
51
- def _get_nested_field_for_entity_with_default(
52
- entity: Entity,
53
- field_path: str,
54
- default: Any = None,
55
- ) -> Any:
55
+ class _HasSchedule:
56
56
  """
57
- Get a nested field from an Entity object, and warn and return default if not found.
58
-
59
- Args:
60
- entity: The entity to get the nested field from.
61
- field_path: The path to the nested field.
62
- default: The default value to return if the field is not found.
57
+ Mixin class that provides schedule functionality for assertions.
63
58
  """
64
- fields = field_path.split(".")
65
- current = entity
66
- last_valid_path = entity.entity_type_name()
67
-
68
- for field in fields:
69
- try:
70
- current = getattr(current, field)
71
- last_valid_path = f"{last_valid_path}.{field}"
72
- except AttributeError:
73
- logger.warning(
74
- f"{entity.entity_type_name().capitalize()} {entity.urn} does not have an `{last_valid_path}` field, defaulting to {default}"
75
- )
76
- return default
77
59
 
78
- return current
60
+ def __init__(self, schedule: models.CronScheduleClass) -> None:
61
+ self._schedule = schedule
62
+
63
+ @property
64
+ def schedule(self) -> models.CronScheduleClass:
65
+ return self._schedule
66
+
67
+ @staticmethod
68
+ def _get_schedule(monitor: Monitor) -> models.CronScheduleClass:
69
+ """Get the schedule from the monitor."""
70
+ assertion_evaluation_specs = _get_nested_field_for_entity_with_default(
71
+ monitor,
72
+ "info.assertionMonitor.assertions",
73
+ [],
74
+ )
75
+ if len(assertion_evaluation_specs) == 0:
76
+ return DEFAULT_SCHEDULE
77
+ assertion_evaluation_spec = assertion_evaluation_specs[0]
78
+ schedule = assertion_evaluation_spec.schedule
79
+ if schedule is None:
80
+ return DEFAULT_SCHEDULE
81
+ return schedule
79
82
 
80
83
 
81
84
  class _HasSmartFunctionality:
@@ -89,10 +92,6 @@ class _HasSmartFunctionality:
89
92
  sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
90
93
  exclusion_windows: list[ExclusionWindowTypes],
91
94
  training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
92
- incident_behavior: list[AssertionIncidentBehavior],
93
- detection_mechanism: Optional[
94
- _DetectionMechanismTypes
95
- ] = DEFAULT_DETECTION_MECHANISM,
96
95
  ) -> None:
97
96
  """
98
97
  Initialize the smart functionality mixin.
@@ -108,8 +107,6 @@ class _HasSmartFunctionality:
108
107
  self._sensitivity = sensitivity
109
108
  self._exclusion_windows = exclusion_windows
110
109
  self._training_data_lookback_days = training_data_lookback_days
111
- self._incident_behavior = incident_behavior
112
- self._detection_mechanism = detection_mechanism
113
110
 
114
111
  @property
115
112
  def sensitivity(self) -> InferenceSensitivity:
@@ -123,14 +120,6 @@ class _HasSmartFunctionality:
123
120
  def training_data_lookback_days(self) -> int:
124
121
  return self._training_data_lookback_days
125
122
 
126
- @property
127
- def incident_behavior(self) -> list[AssertionIncidentBehavior]:
128
- return self._incident_behavior
129
-
130
- @property
131
- def detection_mechanism(self) -> Optional[_DetectionMechanismTypes]:
132
- return self._detection_mechanism
133
-
134
123
  @staticmethod
135
124
  def _get_sensitivity(monitor: Monitor) -> InferenceSensitivity:
136
125
  # 1. Check if the monitor has a sensitivity field
@@ -179,6 +168,115 @@ class _HasSmartFunctionality:
179
168
  assert isinstance(retrieved, int)
180
169
  return retrieved
181
170
 
171
+
172
+ class _AssertionPublic(ABC):
173
+ """
174
+ Abstract base class that represents a public facing assertion and contains the common properties of all assertions.
175
+ """
176
+
177
+ _SUPPORTED_WITH_FILTER_ASSERTION_TYPES = (
178
+ models.FreshnessAssertionInfoClass,
179
+ models.VolumeAssertionInfoClass,
180
+ )
181
+
182
+ def __init__(
183
+ self,
184
+ *,
185
+ urn: AssertionUrn,
186
+ dataset_urn: DatasetUrn,
187
+ display_name: str,
188
+ mode: AssertionMode,
189
+ tags: list[TagUrn],
190
+ incident_behavior: list[AssertionIncidentBehavior],
191
+ detection_mechanism: Optional[
192
+ _DetectionMechanismTypes
193
+ ] = DEFAULT_DETECTION_MECHANISM,
194
+ created_by: Optional[CorpUserUrn] = None,
195
+ created_at: Union[datetime, None] = None,
196
+ updated_by: Optional[CorpUserUrn] = None,
197
+ updated_at: Optional[datetime] = None,
198
+ ):
199
+ """
200
+ Initialize the public facing assertion class.
201
+
202
+ Args:
203
+ urn: The urn of the assertion.
204
+ dataset_urn: The urn of the dataset that the assertion is for.
205
+ display_name: The display name of the assertion.
206
+ mode: The mode of the assertion (active, inactive).
207
+ tags: The tags of the assertion.
208
+ created_by: The urn of the user that created the assertion.
209
+ created_at: The timestamp of when the assertion was created.
210
+ updated_by: The urn of the user that updated the assertion.
211
+ updated_at: The timestamp of when the assertion was updated.
212
+ """
213
+ self._urn = urn
214
+ self._dataset_urn = dataset_urn
215
+ self._display_name = display_name
216
+ self._mode = mode
217
+ self._incident_behavior = incident_behavior
218
+ self._detection_mechanism = detection_mechanism
219
+ self._created_by = created_by
220
+ self._created_at = created_at
221
+ self._updated_by = updated_by
222
+ self._updated_at = updated_at
223
+ self._tags = tags
224
+
225
+ @property
226
+ def urn(self) -> AssertionUrn:
227
+ return self._urn
228
+
229
+ @property
230
+ def dataset_urn(self) -> DatasetUrn:
231
+ return self._dataset_urn
232
+
233
+ @property
234
+ def display_name(self) -> str:
235
+ return self._display_name
236
+
237
+ @property
238
+ def mode(self) -> AssertionMode:
239
+ return self._mode
240
+
241
+ @property
242
+ def incident_behavior(self) -> list[AssertionIncidentBehavior]:
243
+ return self._incident_behavior
244
+
245
+ @property
246
+ def detection_mechanism(self) -> Optional[_DetectionMechanismTypes]:
247
+ return self._detection_mechanism
248
+
249
+ @property
250
+ def created_by(self) -> Optional[CorpUserUrn]:
251
+ return self._created_by
252
+
253
+ @property
254
+ def created_at(self) -> Union[datetime, None]:
255
+ return self._created_at
256
+
257
+ @property
258
+ def updated_by(self) -> Optional[CorpUserUrn]:
259
+ return self._updated_by
260
+
261
+ @property
262
+ def updated_at(self) -> Union[datetime, None]:
263
+ return self._updated_at
264
+
265
+ @property
266
+ def tags(self) -> list[TagUrn]:
267
+ return self._tags
268
+
269
+ @staticmethod
270
+ def _get_incident_behavior(assertion: Assertion) -> list[AssertionIncidentBehavior]:
271
+ incident_behaviors = []
272
+ for action in assertion.on_failure + assertion.on_success:
273
+ if action.type == models.AssertionActionTypeClass.RAISE_INCIDENT:
274
+ incident_behaviors.append(AssertionIncidentBehavior.RAISE_ON_FAIL)
275
+ elif action.type == models.AssertionActionTypeClass.RESOLVE_INCIDENT:
276
+ incident_behaviors.append(AssertionIncidentBehavior.RESOLVE_ON_PASS)
277
+
278
+ return incident_behaviors
279
+
182
280
  @staticmethod
183
281
  def _get_detection_mechanism(
184
282
  assertion: Assertion,
@@ -186,7 +284,7 @@ class _HasSmartFunctionality:
186
284
  default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
187
285
  ) -> Optional[_DetectionMechanismTypes]:
188
286
  """Get the detection mechanism from the monitor and assertion."""
189
- if not _HasSmartFunctionality._has_valid_monitor_info(monitor):
287
+ if not _AssertionPublic._has_valid_monitor_info(monitor):
190
288
  return default
191
289
 
192
290
  # 1. Check if the assertion has a parameters field
@@ -199,70 +297,35 @@ class _HasSmartFunctionality:
199
297
  )
200
298
  return default
201
299
 
202
- parameters = _HasSmartFunctionality._get_assertion_parameters(monitor, default)
300
+ parameters = _AssertionPublic._get_assertion_parameters(monitor, default)
203
301
  if parameters is None:
204
302
  return _warn_and_return_default_detection_mechanism("parameters", default)
205
303
 
206
304
  # 2. Convert the raw detection mechanism to the SDK detection mechanism
207
- if (
208
- parameters.type
209
- == models.AssertionEvaluationParametersTypeClass.DATASET_FRESHNESS
210
- ):
211
- # TODO: Add support for other detection mechanisms when other assertion types are supported
212
- return _HasSmartFunctionality._get_freshness_detection_mechanism(
213
- assertion, parameters, default
214
- )
305
+ if parameters.type in [
306
+ models.AssertionEvaluationParametersTypeClass.DATASET_FRESHNESS,
307
+ models.AssertionEvaluationParametersTypeClass.DATASET_VOLUME,
308
+ ]:
309
+ if assertion.info is None:
310
+ return _warn_and_return_default_detection_mechanism("info", default)
311
+ if isinstance(assertion.info, models.VolumeAssertionInfoClass):
312
+ return _AssertionPublic._get_volume_detection_mechanism(
313
+ assertion, parameters, default
314
+ )
315
+ elif isinstance(assertion.info, models.FreshnessAssertionInfoClass):
316
+ return _AssertionPublic._get_freshness_detection_mechanism(
317
+ assertion, parameters, default
318
+ )
319
+ # TODO: Consider moving the detection mechanism logic to the assertion classes themselves e.g. _get_assertion_specific_detection_mechanism as an abstract method
320
+ # TODO: Add support here for other detection mechanisms when other assertion types are supported
321
+ else:
322
+ raise SDKNotYetSupportedError(
323
+ f"AssertionType {type(assertion.info).__name__}"
324
+ )
215
325
  else:
216
326
  raise SDKNotYetSupportedError(
217
- f"AssertionEvaluationParametersType {parameters.type}"
218
- )
219
-
220
- @staticmethod
221
- def _has_valid_monitor_info(monitor: Monitor) -> bool:
222
- """Check if monitor has valid info and assertion monitor."""
223
-
224
- def _warn_and_return_false(field_name: str) -> bool:
225
- logger.warning(
226
- f"Monitor {monitor.urn} does not have an `{field_name}` field, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
227
- )
228
- return False
229
-
230
- if monitor.info is None:
231
- return _warn_and_return_false("info")
232
- if monitor.info.assertionMonitor is None:
233
- return _warn_and_return_false("assertionMonitor")
234
- if (
235
- monitor.info.assertionMonitor.assertions is None
236
- or len(monitor.info.assertionMonitor.assertions) == 0
237
- ):
238
- return _warn_and_return_false("assertionMonitor.assertions")
239
-
240
- return True
241
-
242
- @staticmethod
243
- def _get_assertion_parameters(
244
- monitor: Monitor,
245
- default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
246
- ) -> Optional[models.AssertionEvaluationParametersClass]:
247
- """Get the assertion parameters from the monitor."""
248
- # We know these are not None from _has_valid_monitor_info check
249
- assert (
250
- monitor is not None
251
- and monitor.info is not None
252
- and monitor.info.assertionMonitor is not None
253
- )
254
- assertion_monitor = monitor.info.assertionMonitor
255
- assert (
256
- assertion_monitor is not None and assertion_monitor.assertions is not None
257
- )
258
- assertions = assertion_monitor.assertions
259
-
260
- if assertions[0].parameters is None:
261
- logger.warning(
262
- f"Monitor {monitor.urn} does not have a assertionMonitor.assertions[0].parameters, defaulting detection mechanism to {default}"
327
+ f"AssertionEvaluationParametersType {parameters.type} not supported"
263
328
  )
264
- return None
265
- return assertions[0].parameters
266
329
 
267
330
  @staticmethod
268
331
  def _get_freshness_detection_mechanism(
@@ -283,7 +346,7 @@ class _HasSmartFunctionality:
283
346
  elif source_type == models.DatasetFreshnessSourceTypeClass.AUDIT_LOG:
284
347
  return DetectionMechanism.AUDIT_LOG
285
348
  elif source_type == models.DatasetFreshnessSourceTypeClass.FIELD_VALUE:
286
- return _HasSmartFunctionality._get_field_value_detection_mechanism(
349
+ return _AssertionPublic._get_field_value_detection_mechanism(
287
350
  assertion, parameters
288
351
  )
289
352
  elif source_type == models.DatasetFreshnessSourceTypeClass.DATAHUB_OPERATION:
@@ -293,6 +356,33 @@ class _HasSmartFunctionality:
293
356
  else:
294
357
  raise SDKNotYetSupportedError(f"DatasetFreshnessSourceType {source_type}")
295
358
 
359
+ @staticmethod
360
+ def _get_volume_detection_mechanism(
361
+ assertion: Assertion,
362
+ parameters: models.AssertionEvaluationParametersClass,
363
+ default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
364
+ ) -> _DetectionMechanismTypes:
365
+ """Get the detection mechanism for volume assertions."""
366
+ if parameters.datasetVolumeParameters is None:
367
+ logger.warning(
368
+ f"Monitor does not have datasetVolumeParameters, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
369
+ )
370
+ if default is None:
371
+ return DEFAULT_DETECTION_MECHANISM
372
+ else:
373
+ return default
374
+
375
+ source_type = parameters.datasetVolumeParameters.sourceType
376
+ if source_type == models.DatasetVolumeSourceTypeClass.INFORMATION_SCHEMA:
377
+ return DetectionMechanism.INFORMATION_SCHEMA
378
+ elif source_type == models.DatasetVolumeSourceTypeClass.QUERY:
379
+ additional_filter = _AssertionPublic._get_additional_filter(assertion)
380
+ return DetectionMechanism.QUERY(additional_filter=additional_filter)
381
+ elif source_type == models.DatasetVolumeSourceTypeClass.DATAHUB_DATASET_PROFILE:
382
+ return DetectionMechanism.DATASET_PROFILE
383
+ else:
384
+ raise SDKNotYetSupportedError(f"DatasetVolumeSourceType {source_type}")
385
+
296
386
  @staticmethod
297
387
  def _get_field_value_detection_mechanism(
298
388
  assertion: Assertion,
@@ -310,7 +400,7 @@ class _HasSmartFunctionality:
310
400
  return DEFAULT_DETECTION_MECHANISM
311
401
 
312
402
  column_name = field.path
313
- additional_filter = _HasSmartFunctionality._get_additional_filter(assertion)
403
+ additional_filter = _AssertionPublic._get_additional_filter(assertion)
314
404
 
315
405
  if field.kind == models.FreshnessFieldKindClass.LAST_MODIFIED:
316
406
  return DetectionMechanism.LAST_MODIFIED_COLUMN(
@@ -332,7 +422,10 @@ class _HasSmartFunctionality:
332
422
  )
333
423
  return None
334
424
  if (
335
- not isinstance(assertion.info, models.FreshnessAssertionInfoClass)
425
+ not isinstance(
426
+ assertion.info,
427
+ _AssertionPublic._SUPPORTED_WITH_FILTER_ASSERTION_TYPES,
428
+ )
336
429
  or assertion.info.filter is None
337
430
  ):
338
431
  logger.warning(
@@ -345,95 +438,52 @@ class _HasSmartFunctionality:
345
438
  )
346
439
  return assertion.info.filter.sql
347
440
 
441
+ @staticmethod
442
+ def _has_valid_monitor_info(monitor: Monitor) -> bool:
443
+ """Check if monitor has valid info and assertion monitor."""
348
444
 
349
- class _AssertionPublic(ABC):
350
- """
351
- Abstract base class that represents a public facing assertion and contains the common properties of all assertions.
352
- """
445
+ def _warn_and_return_false(field_name: str) -> bool:
446
+ logger.warning(
447
+ f"Monitor {monitor.urn} does not have an `{field_name}` field, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
448
+ )
449
+ return False
353
450
 
354
- def __init__(
355
- self,
356
- *,
357
- urn: AssertionUrn,
358
- dataset_urn: DatasetUrn,
359
- display_name: str,
360
- mode: AssertionMode,
361
- tags: list[TagUrn],
362
- created_by: Optional[CorpUserUrn] = None,
363
- created_at: Union[datetime, None] = None,
364
- updated_by: Optional[CorpUserUrn] = None,
365
- updated_at: Optional[datetime] = None,
366
- ):
367
- """
368
- Initialize the public facing assertion class.
451
+ if monitor.info is None:
452
+ return _warn_and_return_false("info")
453
+ if monitor.info.assertionMonitor is None:
454
+ return _warn_and_return_false("assertionMonitor")
455
+ if (
456
+ monitor.info.assertionMonitor.assertions is None
457
+ or len(monitor.info.assertionMonitor.assertions) == 0
458
+ ):
459
+ return _warn_and_return_false("assertionMonitor.assertions")
369
460
 
370
- Args:
371
- urn: The urn of the assertion.
372
- dataset_urn: The urn of the dataset that the assertion is for.
373
- display_name: The display name of the assertion.
374
- mode: The mode of the assertion (active, inactive).
375
- tags: The tags of the assertion.
376
- created_by: The urn of the user that created the assertion.
377
- created_at: The timestamp of when the assertion was created.
378
- updated_by: The urn of the user that updated the assertion.
379
- updated_at: The timestamp of when the assertion was updated.
380
- """
381
- self._urn = urn
382
- self._dataset_urn = dataset_urn
383
- self._display_name = display_name
384
- self._mode = mode
385
- self._created_by = created_by
386
- self._created_at = created_at
387
- self._updated_by = updated_by
388
- self._updated_at = updated_at
389
- self._tags = tags
390
-
391
- @property
392
- def urn(self) -> AssertionUrn:
393
- return self._urn
394
-
395
- @property
396
- def dataset_urn(self) -> DatasetUrn:
397
- return self._dataset_urn
398
-
399
- @property
400
- def display_name(self) -> str:
401
- return self._display_name
402
-
403
- @property
404
- def mode(self) -> AssertionMode:
405
- return self._mode
406
-
407
- @property
408
- def created_by(self) -> Optional[CorpUserUrn]:
409
- return self._created_by
410
-
411
- @property
412
- def created_at(self) -> Union[datetime, None]:
413
- return self._created_at
414
-
415
- @property
416
- def updated_by(self) -> Optional[CorpUserUrn]:
417
- return self._updated_by
418
-
419
- @property
420
- def updated_at(self) -> Union[datetime, None]:
421
- return self._updated_at
422
-
423
- @property
424
- def tags(self) -> list[TagUrn]:
425
- return self._tags
461
+ return True
426
462
 
427
463
  @staticmethod
428
- def _get_incident_behavior(assertion: Assertion) -> list[AssertionIncidentBehavior]:
429
- incident_behaviors = []
430
- for action in assertion.on_failure + assertion.on_success:
431
- if action.type == models.AssertionActionTypeClass.RAISE_INCIDENT:
432
- incident_behaviors.append(AssertionIncidentBehavior.RAISE_ON_FAIL)
433
- elif action.type == models.AssertionActionTypeClass.RESOLVE_INCIDENT:
434
- incident_behaviors.append(AssertionIncidentBehavior.RESOLVE_ON_PASS)
464
+ def _get_assertion_parameters(
465
+ monitor: Monitor,
466
+ default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
467
+ ) -> Optional[models.AssertionEvaluationParametersClass]:
468
+ """Get the assertion parameters from the monitor."""
469
+ # We know these are not None from _has_valid_monitor_info check
470
+ assert (
471
+ monitor is not None
472
+ and monitor.info is not None
473
+ and monitor.info.assertionMonitor is not None
474
+ )
475
+ assertion_monitor = monitor.info.assertionMonitor
476
+ assert (
477
+ assertion_monitor is not None and assertion_monitor.assertions is not None
478
+ )
479
+ assertions = assertion_monitor.assertions
435
480
 
436
- return incident_behaviors
481
+ if assertions[0].parameters is None:
482
+ logger.warning(
483
+ f"Monitor {monitor.urn} does not have a assertionMonitor.assertions[0].parameters, defaulting detection mechanism to {default}"
484
+ )
485
+ return None
486
+ return assertions[0].parameters
437
487
 
438
488
  @staticmethod
439
489
  def _get_created_by(assertion: Assertion) -> Optional[CorpUserUrn]:
@@ -500,19 +550,22 @@ class _AssertionPublic(ABC):
500
550
  return AssertionMode.INACTIVE
501
551
  return AssertionMode(monitor.info.status.mode)
502
552
 
553
+ @classmethod
503
554
  @abstractmethod
504
- def from_entities(
555
+ def _from_entities(
505
556
  cls,
506
557
  assertion: Assertion,
507
558
  monitor: Monitor,
508
559
  ) -> Self:
509
560
  """
510
561
  Create an assertion from the assertion and monitor entities.
562
+
563
+ Note: This is a private method since it is intended to be called internally by the client.
511
564
  """
512
565
  pass
513
566
 
514
567
 
515
- class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
568
+ class SmartFreshnessAssertion(_HasSchedule, _HasSmartFunctionality, _AssertionPublic):
516
569
  """
517
570
  A class that represents a smart freshness assertion.
518
571
  """
@@ -524,6 +577,7 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
524
577
  dataset_urn: DatasetUrn,
525
578
  display_name: str,
526
579
  mode: AssertionMode,
580
+ schedule: models.CronScheduleClass = DEFAULT_SCHEDULE,
527
581
  sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
528
582
  exclusion_windows: list[ExclusionWindowTypes],
529
583
  training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
@@ -547,6 +601,7 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
547
601
  dataset_urn: The urn of the dataset that the assertion is for.
548
602
  display_name: The display name of the assertion.
549
603
  mode: The mode of the assertion (active, inactive).
604
+ schedule: The schedule of the assertion.
550
605
  sensitivity: The sensitivity of the assertion (low, medium, high).
551
606
  exclusion_windows: The exclusion windows of the assertion.
552
607
  training_data_lookback_days: The max number of days of data to use for training the assertion.
@@ -558,14 +613,111 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
558
613
  updated_by: The urn of the user that updated the assertion.
559
614
  updated_at: The timestamp of when the assertion was updated.
560
615
  """
561
- # Initialize the mixin first
616
+ # Initialize the mixins first
617
+ _HasSchedule.__init__(self, schedule=schedule)
562
618
  _HasSmartFunctionality.__init__(
563
619
  self,
564
620
  sensitivity=sensitivity,
565
621
  exclusion_windows=exclusion_windows,
566
622
  training_data_lookback_days=training_data_lookback_days,
623
+ )
624
+ # Then initialize the parent class
625
+ _AssertionPublic.__init__(
626
+ self,
627
+ urn=urn,
628
+ dataset_urn=dataset_urn,
629
+ display_name=display_name,
630
+ mode=mode,
567
631
  incident_behavior=incident_behavior,
568
632
  detection_mechanism=detection_mechanism,
633
+ created_by=created_by,
634
+ created_at=created_at,
635
+ updated_by=updated_by,
636
+ updated_at=updated_at,
637
+ tags=tags,
638
+ )
639
+
640
+ @classmethod
641
+ def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
642
+ """
643
+ Create a smart freshness assertion from the assertion and monitor entities.
644
+
645
+ Note: This is a private method since it is intended to be called internally by the client.
646
+ """
647
+ return cls(
648
+ urn=assertion.urn,
649
+ dataset_urn=assertion.dataset,
650
+ display_name=assertion.description or "",
651
+ mode=cls._get_mode(monitor),
652
+ schedule=cls._get_schedule(monitor),
653
+ sensitivity=cls._get_sensitivity(monitor),
654
+ exclusion_windows=cls._get_exclusion_windows(monitor),
655
+ training_data_lookback_days=cls._get_training_data_lookback_days(monitor),
656
+ incident_behavior=cls._get_incident_behavior(assertion),
657
+ detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
658
+ created_by=cls._get_created_by(assertion),
659
+ created_at=cls._get_created_at(assertion),
660
+ updated_by=cls._get_updated_by(assertion),
661
+ updated_at=cls._get_updated_at(assertion),
662
+ tags=cls._get_tags(assertion),
663
+ )
664
+
665
+
666
+ class SmartVolumeAssertion(_HasSchedule, _HasSmartFunctionality, _AssertionPublic):
667
+ """
668
+ A class that represents a smart volume assertion.
669
+ """
670
+
671
+ def __init__(
672
+ self,
673
+ *,
674
+ urn: AssertionUrn,
675
+ dataset_urn: DatasetUrn,
676
+ display_name: str,
677
+ mode: AssertionMode,
678
+ schedule: models.CronScheduleClass,
679
+ sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
680
+ exclusion_windows: list[ExclusionWindowTypes],
681
+ training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
682
+ incident_behavior: list[AssertionIncidentBehavior],
683
+ detection_mechanism: Optional[
684
+ _DetectionMechanismTypes
685
+ ] = DEFAULT_DETECTION_MECHANISM,
686
+ tags: list[TagUrn],
687
+ created_by: Optional[CorpUserUrn] = None,
688
+ created_at: Union[datetime, None] = None,
689
+ updated_by: Optional[CorpUserUrn] = None,
690
+ updated_at: Optional[datetime] = None,
691
+ ):
692
+ """
693
+ Initialize a smart volume assertion.
694
+
695
+ Note: Values can be accessed, but not set on the assertion object.
696
+ To update an assertion, use the `upsert_*` method.
697
+ Args:
698
+ urn: The urn of the assertion.
699
+ dataset_urn: The urn of the dataset that the assertion is for.
700
+ display_name: The display name of the assertion.
701
+ mode: The mode of the assertion (active, inactive).
702
+ schedule: The schedule of the assertion.
703
+ sensitivity: The sensitivity of the assertion (low, medium, high).
704
+ exclusion_windows: The exclusion windows of the assertion.
705
+ training_data_lookback_days: The max number of days of data to use for training the assertion.
706
+ incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
707
+ detection_mechanism: The detection mechanism of the assertion.
708
+ tags: The tags applied to the assertion.
709
+ created_by: The urn of the user that created the assertion.
710
+ created_at: The timestamp of when the assertion was created.
711
+ updated_by: The urn of the user that updated the assertion.
712
+ updated_at: The timestamp of when the assertion was updated.
713
+ """
714
+ # Initialize the mixins first
715
+ _HasSchedule.__init__(self, schedule=schedule)
716
+ _HasSmartFunctionality.__init__(
717
+ self,
718
+ sensitivity=sensitivity,
719
+ exclusion_windows=exclusion_windows,
720
+ training_data_lookback_days=training_data_lookback_days,
569
721
  )
570
722
  # Then initialize the parent class
571
723
  _AssertionPublic.__init__(
@@ -574,6 +726,8 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
574
726
  dataset_urn=dataset_urn,
575
727
  display_name=display_name,
576
728
  mode=mode,
729
+ incident_behavior=incident_behavior,
730
+ detection_mechanism=detection_mechanism,
577
731
  created_by=created_by,
578
732
  created_at=created_at,
579
733
  updated_by=updated_by,
@@ -582,15 +736,18 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
582
736
  )
583
737
 
584
738
  @classmethod
585
- def from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
739
+ def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
586
740
  """
587
741
  Create a smart freshness assertion from the assertion and monitor entities.
742
+
743
+ Note: This is a private method since it is intended to be called internally by the client.
588
744
  """
589
745
  return cls(
590
746
  urn=assertion.urn,
591
747
  dataset_urn=assertion.dataset,
592
748
  display_name=assertion.description or "",
593
749
  mode=cls._get_mode(monitor),
750
+ schedule=cls._get_schedule(monitor),
594
751
  sensitivity=cls._get_sensitivity(monitor),
595
752
  exclusion_windows=cls._get_exclusion_windows(monitor),
596
753
  training_data_lookback_days=cls._get_training_data_lookback_days(monitor),
@@ -604,7 +761,158 @@ class SmartFreshnessAssertion(_HasSmartFunctionality, _AssertionPublic):
604
761
  )
605
762
 
606
763
 
764
+ class FreshnessAssertion(_HasSchedule, _AssertionPublic):
765
+ """
766
+ A class that represents a freshness assertion.
767
+ """
768
+
769
+ def __init__(
770
+ self,
771
+ *,
772
+ urn: AssertionUrn,
773
+ dataset_urn: DatasetUrn,
774
+ display_name: str,
775
+ mode: AssertionMode,
776
+ schedule: models.CronScheduleClass,
777
+ freshness_schedule_check_type: Union[
778
+ str, models.FreshnessAssertionScheduleTypeClass
779
+ ],
780
+ lookback_window: Optional[TimeWindowSizeInputTypes],
781
+ tags: list[TagUrn],
782
+ incident_behavior: list[AssertionIncidentBehavior],
783
+ detection_mechanism: Optional[
784
+ _DetectionMechanismTypes
785
+ ] = DEFAULT_DETECTION_MECHANISM,
786
+ created_by: Optional[CorpUserUrn] = None,
787
+ created_at: Union[datetime, None] = None,
788
+ updated_by: Optional[CorpUserUrn] = None,
789
+ updated_at: Optional[datetime] = None,
790
+ ):
791
+ """
792
+ Initialize a freshness assertion.
793
+
794
+ Note: Values can be accessed, but not set on the assertion object.
795
+ To update an assertion, use the `upsert_*` method.
796
+ Args:
797
+ urn: The urn of the assertion.
798
+ dataset_urn: The urn of the dataset that the assertion is for.
799
+ display_name: The display name of the assertion.
800
+ mode: The mode of the assertion (active, inactive).
801
+ schedule: The schedule of the assertion.
802
+ freshness_schedule_check_type: The type of freshness schedule check to be used for the assertion.
803
+ lookback_window: The lookback window to be used for the assertion.
804
+ tags: The tags applied to the assertion.
805
+ incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
806
+ detection_mechanism: The detection mechanism of the assertion.
807
+ created_by: The urn of the user that created the assertion.
808
+ created_at: The timestamp of when the assertion was created.
809
+ updated_by: The urn of the user that updated the assertion.
810
+ updated_at: The timestamp of when the assertion was updated.
811
+ """
812
+ _HasSchedule.__init__(self, schedule=schedule)
813
+ _AssertionPublic.__init__(
814
+ self,
815
+ urn=urn,
816
+ dataset_urn=dataset_urn,
817
+ display_name=display_name,
818
+ mode=mode,
819
+ incident_behavior=incident_behavior,
820
+ detection_mechanism=detection_mechanism,
821
+ created_by=created_by,
822
+ created_at=created_at,
823
+ updated_by=updated_by,
824
+ updated_at=updated_at,
825
+ tags=tags,
826
+ )
827
+ self._freshness_schedule_check_type = freshness_schedule_check_type
828
+ self._lookback_window = lookback_window
829
+
830
+ @property
831
+ def freshness_schedule_check_type(
832
+ self,
833
+ ) -> Union[str, models.FreshnessAssertionScheduleTypeClass]:
834
+ return self._freshness_schedule_check_type
835
+
836
+ @property
837
+ def lookback_window(self) -> Optional[TimeWindowSizeInputTypes]:
838
+ return self._lookback_window
839
+
840
+ @staticmethod
841
+ def _get_freshness_schedule_check_type(
842
+ assertion: Assertion,
843
+ ) -> Union[str, models.FreshnessAssertionScheduleTypeClass]:
844
+ if assertion.info is None:
845
+ raise SDKNotYetSupportedError(
846
+ f"Assertion {assertion.urn} does not have a freshness assertion info, which is not supported"
847
+ )
848
+ if isinstance(assertion.info, models.FreshnessAssertionInfoClass):
849
+ if assertion.info.schedule is None:
850
+ raise SDKNotYetSupportedError(
851
+ f"Traditional freshness assertion {assertion.urn} does not have a schedule, which is not supported"
852
+ )
853
+ return assertion.info.schedule.type
854
+ else:
855
+ raise SDKNotYetSupportedError(
856
+ f"Assertion {assertion.urn} is not a freshness assertion"
857
+ )
858
+
859
+ @staticmethod
860
+ def _get_lookback_window(
861
+ assertion: Assertion,
862
+ ) -> Optional[models.FixedIntervalScheduleClass]:
863
+ if assertion.info is None:
864
+ raise SDKNotYetSupportedError(
865
+ f"Assertion {assertion.urn} does not have a freshness assertion info, which is not supported"
866
+ )
867
+ if isinstance(assertion.info, models.FreshnessAssertionInfoClass):
868
+ if assertion.info.schedule is None:
869
+ raise SDKNotYetSupportedError(
870
+ f"Traditional freshness assertion {assertion.urn} does not have a schedule, which is not supported"
871
+ )
872
+ return assertion.info.schedule.fixedInterval
873
+ else:
874
+ raise SDKNotYetSupportedError(
875
+ f"Assertion {assertion.urn} is not a freshness assertion"
876
+ )
877
+
878
+ @classmethod
879
+ def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
880
+ """
881
+ Create a freshness assertion from the assertion and monitor entities.
882
+ """
883
+ return cls(
884
+ urn=assertion.urn,
885
+ dataset_urn=assertion.dataset,
886
+ display_name=assertion.description or "",
887
+ mode=cls._get_mode(monitor),
888
+ schedule=cls._get_schedule(monitor),
889
+ freshness_schedule_check_type=cls._get_freshness_schedule_check_type(
890
+ assertion
891
+ ),
892
+ lookback_window=cls._get_lookback_window(assertion),
893
+ incident_behavior=cls._get_incident_behavior(assertion),
894
+ detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
895
+ created_by=cls._get_created_by(assertion),
896
+ created_at=cls._get_created_at(assertion),
897
+ updated_by=cls._get_updated_by(assertion),
898
+ updated_at=cls._get_updated_at(assertion),
899
+ tags=cls._get_tags(assertion),
900
+ )
901
+
902
+
903
+ class SmartColumnMetricAssertion(_HasSmartFunctionality, _AssertionPublic):
904
+ """
905
+ A class that represents a smart column metric assertion.
906
+ """
907
+
908
+ def __init__(self) -> None:
909
+ raise NotImplementedError("SmartColumnMetricAssertion is not implemented yet")
910
+
911
+
607
912
  AssertionTypes = Union[
608
913
  SmartFreshnessAssertion,
914
+ SmartVolumeAssertion,
915
+ FreshnessAssertion,
916
+ SmartColumnMetricAssertion,
609
917
  # TODO: Add other assertion types here as we add them.
610
918
  ]