acryl-datahub-cloud 0.3.12.4rc2__py3-none-any.whl → 0.3.13rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub-cloud might be problematic. Click here for more details.
- acryl_datahub_cloud/_codegen_config.json +1 -1
- acryl_datahub_cloud/lineage_features/source.py +8 -2
- acryl_datahub_cloud/metadata/_urns/urn_defs.py +112 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/identity/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py +4 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/module/__init__.py +27 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +4 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/template/__init__.py +25 -0
- acryl_datahub_cloud/metadata/schema.avsc +443 -0
- acryl_datahub_cloud/metadata/schema_classes.py +682 -1
- acryl_datahub_cloud/metadata/schemas/CorpUserSettings.avsc +41 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageModuleKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageModuleProperties.avsc +200 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageTemplateKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageTemplateProperties.avsc +175 -0
- acryl_datahub_cloud/metadata/schemas/DataJobInputOutput.avsc +8 -0
- acryl_datahub_cloud/metadata/schemas/GlobalSettingsInfo.avsc +62 -0
- acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc +9 -0
- acryl_datahub_cloud/metadata/schemas/UpstreamLineage.avsc +9 -0
- acryl_datahub_cloud/sdk/assertion/__init__.py +49 -0
- acryl_datahub_cloud/sdk/assertion/assertion_base.py +65 -806
- acryl_datahub_cloud/sdk/assertion/freshness_assertion.py +201 -0
- acryl_datahub_cloud/sdk/assertion/smart_freshness_assertion.py +165 -0
- acryl_datahub_cloud/sdk/assertion/smart_volume_assertion.py +162 -0
- acryl_datahub_cloud/sdk/assertion/sql_assertion.py +256 -0
- acryl_datahub_cloud/sdk/assertion/volume_assertion.py +156 -0
- acryl_datahub_cloud/sdk/assertion_input/assertion_input.py +0 -344
- acryl_datahub_cloud/sdk/assertion_input/smart_freshness_assertion_input.py +220 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_volume_assertion_input.py +191 -0
- acryl_datahub_cloud/sdk/assertions_client.py +6 -2
- {acryl_datahub_cloud-0.3.12.4rc2.dist-info → acryl_datahub_cloud-0.3.13rc1.dist-info}/METADATA +48 -46
- {acryl_datahub_cloud-0.3.12.4rc2.dist-info → acryl_datahub_cloud-0.3.13rc1.dist-info}/RECORD +35 -22
- {acryl_datahub_cloud-0.3.12.4rc2.dist-info → acryl_datahub_cloud-0.3.13rc1.dist-info}/WHEEL +0 -0
- {acryl_datahub_cloud-0.3.12.4rc2.dist-info → acryl_datahub_cloud-0.3.13rc1.dist-info}/entry_points.txt +0 -0
- {acryl_datahub_cloud-0.3.12.4rc2.dist-info → acryl_datahub_cloud-0.3.13rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,31 +1,31 @@
|
|
|
1
1
|
"""
|
|
2
|
-
This module contains the classes
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
This module contains the base classes and mixins for assertions.
|
|
3
|
+
|
|
4
|
+
The actual assertion classes are now split into separate files for better maintainability:
|
|
5
|
+
- SmartFreshnessAssertion -> smart_freshness_assertion.py
|
|
6
|
+
- SmartVolumeAssertion -> smart_volume_assertion.py
|
|
7
|
+
- VolumeAssertion -> volume_assertion.py
|
|
8
|
+
- FreshnessAssertion -> freshness_assertion.py
|
|
9
|
+
- SqlAssertion -> sql_assertion.py
|
|
7
10
|
"""
|
|
8
11
|
|
|
9
12
|
import logging
|
|
10
13
|
from abc import ABC, abstractmethod
|
|
11
14
|
from datetime import datetime
|
|
12
15
|
from enum import Enum
|
|
13
|
-
from typing import Optional, Union
|
|
16
|
+
from typing import TYPE_CHECKING, Optional, Union
|
|
14
17
|
|
|
15
18
|
from typing_extensions import Self
|
|
16
19
|
|
|
17
20
|
from acryl_datahub_cloud.sdk.assertion_input.assertion_input import (
|
|
18
21
|
ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
|
|
19
22
|
DEFAULT_DETECTION_MECHANISM,
|
|
20
|
-
DEFAULT_EVERY_SIX_HOURS_SCHEDULE,
|
|
21
23
|
DEFAULT_SCHEDULE,
|
|
22
24
|
DEFAULT_SENSITIVITY,
|
|
23
25
|
AssertionIncidentBehavior,
|
|
24
26
|
DetectionMechanism,
|
|
25
27
|
ExclusionWindowTypes,
|
|
26
|
-
FixedRangeExclusionWindow,
|
|
27
28
|
InferenceSensitivity,
|
|
28
|
-
TimeWindowSizeInputTypes,
|
|
29
29
|
_DetectionMechanismTypes,
|
|
30
30
|
)
|
|
31
31
|
from acryl_datahub_cloud.sdk.assertion_input.column_metric_constants import (
|
|
@@ -35,13 +35,6 @@ from acryl_datahub_cloud.sdk.assertion_input.column_metric_constants import (
|
|
|
35
35
|
from acryl_datahub_cloud.sdk.assertion_input.smart_column_metric_assertion_input import (
|
|
36
36
|
SmartColumnMetricAssertionParameters,
|
|
37
37
|
)
|
|
38
|
-
from acryl_datahub_cloud.sdk.assertion_input.sql_assertion_input import (
|
|
39
|
-
SqlAssertionCondition,
|
|
40
|
-
SqlAssertionCriteria,
|
|
41
|
-
)
|
|
42
|
-
from acryl_datahub_cloud.sdk.assertion_input.volume_assertion_input import (
|
|
43
|
-
VolumeAssertionCriteria,
|
|
44
|
-
)
|
|
45
38
|
from acryl_datahub_cloud.sdk.entities.assertion import Assertion
|
|
46
39
|
from acryl_datahub_cloud.sdk.entities.monitor import (
|
|
47
40
|
Monitor,
|
|
@@ -163,6 +156,12 @@ class _HasSmartFunctionality:
|
|
|
163
156
|
f"Monitor {monitor.urn} has a fixed range exclusion window with no fixed range, skipping"
|
|
164
157
|
)
|
|
165
158
|
continue
|
|
159
|
+
# Import FixedRangeExclusionWindow locally to avoid circular import
|
|
160
|
+
# between assertion_base.py and assertion_input.py
|
|
161
|
+
from acryl_datahub_cloud.sdk.assertion_input.assertion_input import (
|
|
162
|
+
FixedRangeExclusionWindow,
|
|
163
|
+
)
|
|
164
|
+
|
|
166
165
|
exclusion_windows.append(
|
|
167
166
|
FixedRangeExclusionWindow(
|
|
168
167
|
start=parse_ts_millis(raw_window.fixedRange.startTimeMillis),
|
|
@@ -715,802 +714,62 @@ class _AssertionPublic(ABC):
|
|
|
715
714
|
return parameters
|
|
716
715
|
|
|
717
716
|
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
mode: AssertionMode,
|
|
730
|
-
schedule: models.CronScheduleClass = DEFAULT_SCHEDULE,
|
|
731
|
-
sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
|
|
732
|
-
exclusion_windows: list[ExclusionWindowTypes],
|
|
733
|
-
training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
|
|
734
|
-
incident_behavior: list[AssertionIncidentBehavior],
|
|
735
|
-
detection_mechanism: Optional[
|
|
736
|
-
_DetectionMechanismTypes
|
|
737
|
-
] = DEFAULT_DETECTION_MECHANISM,
|
|
738
|
-
tags: list[TagUrn],
|
|
739
|
-
created_by: Optional[CorpUserUrn] = None,
|
|
740
|
-
created_at: Union[datetime, None] = None,
|
|
741
|
-
updated_by: Optional[CorpUserUrn] = None,
|
|
742
|
-
updated_at: Optional[datetime] = None,
|
|
743
|
-
):
|
|
744
|
-
"""
|
|
745
|
-
Initialize a smart freshness assertion.
|
|
746
|
-
|
|
747
|
-
Note: Values can be accessed, but not set on the assertion object.
|
|
748
|
-
To update an assertion, use the `upsert_*` method.
|
|
749
|
-
Args:
|
|
750
|
-
urn: The urn of the assertion.
|
|
751
|
-
dataset_urn: The urn of the dataset that the assertion is for.
|
|
752
|
-
display_name: The display name of the assertion.
|
|
753
|
-
mode: The mode of the assertion (active, inactive).
|
|
754
|
-
schedule: The schedule of the assertion.
|
|
755
|
-
sensitivity: The sensitivity of the assertion (low, medium, high).
|
|
756
|
-
exclusion_windows: The exclusion windows of the assertion.
|
|
757
|
-
training_data_lookback_days: The max number of days of data to use for training the assertion.
|
|
758
|
-
incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
|
|
759
|
-
detection_mechanism: The detection mechanism of the assertion.
|
|
760
|
-
tags: The tags applied to the assertion.
|
|
761
|
-
created_by: The urn of the user that created the assertion.
|
|
762
|
-
created_at: The timestamp of when the assertion was created.
|
|
763
|
-
updated_by: The urn of the user that updated the assertion.
|
|
764
|
-
updated_at: The timestamp of when the assertion was updated.
|
|
765
|
-
"""
|
|
766
|
-
# Initialize the mixins first
|
|
767
|
-
_HasSchedule.__init__(self, schedule=schedule)
|
|
768
|
-
_HasSmartFunctionality.__init__(
|
|
769
|
-
self,
|
|
770
|
-
sensitivity=sensitivity,
|
|
771
|
-
exclusion_windows=exclusion_windows,
|
|
772
|
-
training_data_lookback_days=training_data_lookback_days,
|
|
773
|
-
)
|
|
774
|
-
# Then initialize the parent class
|
|
775
|
-
_AssertionPublic.__init__(
|
|
776
|
-
self,
|
|
777
|
-
urn=urn,
|
|
778
|
-
dataset_urn=dataset_urn,
|
|
779
|
-
display_name=display_name,
|
|
780
|
-
mode=mode,
|
|
781
|
-
incident_behavior=incident_behavior,
|
|
782
|
-
detection_mechanism=detection_mechanism,
|
|
783
|
-
created_by=created_by,
|
|
784
|
-
created_at=created_at,
|
|
785
|
-
updated_by=updated_by,
|
|
786
|
-
updated_at=updated_at,
|
|
787
|
-
tags=tags,
|
|
788
|
-
)
|
|
789
|
-
|
|
790
|
-
@classmethod
|
|
791
|
-
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
792
|
-
"""
|
|
793
|
-
Create a smart freshness assertion from the assertion and monitor entities.
|
|
794
|
-
|
|
795
|
-
Note: This is a private method since it is intended to be called internally by the client.
|
|
796
|
-
"""
|
|
797
|
-
return cls(
|
|
798
|
-
urn=assertion.urn,
|
|
799
|
-
dataset_urn=assertion.dataset,
|
|
800
|
-
display_name=assertion.description or "",
|
|
801
|
-
mode=cls._get_mode(monitor),
|
|
802
|
-
schedule=cls._get_schedule(monitor),
|
|
803
|
-
sensitivity=cls._get_sensitivity(monitor),
|
|
804
|
-
exclusion_windows=cls._get_exclusion_windows(monitor),
|
|
805
|
-
training_data_lookback_days=cls._get_training_data_lookback_days(monitor),
|
|
806
|
-
incident_behavior=cls._get_incident_behavior(assertion),
|
|
807
|
-
detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
|
|
808
|
-
created_by=cls._get_created_by(assertion),
|
|
809
|
-
created_at=cls._get_created_at(assertion),
|
|
810
|
-
updated_by=cls._get_updated_by(assertion),
|
|
811
|
-
updated_at=cls._get_updated_at(assertion),
|
|
812
|
-
tags=cls._get_tags(assertion),
|
|
813
|
-
)
|
|
814
|
-
|
|
815
|
-
@staticmethod
|
|
816
|
-
def _get_detection_mechanism(
|
|
817
|
-
assertion: Assertion,
|
|
818
|
-
monitor: Monitor,
|
|
819
|
-
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
820
|
-
) -> Optional[_DetectionMechanismTypes]:
|
|
821
|
-
"""Get the detection mechanism for freshness assertions."""
|
|
822
|
-
parameters = _AssertionPublic._get_validated_detection_context(
|
|
823
|
-
monitor,
|
|
824
|
-
assertion,
|
|
825
|
-
models.AssertionEvaluationParametersTypeClass.DATASET_FRESHNESS,
|
|
826
|
-
models.FreshnessAssertionInfoClass,
|
|
827
|
-
default,
|
|
828
|
-
)
|
|
829
|
-
if parameters is None:
|
|
830
|
-
return default
|
|
831
|
-
if parameters.datasetFreshnessParameters is None:
|
|
832
|
-
logger.warning(
|
|
833
|
-
f"Monitor does not have datasetFreshnessParameters, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
|
|
834
|
-
)
|
|
835
|
-
return default
|
|
836
|
-
source_type = parameters.datasetFreshnessParameters.sourceType
|
|
837
|
-
if source_type == models.DatasetFreshnessSourceTypeClass.INFORMATION_SCHEMA:
|
|
838
|
-
return DetectionMechanism.INFORMATION_SCHEMA
|
|
839
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.AUDIT_LOG:
|
|
840
|
-
return DetectionMechanism.AUDIT_LOG
|
|
841
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.FIELD_VALUE:
|
|
842
|
-
return _AssertionPublic._get_field_value_detection_mechanism(
|
|
843
|
-
assertion, parameters
|
|
844
|
-
)
|
|
845
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.DATAHUB_OPERATION:
|
|
846
|
-
return DetectionMechanism.DATAHUB_OPERATION
|
|
847
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.FILE_METADATA:
|
|
848
|
-
raise SDKNotYetSupportedError("FILE_METADATA DatasetFreshnessSourceType")
|
|
849
|
-
else:
|
|
850
|
-
raise SDKNotYetSupportedError(f"DatasetFreshnessSourceType {source_type}")
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
class SmartVolumeAssertion(_HasSchedule, _HasSmartFunctionality, _AssertionPublic):
|
|
854
|
-
"""
|
|
855
|
-
A class that represents a smart volume assertion.
|
|
856
|
-
"""
|
|
857
|
-
|
|
858
|
-
def __init__(
|
|
859
|
-
self,
|
|
860
|
-
*,
|
|
861
|
-
urn: AssertionUrn,
|
|
862
|
-
dataset_urn: DatasetUrn,
|
|
863
|
-
display_name: str,
|
|
864
|
-
mode: AssertionMode,
|
|
865
|
-
schedule: models.CronScheduleClass,
|
|
866
|
-
sensitivity: InferenceSensitivity = DEFAULT_SENSITIVITY,
|
|
867
|
-
exclusion_windows: list[ExclusionWindowTypes],
|
|
868
|
-
training_data_lookback_days: int = ASSERTION_MONITOR_DEFAULT_TRAINING_LOOKBACK_WINDOW_DAYS,
|
|
869
|
-
incident_behavior: list[AssertionIncidentBehavior],
|
|
870
|
-
detection_mechanism: Optional[
|
|
871
|
-
_DetectionMechanismTypes
|
|
872
|
-
] = DEFAULT_DETECTION_MECHANISM,
|
|
873
|
-
tags: list[TagUrn],
|
|
874
|
-
created_by: Optional[CorpUserUrn] = None,
|
|
875
|
-
created_at: Union[datetime, None] = None,
|
|
876
|
-
updated_by: Optional[CorpUserUrn] = None,
|
|
877
|
-
updated_at: Optional[datetime] = None,
|
|
878
|
-
):
|
|
879
|
-
"""
|
|
880
|
-
Initialize a smart volume assertion.
|
|
881
|
-
|
|
882
|
-
Note: Values can be accessed, but not set on the assertion object.
|
|
883
|
-
To update an assertion, use the `upsert_*` method.
|
|
884
|
-
Args:
|
|
885
|
-
urn: The urn of the assertion.
|
|
886
|
-
dataset_urn: The urn of the dataset that the assertion is for.
|
|
887
|
-
display_name: The display name of the assertion.
|
|
888
|
-
mode: The mode of the assertion (active, inactive).
|
|
889
|
-
schedule: The schedule of the assertion.
|
|
890
|
-
sensitivity: The sensitivity of the assertion (low, medium, high).
|
|
891
|
-
exclusion_windows: The exclusion windows of the assertion.
|
|
892
|
-
training_data_lookback_days: The max number of days of data to use for training the assertion.
|
|
893
|
-
incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
|
|
894
|
-
detection_mechanism: The detection mechanism of the assertion.
|
|
895
|
-
tags: The tags applied to the assertion.
|
|
896
|
-
created_by: The urn of the user that created the assertion.
|
|
897
|
-
created_at: The timestamp of when the assertion was created.
|
|
898
|
-
updated_by: The urn of the user that updated the assertion.
|
|
899
|
-
updated_at: The timestamp of when the assertion was updated.
|
|
900
|
-
"""
|
|
901
|
-
# Initialize the mixins first
|
|
902
|
-
_HasSchedule.__init__(self, schedule=schedule)
|
|
903
|
-
_HasSmartFunctionality.__init__(
|
|
904
|
-
self,
|
|
905
|
-
sensitivity=sensitivity,
|
|
906
|
-
exclusion_windows=exclusion_windows,
|
|
907
|
-
training_data_lookback_days=training_data_lookback_days,
|
|
908
|
-
)
|
|
909
|
-
# Then initialize the parent class
|
|
910
|
-
_AssertionPublic.__init__(
|
|
911
|
-
self,
|
|
912
|
-
urn=urn,
|
|
913
|
-
dataset_urn=dataset_urn,
|
|
914
|
-
display_name=display_name,
|
|
915
|
-
mode=mode,
|
|
916
|
-
incident_behavior=incident_behavior,
|
|
917
|
-
detection_mechanism=detection_mechanism,
|
|
918
|
-
created_by=created_by,
|
|
919
|
-
created_at=created_at,
|
|
920
|
-
updated_by=updated_by,
|
|
921
|
-
updated_at=updated_at,
|
|
922
|
-
tags=tags,
|
|
923
|
-
)
|
|
924
|
-
|
|
925
|
-
@classmethod
|
|
926
|
-
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
927
|
-
"""
|
|
928
|
-
Create a smart freshness assertion from the assertion and monitor entities.
|
|
929
|
-
|
|
930
|
-
Note: This is a private method since it is intended to be called internally by the client.
|
|
931
|
-
"""
|
|
932
|
-
return cls(
|
|
933
|
-
urn=assertion.urn,
|
|
934
|
-
dataset_urn=assertion.dataset,
|
|
935
|
-
display_name=assertion.description or "",
|
|
936
|
-
mode=cls._get_mode(monitor),
|
|
937
|
-
schedule=cls._get_schedule(monitor),
|
|
938
|
-
sensitivity=cls._get_sensitivity(monitor),
|
|
939
|
-
exclusion_windows=cls._get_exclusion_windows(monitor),
|
|
940
|
-
training_data_lookback_days=cls._get_training_data_lookback_days(monitor),
|
|
941
|
-
incident_behavior=cls._get_incident_behavior(assertion),
|
|
942
|
-
detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
|
|
943
|
-
created_by=cls._get_created_by(assertion),
|
|
944
|
-
created_at=cls._get_created_at(assertion),
|
|
945
|
-
updated_by=cls._get_updated_by(assertion),
|
|
946
|
-
updated_at=cls._get_updated_at(assertion),
|
|
947
|
-
tags=cls._get_tags(assertion),
|
|
948
|
-
)
|
|
949
|
-
|
|
950
|
-
@staticmethod
|
|
951
|
-
def _get_detection_mechanism(
|
|
952
|
-
assertion: Assertion,
|
|
953
|
-
monitor: Monitor,
|
|
954
|
-
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
955
|
-
) -> Optional[_DetectionMechanismTypes]:
|
|
956
|
-
"""Get the detection mechanism for volume assertions."""
|
|
957
|
-
parameters = _AssertionPublic._get_validated_detection_context(
|
|
958
|
-
monitor,
|
|
959
|
-
assertion,
|
|
960
|
-
models.AssertionEvaluationParametersTypeClass.DATASET_VOLUME,
|
|
961
|
-
models.VolumeAssertionInfoClass,
|
|
962
|
-
default,
|
|
963
|
-
)
|
|
964
|
-
if parameters is None:
|
|
965
|
-
return default
|
|
966
|
-
if parameters.datasetVolumeParameters is None:
|
|
967
|
-
logger.warning(
|
|
968
|
-
f"Monitor does not have datasetVolumeParameters, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
|
|
969
|
-
)
|
|
970
|
-
if default is None:
|
|
971
|
-
return DEFAULT_DETECTION_MECHANISM
|
|
972
|
-
else:
|
|
973
|
-
return default
|
|
974
|
-
source_type = parameters.datasetVolumeParameters.sourceType
|
|
975
|
-
if source_type == models.DatasetVolumeSourceTypeClass.INFORMATION_SCHEMA:
|
|
976
|
-
return DetectionMechanism.INFORMATION_SCHEMA
|
|
977
|
-
elif source_type == models.DatasetVolumeSourceTypeClass.QUERY:
|
|
978
|
-
additional_filter = _AssertionPublic._get_additional_filter(assertion)
|
|
979
|
-
return DetectionMechanism.QUERY(additional_filter=additional_filter)
|
|
980
|
-
elif source_type == models.DatasetVolumeSourceTypeClass.DATAHUB_DATASET_PROFILE:
|
|
981
|
-
return DetectionMechanism.DATASET_PROFILE
|
|
982
|
-
else:
|
|
983
|
-
raise SDKNotYetSupportedError(f"DatasetVolumeSourceType {source_type}")
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
class VolumeAssertion(_HasSchedule, _AssertionPublic):
|
|
987
|
-
"""
|
|
988
|
-
A class that represents a volume assertion.
|
|
989
|
-
"""
|
|
990
|
-
|
|
991
|
-
def __init__(
|
|
992
|
-
self,
|
|
993
|
-
*,
|
|
994
|
-
urn: AssertionUrn,
|
|
995
|
-
dataset_urn: DatasetUrn,
|
|
996
|
-
display_name: str,
|
|
997
|
-
mode: AssertionMode,
|
|
998
|
-
schedule: models.CronScheduleClass,
|
|
999
|
-
criteria: VolumeAssertionCriteria,
|
|
1000
|
-
tags: list[TagUrn],
|
|
1001
|
-
incident_behavior: list[AssertionIncidentBehavior],
|
|
1002
|
-
detection_mechanism: Optional[
|
|
1003
|
-
_DetectionMechanismTypes
|
|
1004
|
-
] = DEFAULT_DETECTION_MECHANISM,
|
|
1005
|
-
created_by: Optional[CorpUserUrn] = None,
|
|
1006
|
-
created_at: Union[datetime, None] = None,
|
|
1007
|
-
updated_by: Optional[CorpUserUrn] = None,
|
|
1008
|
-
updated_at: Optional[datetime] = None,
|
|
1009
|
-
):
|
|
1010
|
-
"""
|
|
1011
|
-
Initialize a volume assertion.
|
|
1012
|
-
|
|
1013
|
-
Note: Values can be accessed, but not set on the assertion object.
|
|
1014
|
-
To update an assertion, use the `upsert_*` method.
|
|
1015
|
-
Args:
|
|
1016
|
-
urn: The urn of the assertion.
|
|
1017
|
-
dataset_urn: The urn of the dataset that the assertion is for.
|
|
1018
|
-
display_name: The display name of the assertion.
|
|
1019
|
-
mode: The mode of the assertion (active, inactive).
|
|
1020
|
-
schedule: The schedule of the assertion.
|
|
1021
|
-
criteria: The volume assertion criteria.
|
|
1022
|
-
tags: The tags applied to the assertion.
|
|
1023
|
-
incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
|
|
1024
|
-
detection_mechanism: The detection mechanism of the assertion.
|
|
1025
|
-
created_by: The urn of the user that created the assertion.
|
|
1026
|
-
created_at: The timestamp of when the assertion was created.
|
|
1027
|
-
updated_by: The urn of the user that updated the assertion.
|
|
1028
|
-
updated_at: The timestamp of when the assertion was updated.
|
|
1029
|
-
"""
|
|
1030
|
-
_HasSchedule.__init__(self, schedule=schedule)
|
|
1031
|
-
_AssertionPublic.__init__(
|
|
1032
|
-
self,
|
|
1033
|
-
urn=urn,
|
|
1034
|
-
dataset_urn=dataset_urn,
|
|
1035
|
-
display_name=display_name,
|
|
1036
|
-
mode=mode,
|
|
1037
|
-
incident_behavior=incident_behavior,
|
|
1038
|
-
detection_mechanism=detection_mechanism,
|
|
1039
|
-
created_by=created_by,
|
|
1040
|
-
created_at=created_at,
|
|
1041
|
-
updated_by=updated_by,
|
|
1042
|
-
updated_at=updated_at,
|
|
1043
|
-
tags=tags,
|
|
1044
|
-
)
|
|
1045
|
-
self._criteria = criteria
|
|
1046
|
-
|
|
1047
|
-
@property
|
|
1048
|
-
def criteria(self) -> VolumeAssertionCriteria:
|
|
1049
|
-
return self._criteria
|
|
1050
|
-
|
|
1051
|
-
@staticmethod
|
|
1052
|
-
def _get_volume_definition(
|
|
1053
|
-
assertion: Assertion,
|
|
1054
|
-
) -> VolumeAssertionCriteria:
|
|
1055
|
-
"""Get volume assertion definition from a DataHub assertion entity."""
|
|
1056
|
-
return VolumeAssertionCriteria.from_assertion(assertion)
|
|
1057
|
-
|
|
1058
|
-
@staticmethod
|
|
1059
|
-
def _get_detection_mechanism(
|
|
1060
|
-
assertion: Assertion,
|
|
1061
|
-
monitor: Monitor,
|
|
1062
|
-
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
1063
|
-
) -> Optional[_DetectionMechanismTypes]:
|
|
1064
|
-
"""Get the detection mechanism for volume assertions."""
|
|
1065
|
-
parameters = _AssertionPublic._get_validated_detection_context(
|
|
1066
|
-
monitor,
|
|
1067
|
-
assertion,
|
|
1068
|
-
models.AssertionEvaluationParametersTypeClass.DATASET_VOLUME,
|
|
1069
|
-
models.VolumeAssertionInfoClass,
|
|
1070
|
-
default,
|
|
1071
|
-
)
|
|
1072
|
-
if parameters is None:
|
|
1073
|
-
return default
|
|
1074
|
-
if parameters.datasetVolumeParameters is None:
|
|
1075
|
-
logger.warning(
|
|
1076
|
-
f"Monitor does not have datasetVolumeParameters, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
|
|
1077
|
-
)
|
|
1078
|
-
if default is None:
|
|
1079
|
-
return DEFAULT_DETECTION_MECHANISM
|
|
1080
|
-
else:
|
|
1081
|
-
return default
|
|
1082
|
-
source_type = parameters.datasetVolumeParameters.sourceType
|
|
1083
|
-
if source_type == models.DatasetVolumeSourceTypeClass.INFORMATION_SCHEMA:
|
|
1084
|
-
return DetectionMechanism.INFORMATION_SCHEMA
|
|
1085
|
-
elif source_type == models.DatasetVolumeSourceTypeClass.QUERY:
|
|
1086
|
-
additional_filter = _AssertionPublic._get_additional_filter(assertion)
|
|
1087
|
-
return DetectionMechanism.QUERY(additional_filter=additional_filter)
|
|
1088
|
-
elif source_type == models.DatasetVolumeSourceTypeClass.DATAHUB_DATASET_PROFILE:
|
|
1089
|
-
return DetectionMechanism.DATASET_PROFILE
|
|
1090
|
-
else:
|
|
1091
|
-
raise SDKNotYetSupportedError(f"DatasetVolumeSourceType {source_type}")
|
|
1092
|
-
|
|
1093
|
-
@classmethod
|
|
1094
|
-
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
1095
|
-
"""
|
|
1096
|
-
Create a volume assertion from the assertion and monitor entities.
|
|
1097
|
-
"""
|
|
1098
|
-
return cls(
|
|
1099
|
-
urn=assertion.urn,
|
|
1100
|
-
dataset_urn=assertion.dataset,
|
|
1101
|
-
display_name=assertion.description or "",
|
|
1102
|
-
mode=cls._get_mode(monitor),
|
|
1103
|
-
schedule=cls._get_schedule(monitor),
|
|
1104
|
-
criteria=cls._get_volume_definition(assertion),
|
|
1105
|
-
incident_behavior=cls._get_incident_behavior(assertion),
|
|
1106
|
-
detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
|
|
1107
|
-
created_by=cls._get_created_by(assertion),
|
|
1108
|
-
created_at=cls._get_created_at(assertion),
|
|
1109
|
-
updated_by=cls._get_updated_by(assertion),
|
|
1110
|
-
updated_at=cls._get_updated_at(assertion),
|
|
1111
|
-
tags=cls._get_tags(assertion),
|
|
1112
|
-
)
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
class FreshnessAssertion(_HasSchedule, _AssertionPublic):
|
|
1116
|
-
"""
|
|
1117
|
-
A class that represents a freshness assertion.
|
|
1118
|
-
"""
|
|
717
|
+
if TYPE_CHECKING:
|
|
718
|
+
# Import the assertion classes from their separate files for type checking
|
|
719
|
+
from acryl_datahub_cloud.sdk.assertion.freshness_assertion import FreshnessAssertion
|
|
720
|
+
from acryl_datahub_cloud.sdk.assertion.smart_freshness_assertion import (
|
|
721
|
+
SmartFreshnessAssertion,
|
|
722
|
+
)
|
|
723
|
+
from acryl_datahub_cloud.sdk.assertion.smart_volume_assertion import (
|
|
724
|
+
SmartVolumeAssertion,
|
|
725
|
+
)
|
|
726
|
+
from acryl_datahub_cloud.sdk.assertion.sql_assertion import SqlAssertion
|
|
727
|
+
from acryl_datahub_cloud.sdk.assertion.volume_assertion import VolumeAssertion
|
|
1119
728
|
|
|
1120
|
-
def __init__(
|
|
1121
|
-
self,
|
|
1122
|
-
*,
|
|
1123
|
-
urn: AssertionUrn,
|
|
1124
|
-
dataset_urn: DatasetUrn,
|
|
1125
|
-
display_name: str,
|
|
1126
|
-
mode: AssertionMode,
|
|
1127
|
-
schedule: models.CronScheduleClass,
|
|
1128
|
-
freshness_schedule_check_type: Union[
|
|
1129
|
-
str, models.FreshnessAssertionScheduleTypeClass
|
|
1130
|
-
],
|
|
1131
|
-
lookback_window: Optional[TimeWindowSizeInputTypes],
|
|
1132
|
-
tags: list[TagUrn],
|
|
1133
|
-
incident_behavior: list[AssertionIncidentBehavior],
|
|
1134
|
-
detection_mechanism: Optional[
|
|
1135
|
-
_DetectionMechanismTypes
|
|
1136
|
-
] = DEFAULT_DETECTION_MECHANISM,
|
|
1137
|
-
created_by: Optional[CorpUserUrn] = None,
|
|
1138
|
-
created_at: Union[datetime, None] = None,
|
|
1139
|
-
updated_by: Optional[CorpUserUrn] = None,
|
|
1140
|
-
updated_at: Optional[datetime] = None,
|
|
1141
|
-
):
|
|
1142
|
-
"""
|
|
1143
|
-
Initialize a freshness assertion.
|
|
1144
729
|
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
mode: The mode of the assertion (active, inactive).
|
|
1152
|
-
schedule: The schedule of the assertion.
|
|
1153
|
-
freshness_schedule_check_type: The type of freshness schedule check to be used for the assertion.
|
|
1154
|
-
lookback_window: The lookback window to be used for the assertion.
|
|
1155
|
-
tags: The tags applied to the assertion.
|
|
1156
|
-
incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
|
|
1157
|
-
detection_mechanism: The detection mechanism of the assertion.
|
|
1158
|
-
created_by: The urn of the user that created the assertion.
|
|
1159
|
-
created_at: The timestamp of when the assertion was created.
|
|
1160
|
-
updated_by: The urn of the user that updated the assertion.
|
|
1161
|
-
updated_at: The timestamp of when the assertion was updated.
|
|
1162
|
-
"""
|
|
1163
|
-
_HasSchedule.__init__(self, schedule=schedule)
|
|
1164
|
-
_AssertionPublic.__init__(
|
|
1165
|
-
self,
|
|
1166
|
-
urn=urn,
|
|
1167
|
-
dataset_urn=dataset_urn,
|
|
1168
|
-
display_name=display_name,
|
|
1169
|
-
mode=mode,
|
|
1170
|
-
incident_behavior=incident_behavior,
|
|
1171
|
-
detection_mechanism=detection_mechanism,
|
|
1172
|
-
created_by=created_by,
|
|
1173
|
-
created_at=created_at,
|
|
1174
|
-
updated_by=updated_by,
|
|
1175
|
-
updated_at=updated_at,
|
|
1176
|
-
tags=tags,
|
|
730
|
+
# For runtime access, we'll use dynamic imports to avoid circular dependencies
|
|
731
|
+
def __getattr__(name: str) -> type:
|
|
732
|
+
"""Dynamic import for assertion classes to avoid circular dependencies."""
|
|
733
|
+
if name == "SmartFreshnessAssertion":
|
|
734
|
+
from acryl_datahub_cloud.sdk.assertion.smart_freshness_assertion import (
|
|
735
|
+
SmartFreshnessAssertion,
|
|
1177
736
|
)
|
|
1178
|
-
self._freshness_schedule_check_type = freshness_schedule_check_type
|
|
1179
|
-
self._lookback_window = lookback_window
|
|
1180
|
-
|
|
1181
|
-
@property
|
|
1182
|
-
def freshness_schedule_check_type(
|
|
1183
|
-
self,
|
|
1184
|
-
) -> Union[str, models.FreshnessAssertionScheduleTypeClass]:
|
|
1185
|
-
return self._freshness_schedule_check_type
|
|
1186
737
|
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
@staticmethod
|
|
1192
|
-
def _get_freshness_schedule_check_type(
|
|
1193
|
-
assertion: Assertion,
|
|
1194
|
-
) -> Union[str, models.FreshnessAssertionScheduleTypeClass]:
|
|
1195
|
-
if assertion.info is None:
|
|
1196
|
-
raise SDKNotYetSupportedError(
|
|
1197
|
-
f"Assertion {assertion.urn} does not have a freshness assertion info, which is not supported"
|
|
1198
|
-
)
|
|
1199
|
-
if isinstance(assertion.info, models.FreshnessAssertionInfoClass):
|
|
1200
|
-
if assertion.info.schedule is None:
|
|
1201
|
-
raise SDKNotYetSupportedError(
|
|
1202
|
-
f"Traditional freshness assertion {assertion.urn} does not have a schedule, which is not supported"
|
|
1203
|
-
)
|
|
1204
|
-
return assertion.info.schedule.type
|
|
1205
|
-
else:
|
|
1206
|
-
raise SDKNotYetSupportedError(
|
|
1207
|
-
f"Assertion {assertion.urn} is not a freshness assertion"
|
|
1208
|
-
)
|
|
1209
|
-
|
|
1210
|
-
@staticmethod
|
|
1211
|
-
def _get_lookback_window(
|
|
1212
|
-
assertion: Assertion,
|
|
1213
|
-
) -> Optional[models.FixedIntervalScheduleClass]:
|
|
1214
|
-
if assertion.info is None:
|
|
1215
|
-
raise SDKNotYetSupportedError(
|
|
1216
|
-
f"Assertion {assertion.urn} does not have a freshness assertion info, which is not supported"
|
|
1217
|
-
)
|
|
1218
|
-
if isinstance(assertion.info, models.FreshnessAssertionInfoClass):
|
|
1219
|
-
if assertion.info.schedule is None:
|
|
1220
|
-
raise SDKNotYetSupportedError(
|
|
1221
|
-
f"Traditional freshness assertion {assertion.urn} does not have a schedule, which is not supported"
|
|
1222
|
-
)
|
|
1223
|
-
return assertion.info.schedule.fixedInterval
|
|
1224
|
-
else:
|
|
1225
|
-
raise SDKNotYetSupportedError(
|
|
1226
|
-
f"Assertion {assertion.urn} is not a freshness assertion"
|
|
1227
|
-
)
|
|
1228
|
-
|
|
1229
|
-
@staticmethod
|
|
1230
|
-
def _get_detection_mechanism(
|
|
1231
|
-
assertion: Assertion,
|
|
1232
|
-
monitor: Monitor,
|
|
1233
|
-
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
1234
|
-
) -> Optional[_DetectionMechanismTypes]:
|
|
1235
|
-
"""Get the detection mechanism for freshness assertions."""
|
|
1236
|
-
parameters = _AssertionPublic._get_validated_detection_context(
|
|
1237
|
-
monitor,
|
|
1238
|
-
assertion,
|
|
1239
|
-
models.AssertionEvaluationParametersTypeClass.DATASET_FRESHNESS,
|
|
1240
|
-
models.FreshnessAssertionInfoClass,
|
|
1241
|
-
default,
|
|
738
|
+
return SmartFreshnessAssertion
|
|
739
|
+
elif name == "SmartVolumeAssertion":
|
|
740
|
+
from acryl_datahub_cloud.sdk.assertion.smart_volume_assertion import (
|
|
741
|
+
SmartVolumeAssertion,
|
|
1242
742
|
)
|
|
1243
|
-
if parameters is None:
|
|
1244
|
-
return default
|
|
1245
|
-
if parameters.datasetFreshnessParameters is None:
|
|
1246
|
-
logger.warning(
|
|
1247
|
-
f"Monitor does not have datasetFreshnessParameters, defaulting detection mechanism to {DEFAULT_DETECTION_MECHANISM}"
|
|
1248
|
-
)
|
|
1249
|
-
return default
|
|
1250
|
-
source_type = parameters.datasetFreshnessParameters.sourceType
|
|
1251
|
-
if source_type == models.DatasetFreshnessSourceTypeClass.INFORMATION_SCHEMA:
|
|
1252
|
-
return DetectionMechanism.INFORMATION_SCHEMA
|
|
1253
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.AUDIT_LOG:
|
|
1254
|
-
return DetectionMechanism.AUDIT_LOG
|
|
1255
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.FIELD_VALUE:
|
|
1256
|
-
return _AssertionPublic._get_field_value_detection_mechanism(
|
|
1257
|
-
assertion, parameters
|
|
1258
|
-
)
|
|
1259
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.DATAHUB_OPERATION:
|
|
1260
|
-
return DetectionMechanism.DATAHUB_OPERATION
|
|
1261
|
-
elif source_type == models.DatasetFreshnessSourceTypeClass.FILE_METADATA:
|
|
1262
|
-
raise SDKNotYetSupportedError("FILE_METADATA DatasetFreshnessSourceType")
|
|
1263
|
-
else:
|
|
1264
|
-
raise SDKNotYetSupportedError(f"DatasetFreshnessSourceType {source_type}")
|
|
1265
|
-
|
|
1266
|
-
@classmethod
|
|
1267
|
-
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
1268
|
-
"""
|
|
1269
|
-
Create a freshness assertion from the assertion and monitor entities.
|
|
1270
|
-
"""
|
|
1271
|
-
return cls(
|
|
1272
|
-
urn=assertion.urn,
|
|
1273
|
-
dataset_urn=assertion.dataset,
|
|
1274
|
-
display_name=assertion.description or "",
|
|
1275
|
-
mode=cls._get_mode(monitor),
|
|
1276
|
-
schedule=cls._get_schedule(monitor),
|
|
1277
|
-
freshness_schedule_check_type=cls._get_freshness_schedule_check_type(
|
|
1278
|
-
assertion
|
|
1279
|
-
),
|
|
1280
|
-
lookback_window=cls._get_lookback_window(assertion),
|
|
1281
|
-
incident_behavior=cls._get_incident_behavior(assertion),
|
|
1282
|
-
detection_mechanism=cls._get_detection_mechanism(assertion, monitor),
|
|
1283
|
-
created_by=cls._get_created_by(assertion),
|
|
1284
|
-
created_at=cls._get_created_at(assertion),
|
|
1285
|
-
updated_by=cls._get_updated_by(assertion),
|
|
1286
|
-
updated_at=cls._get_updated_at(assertion),
|
|
1287
|
-
tags=cls._get_tags(assertion),
|
|
1288
|
-
)
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
class SqlAssertion(_AssertionPublic, _HasSchedule):
|
|
1292
|
-
"""
|
|
1293
|
-
A class that represents a SQL assertion.
|
|
1294
|
-
"""
|
|
1295
|
-
|
|
1296
|
-
def __init__(
|
|
1297
|
-
self,
|
|
1298
|
-
*,
|
|
1299
|
-
urn: AssertionUrn,
|
|
1300
|
-
dataset_urn: DatasetUrn,
|
|
1301
|
-
display_name: str,
|
|
1302
|
-
mode: AssertionMode,
|
|
1303
|
-
statement: str,
|
|
1304
|
-
criteria: SqlAssertionCriteria,
|
|
1305
|
-
schedule: models.CronScheduleClass,
|
|
1306
|
-
tags: list[TagUrn],
|
|
1307
|
-
incident_behavior: list[AssertionIncidentBehavior],
|
|
1308
|
-
created_by: Optional[CorpUserUrn] = None,
|
|
1309
|
-
created_at: Union[datetime, None] = None,
|
|
1310
|
-
updated_by: Optional[CorpUserUrn] = None,
|
|
1311
|
-
updated_at: Optional[datetime] = None,
|
|
1312
|
-
):
|
|
1313
|
-
"""
|
|
1314
|
-
Initialize a SQL assertion.
|
|
1315
|
-
|
|
1316
|
-
Note: Values can be accessed, but not set on the assertion object.
|
|
1317
|
-
To update an assertion, use the `upsert_*` method.
|
|
1318
|
-
Args:
|
|
1319
|
-
urn: The urn of the assertion.
|
|
1320
|
-
dataset_urn: The urn of the dataset that the assertion is for.
|
|
1321
|
-
display_name: The display name of the assertion.
|
|
1322
|
-
mode: The mode of the assertion (active, inactive).
|
|
1323
|
-
statement: The SQL statement to be used for the assertion.
|
|
1324
|
-
criteria: The criteria to be used for the assertion.
|
|
1325
|
-
schedule: The schedule of the assertion.
|
|
1326
|
-
tags: The tags applied to the assertion.
|
|
1327
|
-
incident_behavior: Whether to raise or resolve an incident when the assertion fails / passes.
|
|
1328
|
-
created_by: The urn of the user that created the assertion.
|
|
1329
|
-
created_at: The timestamp of when the assertion was created.
|
|
1330
|
-
updated_by: The urn of the user that updated the assertion.
|
|
1331
|
-
updated_at: The timestamp of when the assertion was updated.
|
|
1332
|
-
"""
|
|
1333
|
-
# Initialize the mixins first
|
|
1334
|
-
_AssertionPublic.__init__(
|
|
1335
|
-
self,
|
|
1336
|
-
urn=urn,
|
|
1337
|
-
dataset_urn=dataset_urn,
|
|
1338
|
-
display_name=display_name,
|
|
1339
|
-
mode=mode,
|
|
1340
|
-
tags=tags,
|
|
1341
|
-
incident_behavior=incident_behavior,
|
|
1342
|
-
created_by=created_by,
|
|
1343
|
-
created_at=created_at,
|
|
1344
|
-
updated_by=updated_by,
|
|
1345
|
-
updated_at=updated_at,
|
|
1346
|
-
)
|
|
1347
|
-
_HasSchedule.__init__(self, schedule=schedule)
|
|
1348
|
-
# Then initialize the parent class
|
|
1349
|
-
self._statement = statement
|
|
1350
|
-
self._criteria = criteria
|
|
1351
|
-
|
|
1352
|
-
@property
|
|
1353
|
-
def statement(self) -> str:
|
|
1354
|
-
return self._statement
|
|
1355
|
-
|
|
1356
|
-
@property
|
|
1357
|
-
def criteria_condition(self) -> Union[SqlAssertionCondition, str]:
|
|
1358
|
-
return self._criteria.condition
|
|
1359
743
|
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
) -> Union[Union[float, int], tuple[Union[float, int], Union[float, int]]]:
|
|
1364
|
-
return self._criteria.parameters
|
|
744
|
+
return SmartVolumeAssertion
|
|
745
|
+
elif name == "VolumeAssertion":
|
|
746
|
+
from acryl_datahub_cloud.sdk.assertion.volume_assertion import VolumeAssertion
|
|
1365
747
|
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
assertion
|
|
1369
|
-
|
|
1370
|
-
default: Optional[_DetectionMechanismTypes] = DEFAULT_DETECTION_MECHANISM,
|
|
1371
|
-
) -> Optional[_DetectionMechanismTypes]:
|
|
1372
|
-
"""Sql assertions do not have a detection mechanism."""
|
|
1373
|
-
return None
|
|
1374
|
-
|
|
1375
|
-
@staticmethod
|
|
1376
|
-
def _get_statement(assertion: Assertion) -> str:
|
|
1377
|
-
if assertion.info is None:
|
|
1378
|
-
raise SDKNotYetSupportedError(
|
|
1379
|
-
f"Assertion {assertion.urn} does not have a SQL assertion info, which is not supported"
|
|
1380
|
-
)
|
|
1381
|
-
if isinstance(assertion.info, models.SqlAssertionInfoClass):
|
|
1382
|
-
return assertion.info.statement
|
|
1383
|
-
else:
|
|
1384
|
-
raise SDKNotYetSupportedError(
|
|
1385
|
-
f"Assertion {assertion.urn} is not a SQL assertion"
|
|
1386
|
-
)
|
|
1387
|
-
|
|
1388
|
-
@staticmethod
|
|
1389
|
-
def _get_condition_from_model_assertion_info(
|
|
1390
|
-
assertion_info: models.SqlAssertionInfoClass,
|
|
1391
|
-
) -> SqlAssertionCondition:
|
|
1392
|
-
"""Convert stored assertion info to condition enum."""
|
|
1393
|
-
# Handle value-based conditions (no change type)
|
|
1394
|
-
if str(assertion_info.type) == str(models.SqlAssertionTypeClass.METRIC):
|
|
1395
|
-
value_conditions = {
|
|
1396
|
-
str(
|
|
1397
|
-
models.AssertionStdOperatorClass.EQUAL_TO
|
|
1398
|
-
): SqlAssertionCondition.IS_EQUAL_TO,
|
|
1399
|
-
str(
|
|
1400
|
-
models.AssertionStdOperatorClass.NOT_EQUAL_TO
|
|
1401
|
-
): SqlAssertionCondition.IS_NOT_EQUAL_TO,
|
|
1402
|
-
str(
|
|
1403
|
-
models.AssertionStdOperatorClass.GREATER_THAN
|
|
1404
|
-
): SqlAssertionCondition.IS_GREATER_THAN,
|
|
1405
|
-
str(
|
|
1406
|
-
models.AssertionStdOperatorClass.LESS_THAN
|
|
1407
|
-
): SqlAssertionCondition.IS_LESS_THAN,
|
|
1408
|
-
str(
|
|
1409
|
-
models.AssertionStdOperatorClass.BETWEEN
|
|
1410
|
-
): SqlAssertionCondition.IS_WITHIN_A_RANGE,
|
|
1411
|
-
}
|
|
1412
|
-
if str(assertion_info.operator) in value_conditions:
|
|
1413
|
-
return value_conditions[str(assertion_info.operator)]
|
|
1414
|
-
|
|
1415
|
-
# Handle growth-based conditions (with change type)
|
|
1416
|
-
elif str(assertion_info.type) == str(
|
|
1417
|
-
models.SqlAssertionTypeClass.METRIC_CHANGE
|
|
1418
|
-
):
|
|
1419
|
-
assert assertion_info.changeType is not None, (
|
|
1420
|
-
"changeType must be present for METRIC_CHANGE assertions"
|
|
1421
|
-
)
|
|
1422
|
-
|
|
1423
|
-
growth_conditions = {
|
|
1424
|
-
(
|
|
1425
|
-
str(models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO),
|
|
1426
|
-
str(models.AssertionValueChangeTypeClass.ABSOLUTE),
|
|
1427
|
-
): SqlAssertionCondition.GROWS_AT_MOST_ABSOLUTE,
|
|
1428
|
-
(
|
|
1429
|
-
str(models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO),
|
|
1430
|
-
str(models.AssertionValueChangeTypeClass.PERCENTAGE),
|
|
1431
|
-
): SqlAssertionCondition.GROWS_AT_MOST_PERCENTAGE,
|
|
1432
|
-
(
|
|
1433
|
-
str(models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO),
|
|
1434
|
-
str(models.AssertionValueChangeTypeClass.ABSOLUTE),
|
|
1435
|
-
): SqlAssertionCondition.GROWS_AT_LEAST_ABSOLUTE,
|
|
1436
|
-
(
|
|
1437
|
-
str(models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO),
|
|
1438
|
-
str(models.AssertionValueChangeTypeClass.PERCENTAGE),
|
|
1439
|
-
): SqlAssertionCondition.GROWS_AT_LEAST_PERCENTAGE,
|
|
1440
|
-
(
|
|
1441
|
-
str(models.AssertionStdOperatorClass.BETWEEN),
|
|
1442
|
-
str(models.AssertionValueChangeTypeClass.ABSOLUTE),
|
|
1443
|
-
): SqlAssertionCondition.GROWS_WITHIN_A_RANGE_ABSOLUTE,
|
|
1444
|
-
(
|
|
1445
|
-
str(models.AssertionStdOperatorClass.BETWEEN),
|
|
1446
|
-
str(models.AssertionValueChangeTypeClass.PERCENTAGE),
|
|
1447
|
-
): SqlAssertionCondition.GROWS_WITHIN_A_RANGE_PERCENTAGE,
|
|
1448
|
-
}
|
|
1449
|
-
|
|
1450
|
-
key = (str(assertion_info.operator), str(assertion_info.changeType))
|
|
1451
|
-
if key in growth_conditions:
|
|
1452
|
-
return growth_conditions[key]
|
|
1453
|
-
|
|
1454
|
-
raise ValueError(
|
|
1455
|
-
f"Unsupported combination: type={assertion_info.type}, operator={assertion_info.operator}, changeType={assertion_info.changeType}"
|
|
748
|
+
return VolumeAssertion
|
|
749
|
+
elif name == "FreshnessAssertion":
|
|
750
|
+
from acryl_datahub_cloud.sdk.assertion.freshness_assertion import (
|
|
751
|
+
FreshnessAssertion,
|
|
1456
752
|
)
|
|
1457
753
|
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
)
|
|
1481
|
-
|
|
1482
|
-
condition = SqlAssertion._get_condition_from_model_assertion_info(
|
|
1483
|
-
assertion.info
|
|
1484
|
-
)
|
|
1485
|
-
|
|
1486
|
-
return SqlAssertionCriteria(
|
|
1487
|
-
condition=condition,
|
|
1488
|
-
parameters=parameters,
|
|
1489
|
-
)
|
|
1490
|
-
else:
|
|
1491
|
-
raise SDKNotYetSupportedError(
|
|
1492
|
-
f"Assertion {assertion.urn} is not a SQL assertion"
|
|
1493
|
-
)
|
|
1494
|
-
|
|
1495
|
-
@classmethod
|
|
1496
|
-
def _from_entities(cls, assertion: Assertion, monitor: Monitor) -> Self:
|
|
1497
|
-
"""
|
|
1498
|
-
Create a SQL assertion from the assertion and monitor entities.
|
|
1499
|
-
"""
|
|
1500
|
-
return cls(
|
|
1501
|
-
urn=assertion.urn,
|
|
1502
|
-
dataset_urn=assertion.dataset,
|
|
1503
|
-
display_name=assertion.description or "",
|
|
1504
|
-
mode=cls._get_mode(monitor),
|
|
1505
|
-
statement=cls._get_statement(assertion),
|
|
1506
|
-
criteria=cls._get_criteria(assertion),
|
|
1507
|
-
schedule=cls._get_schedule(
|
|
1508
|
-
monitor, default=DEFAULT_EVERY_SIX_HOURS_SCHEDULE
|
|
1509
|
-
),
|
|
1510
|
-
tags=cls._get_tags(assertion),
|
|
1511
|
-
incident_behavior=cls._get_incident_behavior(assertion),
|
|
1512
|
-
created_by=cls._get_created_by(assertion),
|
|
1513
|
-
created_at=cls._get_created_at(assertion),
|
|
1514
|
-
updated_by=cls._get_updated_by(assertion),
|
|
1515
|
-
updated_at=cls._get_updated_at(assertion),
|
|
1516
|
-
)
|
|
754
|
+
return FreshnessAssertion
|
|
755
|
+
elif name == "SqlAssertion":
|
|
756
|
+
from acryl_datahub_cloud.sdk.assertion.sql_assertion import SqlAssertion
|
|
757
|
+
|
|
758
|
+
return SqlAssertion
|
|
759
|
+
else:
|
|
760
|
+
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
# Export all classes for backward compatibility
|
|
764
|
+
__all__ = [
|
|
765
|
+
"AssertionMode",
|
|
766
|
+
"_HasSchedule",
|
|
767
|
+
"_HasSmartFunctionality",
|
|
768
|
+
"_HasColumnMetricFunctionality",
|
|
769
|
+
"_AssertionPublic",
|
|
770
|
+
"SmartFreshnessAssertion",
|
|
771
|
+
"SmartVolumeAssertion",
|
|
772
|
+
"VolumeAssertion",
|
|
773
|
+
"FreshnessAssertion",
|
|
774
|
+
"SqlAssertion",
|
|
775
|
+
]
|