acryl-datahub-cloud 0.3.10rc4__py3-none-any.whl → 0.3.16.1rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub-cloud might be problematic. Click here for more details.
- acryl_datahub_cloud/_codegen_config.json +1 -1
- acryl_datahub_cloud/acryl_cs_issues/acryl_customer.py +1 -1
- acryl_datahub_cloud/acryl_cs_issues/models.py +5 -3
- acryl_datahub_cloud/action_request/action_request_owner_source.py +37 -8
- acryl_datahub_cloud/datahub_forms_notifications/__init__.py +0 -0
- acryl_datahub_cloud/datahub_forms_notifications/forms_notifications_source.py +569 -0
- acryl_datahub_cloud/datahub_forms_notifications/get_feature_flag.gql +7 -0
- acryl_datahub_cloud/datahub_forms_notifications/get_search_results_total.gql +14 -0
- acryl_datahub_cloud/datahub_forms_notifications/query.py +17 -0
- acryl_datahub_cloud/datahub_forms_notifications/scroll_forms_for_notification.gql +29 -0
- acryl_datahub_cloud/datahub_forms_notifications/send_form_notification_request.gql +5 -0
- acryl_datahub_cloud/datahub_reporting/datahub_dataset.py +39 -19
- acryl_datahub_cloud/datahub_reporting/datahub_form_reporting.py +60 -25
- acryl_datahub_cloud/datahub_reporting/extract_graph.py +9 -3
- acryl_datahub_cloud/datahub_reporting/extract_sql.py +248 -52
- acryl_datahub_cloud/datahub_reporting/forms.py +1 -1
- acryl_datahub_cloud/datahub_reporting/forms_config.py +3 -2
- acryl_datahub_cloud/datahub_restore/source.py +3 -2
- acryl_datahub_cloud/datahub_usage_reporting/excluded.py +94 -0
- acryl_datahub_cloud/datahub_usage_reporting/query_builder.py +48 -8
- acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py +532 -109
- acryl_datahub_cloud/elasticsearch/graph_service.py +76 -14
- acryl_datahub_cloud/graphql_utils.py +64 -0
- acryl_datahub_cloud/lineage_features/source.py +555 -49
- acryl_datahub_cloud/metadata/_urns/urn_defs.py +2390 -1938
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/actionworkflow/__init__.py +53 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/anomaly/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/application/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/assertion/__init__.py +6 -2
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/common/__init__.py +6 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/conversation/__init__.py +29 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/event/notification/settings/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/execution/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/file/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/form/__init__.py +8 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/identity/__init__.py +8 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/knowledge/__init__.py +33 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/logical/__init__.py +15 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py +14 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/search/features/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/module/__init__.py +31 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/monitor/__init__.py +6 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/notification/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/platform/event/v1/__init__.py +4 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/role/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +28 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/template/__init__.py +31 -0
- acryl_datahub_cloud/metadata/schema.avsc +27843 -23200
- acryl_datahub_cloud/metadata/schema_classes.py +29901 -24310
- acryl_datahub_cloud/metadata/schemas/ActionRequestInfo.avsc +235 -2
- acryl_datahub_cloud/metadata/schemas/ActionWorkflowInfo.avsc +683 -0
- acryl_datahub_cloud/metadata/schemas/ActionWorkflowKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/Actors.avsc +38 -1
- acryl_datahub_cloud/metadata/schemas/ApplicationKey.avsc +31 -0
- acryl_datahub_cloud/metadata/schemas/ApplicationProperties.avsc +75 -0
- acryl_datahub_cloud/metadata/schemas/Applications.avsc +38 -0
- acryl_datahub_cloud/metadata/schemas/AssertionAnalyticsRunEvent.avsc +375 -212
- acryl_datahub_cloud/metadata/schemas/AssertionInfo.avsc +147 -20
- acryl_datahub_cloud/metadata/schemas/AssertionKey.avsc +1 -1
- acryl_datahub_cloud/metadata/schemas/AssertionRunEvent.avsc +191 -21
- acryl_datahub_cloud/metadata/schemas/{AssertionSummary.avsc → AssertionRunSummary.avsc} +15 -2
- acryl_datahub_cloud/metadata/schemas/AssertionsSummary.avsc +54 -0
- acryl_datahub_cloud/metadata/schemas/AssetSettings.avsc +63 -0
- acryl_datahub_cloud/metadata/schemas/BusinessAttributeInfo.avsc +7 -3
- acryl_datahub_cloud/metadata/schemas/ChartInfo.avsc +20 -6
- acryl_datahub_cloud/metadata/schemas/ChartKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/ConstraintInfo.avsc +12 -1
- acryl_datahub_cloud/metadata/schemas/ContainerKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/ContainerProperties.avsc +16 -5
- acryl_datahub_cloud/metadata/schemas/CorpGroupEditableInfo.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/CorpGroupInfo.avsc +7 -3
- acryl_datahub_cloud/metadata/schemas/CorpGroupKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/CorpGroupSettings.avsc +127 -2
- acryl_datahub_cloud/metadata/schemas/CorpUserEditableInfo.avsc +1 -1
- acryl_datahub_cloud/metadata/schemas/CorpUserInfo.avsc +18 -2
- acryl_datahub_cloud/metadata/schemas/CorpUserInvitationStatus.avsc +106 -0
- acryl_datahub_cloud/metadata/schemas/CorpUserKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/CorpUserSettings.avsc +304 -2
- acryl_datahub_cloud/metadata/schemas/CorpUserUsageFeatures.avsc +86 -0
- acryl_datahub_cloud/metadata/schemas/DashboardInfo.avsc +11 -5
- acryl_datahub_cloud/metadata/schemas/DashboardKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataContractKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/DataFlowInfo.avsc +15 -5
- acryl_datahub_cloud/metadata/schemas/DataFlowKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataHubAiConversationInfo.avsc +256 -0
- acryl_datahub_cloud/metadata/schemas/DataHubAiConversationKey.avsc +22 -0
- acryl_datahub_cloud/metadata/schemas/DataHubFileInfo.avsc +234 -0
- acryl_datahub_cloud/metadata/schemas/DataHubFileKey.avsc +22 -0
- acryl_datahub_cloud/metadata/schemas/DataHubIngestionSourceKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/DataHubOpenAPISchemaKey.avsc +22 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageModuleKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageModuleProperties.avsc +308 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageTemplateKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageTemplateProperties.avsc +251 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
- acryl_datahub_cloud/metadata/schemas/DataJobInfo.avsc +13 -4
- acryl_datahub_cloud/metadata/schemas/DataJobInputOutput.avsc +8 -0
- acryl_datahub_cloud/metadata/schemas/DataJobKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataPlatformInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/DataPlatformInstanceProperties.avsc +5 -2
- acryl_datahub_cloud/metadata/schemas/DataProcessKey.avsc +4 -0
- acryl_datahub_cloud/metadata/schemas/DataProductKey.avsc +2 -0
- acryl_datahub_cloud/metadata/schemas/DataProductProperties.avsc +6 -3
- acryl_datahub_cloud/metadata/schemas/DataTransformLogic.avsc +4 -2
- acryl_datahub_cloud/metadata/schemas/DataTypeInfo.avsc +5 -0
- acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc +10 -2
- acryl_datahub_cloud/metadata/schemas/DatasetProperties.avsc +12 -5
- acryl_datahub_cloud/metadata/schemas/DatasetUsageStatistics.avsc +8 -0
- acryl_datahub_cloud/metadata/schemas/DocumentInfo.avsc +407 -0
- acryl_datahub_cloud/metadata/schemas/DocumentKey.avsc +35 -0
- acryl_datahub_cloud/metadata/schemas/DocumentSettings.avsc +79 -0
- acryl_datahub_cloud/metadata/schemas/DomainKey.avsc +2 -0
- acryl_datahub_cloud/metadata/schemas/DomainProperties.avsc +7 -3
- acryl_datahub_cloud/metadata/schemas/EditableContainerProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDashboardProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDataFlowProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDataJobProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDatasetProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableERModelRelationshipProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLFeatureProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLFeatureTableProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLModelGroupProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLModelProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableNotebookProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableSchemaMetadata.avsc +4 -2
- acryl_datahub_cloud/metadata/schemas/EntityTypeInfo.avsc +5 -0
- acryl_datahub_cloud/metadata/schemas/ExecutionRequestArtifactsLocation.avsc +16 -0
- acryl_datahub_cloud/metadata/schemas/ExecutionRequestKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/FormAssignmentStatus.avsc +36 -0
- acryl_datahub_cloud/metadata/schemas/FormInfo.avsc +6 -0
- acryl_datahub_cloud/metadata/schemas/FormKey.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/FormNotifications.avsc +69 -0
- acryl_datahub_cloud/metadata/schemas/FormSettings.avsc +30 -0
- acryl_datahub_cloud/metadata/schemas/GlobalSettingsInfo.avsc +416 -0
- acryl_datahub_cloud/metadata/schemas/GlobalTags.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/GlossaryNodeInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/GlossaryNodeKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/GlossaryTermInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/GlossaryTermKey.avsc +2 -0
- acryl_datahub_cloud/metadata/schemas/IcebergWarehouseInfo.avsc +4 -0
- acryl_datahub_cloud/metadata/schemas/IncidentActivityEvent.avsc +3 -3
- acryl_datahub_cloud/metadata/schemas/IncidentInfo.avsc +3 -3
- acryl_datahub_cloud/metadata/schemas/InferredMetadata.avsc +71 -1
- acryl_datahub_cloud/metadata/schemas/InputFields.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/InviteToken.avsc +26 -0
- acryl_datahub_cloud/metadata/schemas/LineageFeatures.avsc +67 -42
- acryl_datahub_cloud/metadata/schemas/LogicalParent.avsc +145 -0
- acryl_datahub_cloud/metadata/schemas/MLFeatureKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/MLFeatureTableKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/MLModelDeploymentKey.avsc +7 -1
- acryl_datahub_cloud/metadata/schemas/MLModelDeploymentProperties.avsc +3 -0
- acryl_datahub_cloud/metadata/schemas/MLModelGroupKey.avsc +9 -1
- acryl_datahub_cloud/metadata/schemas/MLModelKey.avsc +9 -1
- acryl_datahub_cloud/metadata/schemas/MLModelProperties.avsc +4 -2
- acryl_datahub_cloud/metadata/schemas/MLPrimaryKeyKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc +424 -97
- acryl_datahub_cloud/metadata/schemas/MetadataChangeLog.avsc +65 -44
- acryl_datahub_cloud/metadata/schemas/MetadataChangeProposal.avsc +64 -0
- acryl_datahub_cloud/metadata/schemas/MonitorAnomalyEvent.avsc +84 -29
- acryl_datahub_cloud/metadata/schemas/MonitorInfo.avsc +221 -23
- acryl_datahub_cloud/metadata/schemas/MonitorKey.avsc +9 -1
- acryl_datahub_cloud/metadata/schemas/MonitorSuiteInfo.avsc +128 -3
- acryl_datahub_cloud/metadata/schemas/NotebookInfo.avsc +5 -2
- acryl_datahub_cloud/metadata/schemas/NotebookKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/NotificationRequest.avsc +91 -4
- acryl_datahub_cloud/metadata/schemas/Operation.avsc +17 -0
- acryl_datahub_cloud/metadata/schemas/Ownership.avsc +71 -1
- acryl_datahub_cloud/metadata/schemas/QueryProperties.avsc +4 -2
- acryl_datahub_cloud/metadata/schemas/QuerySubjects.avsc +2 -13
- acryl_datahub_cloud/metadata/schemas/RelationshipChangeEvent.avsc +215 -0
- acryl_datahub_cloud/metadata/schemas/RoleProperties.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/SchemaFieldInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/SchemaFieldKey.avsc +3 -0
- acryl_datahub_cloud/metadata/schemas/SchemaMetadata.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/SemanticContent.avsc +123 -0
- acryl_datahub_cloud/metadata/schemas/StructuredProperties.avsc +69 -0
- acryl_datahub_cloud/metadata/schemas/StructuredPropertyDefinition.avsc +15 -4
- acryl_datahub_cloud/metadata/schemas/StructuredPropertySettings.avsc +9 -0
- acryl_datahub_cloud/metadata/schemas/SubscriptionInfo.avsc +136 -5
- acryl_datahub_cloud/metadata/schemas/SubscriptionKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/SystemMetadata.avsc +147 -0
- acryl_datahub_cloud/metadata/schemas/TagProperties.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/TestInfo.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/UpstreamLineage.avsc +9 -0
- acryl_datahub_cloud/metadata/schemas/UsageFeatures.avsc +10 -0
- acryl_datahub_cloud/metadata/schemas/__init__.py +3 -3
- acryl_datahub_cloud/notifications/__init__.py +0 -0
- acryl_datahub_cloud/notifications/notification_recipient_builder.py +399 -0
- acryl_datahub_cloud/sdk/__init__.py +69 -0
- acryl_datahub_cloud/sdk/assertion/__init__.py +58 -0
- acryl_datahub_cloud/sdk/assertion/assertion_base.py +779 -0
- acryl_datahub_cloud/sdk/assertion/column_metric_assertion.py +191 -0
- acryl_datahub_cloud/sdk/assertion/column_value_assertion.py +431 -0
- acryl_datahub_cloud/sdk/assertion/freshness_assertion.py +201 -0
- acryl_datahub_cloud/sdk/assertion/schema_assertion.py +268 -0
- acryl_datahub_cloud/sdk/assertion/smart_column_metric_assertion.py +212 -0
- acryl_datahub_cloud/sdk/assertion/smart_freshness_assertion.py +165 -0
- acryl_datahub_cloud/sdk/assertion/smart_sql_assertion.py +156 -0
- acryl_datahub_cloud/sdk/assertion/smart_volume_assertion.py +162 -0
- acryl_datahub_cloud/sdk/assertion/sql_assertion.py +273 -0
- acryl_datahub_cloud/sdk/assertion/types.py +20 -0
- acryl_datahub_cloud/sdk/assertion/volume_assertion.py +156 -0
- acryl_datahub_cloud/sdk/assertion_client/__init__.py +0 -0
- acryl_datahub_cloud/sdk/assertion_client/column_metric.py +545 -0
- acryl_datahub_cloud/sdk/assertion_client/column_value.py +617 -0
- acryl_datahub_cloud/sdk/assertion_client/freshness.py +371 -0
- acryl_datahub_cloud/sdk/assertion_client/helpers.py +166 -0
- acryl_datahub_cloud/sdk/assertion_client/schema.py +358 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_column_metric.py +540 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_freshness.py +373 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_sql.py +411 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_volume.py +380 -0
- acryl_datahub_cloud/sdk/assertion_client/sql.py +410 -0
- acryl_datahub_cloud/sdk/assertion_client/volume.py +446 -0
- acryl_datahub_cloud/sdk/assertion_input/__init__.py +0 -0
- acryl_datahub_cloud/sdk/assertion_input/assertion_input.py +1470 -0
- acryl_datahub_cloud/sdk/assertion_input/column_assertion_constants.py +114 -0
- acryl_datahub_cloud/sdk/assertion_input/column_assertion_utils.py +284 -0
- acryl_datahub_cloud/sdk/assertion_input/column_metric_assertion_input.py +759 -0
- acryl_datahub_cloud/sdk/assertion_input/column_metric_constants.py +109 -0
- acryl_datahub_cloud/sdk/assertion_input/column_value_assertion_input.py +810 -0
- acryl_datahub_cloud/sdk/assertion_input/freshness_assertion_input.py +305 -0
- acryl_datahub_cloud/sdk/assertion_input/schema_assertion_input.py +413 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_column_metric_assertion_input.py +793 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_freshness_assertion_input.py +218 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_sql_assertion_input.py +181 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_volume_assertion_input.py +189 -0
- acryl_datahub_cloud/sdk/assertion_input/sql_assertion_input.py +320 -0
- acryl_datahub_cloud/sdk/assertion_input/volume_assertion_input.py +635 -0
- acryl_datahub_cloud/sdk/assertions_client.py +1074 -0
- acryl_datahub_cloud/sdk/entities/__init__.py +0 -0
- acryl_datahub_cloud/sdk/entities/assertion.py +439 -0
- acryl_datahub_cloud/sdk/entities/monitor.py +291 -0
- acryl_datahub_cloud/sdk/entities/subscription.py +100 -0
- acryl_datahub_cloud/sdk/errors.py +34 -0
- acryl_datahub_cloud/sdk/resolver_client.py +42 -0
- acryl_datahub_cloud/sdk/subscription_client.py +737 -0
- {acryl_datahub_cloud-0.3.10rc4.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/METADATA +49 -43
- {acryl_datahub_cloud-0.3.10rc4.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/RECORD +243 -145
- {acryl_datahub_cloud-0.3.10rc4.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/WHEEL +1 -1
- {acryl_datahub_cloud-0.3.10rc4.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/entry_points.txt +1 -0
- {acryl_datahub_cloud-0.3.10rc4.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,635 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from typing import Any, Optional, Tuple, Union
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Extra
|
|
6
|
+
|
|
7
|
+
from acryl_datahub_cloud.sdk.assertion_input.assertion_input import (
|
|
8
|
+
DEFAULT_DAILY_SCHEDULE,
|
|
9
|
+
AssertionIncidentBehaviorInputTypes,
|
|
10
|
+
DetectionMechanismInputTypes,
|
|
11
|
+
FieldSpecType,
|
|
12
|
+
_AssertionInput,
|
|
13
|
+
_DatasetProfile,
|
|
14
|
+
_InformationSchema,
|
|
15
|
+
_Query,
|
|
16
|
+
)
|
|
17
|
+
from acryl_datahub_cloud.sdk.entities.assertion import (
|
|
18
|
+
Assertion,
|
|
19
|
+
AssertionInfoInputType,
|
|
20
|
+
TagsInputType,
|
|
21
|
+
)
|
|
22
|
+
from acryl_datahub_cloud.sdk.errors import SDKNotYetSupportedError, SDKUsageError
|
|
23
|
+
from datahub.metadata import schema_classes as models
|
|
24
|
+
from datahub.metadata.urns import (
|
|
25
|
+
AssertionUrn,
|
|
26
|
+
CorpUserUrn,
|
|
27
|
+
DatasetUrn,
|
|
28
|
+
)
|
|
29
|
+
from datahub.sdk.entity_client import EntityClient
|
|
30
|
+
|
|
31
|
+
# Type aliases and enums for volume assertions
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class VolumeAssertionCondition(Enum):
|
|
35
|
+
"""Valid conditions for volume assertions combining type, operator, and change kind."""
|
|
36
|
+
|
|
37
|
+
# Row count total conditions
|
|
38
|
+
ROW_COUNT_IS_LESS_THAN_OR_EQUAL_TO = "ROW_COUNT_IS_LESS_THAN_OR_EQUAL_TO" # models.VolumeAssertionTypeClass.ROW_COUNT_TOTAL + models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO
|
|
39
|
+
ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO = "ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO" # models.VolumeAssertionTypeClass.ROW_COUNT_TOTAL + models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO
|
|
40
|
+
ROW_COUNT_IS_WITHIN_A_RANGE = "ROW_COUNT_IS_WITHIN_A_RANGE" # models.VolumeAssertionTypeClass.ROW_COUNT_TOTAL + models.AssertionStdOperatorClass.BETWEEN
|
|
41
|
+
|
|
42
|
+
# Row count change conditions - absolute
|
|
43
|
+
ROW_COUNT_GROWS_BY_AT_MOST_ABSOLUTE = "ROW_COUNT_GROWS_BY_AT_MOST_ABSOLUTE" # models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE + models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO + models.AssertionValueChangeTypeClass.ABSOLUTE
|
|
44
|
+
ROW_COUNT_GROWS_BY_AT_LEAST_ABSOLUTE = "ROW_COUNT_GROWS_BY_AT_LEAST_ABSOLUTE" # models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE + models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO + models.AssertionValueChangeTypeClass.ABSOLUTE
|
|
45
|
+
ROW_COUNT_GROWS_WITHIN_A_RANGE_ABSOLUTE = "ROW_COUNT_GROWS_WITHIN_A_RANGE_ABSOLUTE" # models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE + models.AssertionStdOperatorClass.BETWEEN + models.AssertionValueChangeTypeClass.ABSOLUTE
|
|
46
|
+
|
|
47
|
+
# Row count change conditions - percentage
|
|
48
|
+
ROW_COUNT_GROWS_BY_AT_MOST_PERCENTAGE = "ROW_COUNT_GROWS_BY_AT_MOST_PERCENTAGE" # models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE + models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO + models.AssertionValueChangeTypeClass.PERCENTAGE
|
|
49
|
+
ROW_COUNT_GROWS_BY_AT_LEAST_PERCENTAGE = "ROW_COUNT_GROWS_BY_AT_LEAST_PERCENTAGE" # models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE + models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO + models.AssertionValueChangeTypeClass.PERCENTAGE
|
|
50
|
+
ROW_COUNT_GROWS_WITHIN_A_RANGE_PERCENTAGE = "ROW_COUNT_GROWS_WITHIN_A_RANGE_PERCENTAGE" # models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE + models.AssertionStdOperatorClass.BETWEEN + models.AssertionValueChangeTypeClass.PERCENTAGE
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
VolumeAssertionDefinitionParameters = Union[float, Tuple[float, float]]
|
|
54
|
+
|
|
55
|
+
VolumeAssertionCriteriaInputTypes = Union[dict[str, Any], "VolumeAssertionCriteria"]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class VolumeAssertionCriteria(BaseModel):
|
|
59
|
+
condition: VolumeAssertionCondition
|
|
60
|
+
parameters: VolumeAssertionDefinitionParameters
|
|
61
|
+
|
|
62
|
+
class Config:
|
|
63
|
+
extra = Extra.forbid
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def parse(criteria: VolumeAssertionCriteriaInputTypes) -> "VolumeAssertionCriteria":
|
|
67
|
+
"""Parse and validate volume assertion criteria.
|
|
68
|
+
|
|
69
|
+
This method converts dictionary-based volume assertion criteria into typed volume
|
|
70
|
+
assertion objects, or validates already instantiated volume assertion objects. It
|
|
71
|
+
supports nine volume assertion conditions covering both total row count checks and
|
|
72
|
+
row count change monitoring with absolute or percentage thresholds.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
criteria: A volume assertion criteria that can be either:
|
|
76
|
+
- A dictionary containing volume assertion configuration with keys:
|
|
77
|
+
- condition: Must be one of the VolumeAssertionCondition enum values:
|
|
78
|
+
- "ROW_COUNT_IS_LESS_THAN_OR_EQUAL_TO": Total row count threshold (upper bound)
|
|
79
|
+
- "ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO": Total row count threshold (lower bound)
|
|
80
|
+
- "ROW_COUNT_IS_WITHIN_A_RANGE": Total row count within specified range
|
|
81
|
+
- "ROW_COUNT_GROWS_BY_AT_MOST_ABSOLUTE": Row count change upper bound (absolute)
|
|
82
|
+
- "ROW_COUNT_GROWS_BY_AT_LEAST_ABSOLUTE": Row count change lower bound (absolute)
|
|
83
|
+
- "ROW_COUNT_GROWS_WITHIN_A_RANGE_ABSOLUTE": Row count change within range (absolute)
|
|
84
|
+
- "ROW_COUNT_GROWS_BY_AT_MOST_PERCENTAGE": Row count change upper bound (percentage)
|
|
85
|
+
- "ROW_COUNT_GROWS_BY_AT_LEAST_PERCENTAGE": Row count change lower bound (percentage)
|
|
86
|
+
- "ROW_COUNT_GROWS_WITHIN_A_RANGE_PERCENTAGE": Row count change within range (percentage)
|
|
87
|
+
- parameters: Numeric threshold(s) for the condition:
|
|
88
|
+
- Single number (int/float) for single-bound conditions
|
|
89
|
+
- Tuple of two numbers for range conditions (WITHIN_A_RANGE)
|
|
90
|
+
- An already instantiated VolumeAssertionCriteria object
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
A validated VolumeAssertionCriteria object with the specified condition and parameters.
|
|
94
|
+
|
|
95
|
+
Raises:
|
|
96
|
+
SDKUsageError: If the criteria is invalid, including:
|
|
97
|
+
- Invalid input type (not dict or VolumeAssertionCriteria object)
|
|
98
|
+
- Missing required fields (condition or parameters)
|
|
99
|
+
- Invalid condition value (not in VolumeAssertionCondition enum)
|
|
100
|
+
- Invalid parameter structure for condition:
|
|
101
|
+
- Single-bound conditions require a single number
|
|
102
|
+
- Range conditions require a tuple of two numbers
|
|
103
|
+
- Parameters must be numeric (int or float)
|
|
104
|
+
- Parameter validation failures (negative values, invalid ranges)
|
|
105
|
+
|
|
106
|
+
Examples:
|
|
107
|
+
Parse a total row count assertion with single threshold:
|
|
108
|
+
>>> criteria = {
|
|
109
|
+
... "condition": "ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO",
|
|
110
|
+
... "parameters": 100
|
|
111
|
+
... }
|
|
112
|
+
>>> result = VolumeAssertionCriteria.parse(criteria)
|
|
113
|
+
>>> result.condition.value
|
|
114
|
+
"ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO"
|
|
115
|
+
|
|
116
|
+
Parse a row count change assertion with range:
|
|
117
|
+
>>> criteria = {
|
|
118
|
+
... "condition": "ROW_COUNT_GROWS_WITHIN_A_RANGE_PERCENTAGE",
|
|
119
|
+
... "parameters": (10.0, 50.0)
|
|
120
|
+
... }
|
|
121
|
+
>>> result = VolumeAssertionCriteria.parse(criteria)
|
|
122
|
+
>>> result.parameters
|
|
123
|
+
(10.0, 50.0)
|
|
124
|
+
|
|
125
|
+
Parse an already instantiated object:
|
|
126
|
+
>>> obj = VolumeAssertionCriteria(
|
|
127
|
+
... condition=VolumeAssertionCondition.ROW_COUNT_IS_LESS_THAN_OR_EQUAL_TO,
|
|
128
|
+
... parameters=200
|
|
129
|
+
... )
|
|
130
|
+
>>> result = VolumeAssertionCriteria.parse(obj)
|
|
131
|
+
>>> result == obj
|
|
132
|
+
True
|
|
133
|
+
"""
|
|
134
|
+
if isinstance(criteria, VolumeAssertionCriteria):
|
|
135
|
+
return criteria
|
|
136
|
+
|
|
137
|
+
if isinstance(criteria, dict):
|
|
138
|
+
condition = criteria.get("condition")
|
|
139
|
+
parameters = criteria.get("parameters")
|
|
140
|
+
|
|
141
|
+
if condition is None:
|
|
142
|
+
raise SDKUsageError(
|
|
143
|
+
"Volume assertion criteria must include a 'condition' field"
|
|
144
|
+
)
|
|
145
|
+
if parameters is None:
|
|
146
|
+
raise SDKUsageError(
|
|
147
|
+
"Volume assertion criteria must include a 'parameters' field"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
# Validate condition and parameters compatibility
|
|
151
|
+
VolumeAssertionCriteria._validate_condition_and_parameters(
|
|
152
|
+
condition, parameters
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
return VolumeAssertionCriteria(condition=condition, parameters=parameters)
|
|
156
|
+
|
|
157
|
+
raise SDKUsageError(
|
|
158
|
+
f"Volume assertion criteria must be a dict or VolumeAssertionCriteria object, got: {type(criteria)}"
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
@staticmethod
|
|
162
|
+
def build_model_volume_info(
|
|
163
|
+
criteria: "VolumeAssertionCriteria",
|
|
164
|
+
dataset_urn: str,
|
|
165
|
+
filter: Optional[models.DatasetFilterClass] = None,
|
|
166
|
+
) -> models.VolumeAssertionInfoClass:
|
|
167
|
+
"""Build a DataHub VolumeAssertionInfoClass from volume assertion criteria."""
|
|
168
|
+
condition = criteria.condition
|
|
169
|
+
parameters = criteria.parameters
|
|
170
|
+
|
|
171
|
+
# Convert condition to DataHub models based on condition type
|
|
172
|
+
if condition.value.startswith("ROW_COUNT_IS_"):
|
|
173
|
+
volume_info = VolumeAssertionCriteria._build_row_count_total_info(
|
|
174
|
+
condition, parameters, dataset_urn
|
|
175
|
+
)
|
|
176
|
+
elif condition.value.startswith("ROW_COUNT_GROWS_"):
|
|
177
|
+
volume_info = VolumeAssertionCriteria._build_row_count_change_info(
|
|
178
|
+
condition, parameters, dataset_urn
|
|
179
|
+
)
|
|
180
|
+
else:
|
|
181
|
+
raise SDKUsageError(f"Unsupported volume assertion condition: {condition}")
|
|
182
|
+
|
|
183
|
+
if filter is not None:
|
|
184
|
+
volume_info.filter = filter
|
|
185
|
+
return volume_info
|
|
186
|
+
|
|
187
|
+
@staticmethod
|
|
188
|
+
def _build_row_count_total_info(
|
|
189
|
+
condition: VolumeAssertionCondition,
|
|
190
|
+
parameters: VolumeAssertionDefinitionParameters,
|
|
191
|
+
dataset_urn: str,
|
|
192
|
+
) -> models.VolumeAssertionInfoClass:
|
|
193
|
+
"""Build VolumeAssertionInfoClass for row count total assertions."""
|
|
194
|
+
if condition.value.endswith("_WITHIN_A_RANGE"):
|
|
195
|
+
operator = models.AssertionStdOperatorClass.BETWEEN
|
|
196
|
+
elif condition.value.endswith("_LESS_THAN_OR_EQUAL_TO"):
|
|
197
|
+
operator = models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO
|
|
198
|
+
elif condition.value.endswith("_GREATER_THAN_OR_EQUAL_TO"):
|
|
199
|
+
operator = models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO
|
|
200
|
+
else:
|
|
201
|
+
raise SDKUsageError(f"Unknown row count condition: {condition}")
|
|
202
|
+
|
|
203
|
+
return models.VolumeAssertionInfoClass(
|
|
204
|
+
type=models.VolumeAssertionTypeClass.ROW_COUNT_TOTAL,
|
|
205
|
+
entity=dataset_urn,
|
|
206
|
+
rowCountTotal=models.RowCountTotalClass(
|
|
207
|
+
operator=operator,
|
|
208
|
+
parameters=VolumeAssertionCriteria._build_assertion_parameters(
|
|
209
|
+
operator, parameters
|
|
210
|
+
),
|
|
211
|
+
),
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
@staticmethod
|
|
215
|
+
def _build_row_count_change_info(
|
|
216
|
+
condition: VolumeAssertionCondition,
|
|
217
|
+
parameters: VolumeAssertionDefinitionParameters,
|
|
218
|
+
dataset_urn: str,
|
|
219
|
+
) -> models.VolumeAssertionInfoClass:
|
|
220
|
+
"""Build VolumeAssertionInfoClass for row count change assertions."""
|
|
221
|
+
# Determine operator
|
|
222
|
+
if condition.value.endswith(
|
|
223
|
+
"_WITHIN_A_RANGE_ABSOLUTE"
|
|
224
|
+
) or condition.value.endswith("_WITHIN_A_RANGE_PERCENTAGE"):
|
|
225
|
+
operator = models.AssertionStdOperatorClass.BETWEEN
|
|
226
|
+
elif condition.value.endswith("_AT_MOST_ABSOLUTE") or condition.value.endswith(
|
|
227
|
+
"_AT_MOST_PERCENTAGE"
|
|
228
|
+
):
|
|
229
|
+
operator = models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO
|
|
230
|
+
elif condition.value.endswith("_AT_LEAST_ABSOLUTE") or condition.value.endswith(
|
|
231
|
+
"_AT_LEAST_PERCENTAGE"
|
|
232
|
+
):
|
|
233
|
+
operator = models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO
|
|
234
|
+
else:
|
|
235
|
+
raise SDKUsageError(f"Unknown row count change condition: {condition}")
|
|
236
|
+
|
|
237
|
+
# Determine change type
|
|
238
|
+
if condition.value.endswith("_ABSOLUTE") or condition.value.endswith(
|
|
239
|
+
"_ABSOLUTE_WITHIN_A_RANGE"
|
|
240
|
+
):
|
|
241
|
+
change_type = models.AssertionValueChangeTypeClass.ABSOLUTE
|
|
242
|
+
elif condition.value.endswith("_PERCENTAGE") or condition.value.endswith(
|
|
243
|
+
"_PERCENTAGE_WITHIN_A_RANGE"
|
|
244
|
+
):
|
|
245
|
+
change_type = models.AssertionValueChangeTypeClass.PERCENTAGE
|
|
246
|
+
else:
|
|
247
|
+
raise SDKUsageError(
|
|
248
|
+
f"Cannot determine change type for condition: {condition}"
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
return models.VolumeAssertionInfoClass(
|
|
252
|
+
type=models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE,
|
|
253
|
+
entity=dataset_urn,
|
|
254
|
+
rowCountChange=models.RowCountChangeClass(
|
|
255
|
+
type=change_type,
|
|
256
|
+
operator=operator,
|
|
257
|
+
parameters=VolumeAssertionCriteria._build_assertion_parameters(
|
|
258
|
+
operator, parameters
|
|
259
|
+
),
|
|
260
|
+
),
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
@staticmethod
|
|
264
|
+
def from_assertion(assertion: Assertion) -> "VolumeAssertionCriteria":
|
|
265
|
+
"""Create volume assertion criteria from a DataHub assertion entity."""
|
|
266
|
+
VolumeAssertionCriteria._validate_assertion_info(assertion)
|
|
267
|
+
|
|
268
|
+
# Type narrowing: we know assertion.info is VolumeAssertionInfoClass after validation
|
|
269
|
+
assert isinstance(assertion.info, models.VolumeAssertionInfoClass)
|
|
270
|
+
|
|
271
|
+
if assertion.info.type == models.VolumeAssertionTypeClass.ROW_COUNT_TOTAL:
|
|
272
|
+
return VolumeAssertionCriteria._extract_row_count_total_criteria(assertion)
|
|
273
|
+
elif assertion.info.type == models.VolumeAssertionTypeClass.ROW_COUNT_CHANGE:
|
|
274
|
+
return VolumeAssertionCriteria._extract_row_count_change_criteria(assertion)
|
|
275
|
+
else:
|
|
276
|
+
raise SDKNotYetSupportedError(
|
|
277
|
+
f"Unsupported volume assertion type: {assertion.info.type}"
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
@staticmethod
|
|
281
|
+
def _validate_assertion_info(assertion: Assertion) -> None:
|
|
282
|
+
"""Validate that assertion has valid volume assertion info."""
|
|
283
|
+
if assertion.info is None:
|
|
284
|
+
raise SDKNotYetSupportedError(
|
|
285
|
+
f"Assertion {assertion.urn} does not have a volume assertion info, which is not supported"
|
|
286
|
+
)
|
|
287
|
+
if not isinstance(assertion.info, models.VolumeAssertionInfoClass):
|
|
288
|
+
raise SDKNotYetSupportedError(
|
|
289
|
+
f"Assertion {assertion.urn} is not a volume assertion"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
@staticmethod
|
|
293
|
+
def _extract_row_count_total_criteria(
|
|
294
|
+
assertion: Assertion,
|
|
295
|
+
) -> "VolumeAssertionCriteria":
|
|
296
|
+
"""Extract criteria from row count total assertion."""
|
|
297
|
+
# Type narrowing: we know assertion.info is VolumeAssertionInfoClass
|
|
298
|
+
assert isinstance(assertion.info, models.VolumeAssertionInfoClass)
|
|
299
|
+
|
|
300
|
+
if assertion.info.rowCountTotal is None:
|
|
301
|
+
raise SDKNotYetSupportedError(
|
|
302
|
+
f"Volume assertion {assertion.urn} has ROW_COUNT_TOTAL type but no rowCountTotal"
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
operator = assertion.info.rowCountTotal.operator
|
|
306
|
+
parameters = VolumeAssertionCriteria._extract_volume_parameters(
|
|
307
|
+
str(assertion.urn), str(operator), assertion.info.rowCountTotal.parameters
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
if operator == models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO:
|
|
311
|
+
condition = VolumeAssertionCondition.ROW_COUNT_IS_LESS_THAN_OR_EQUAL_TO
|
|
312
|
+
elif operator == models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO:
|
|
313
|
+
condition = VolumeAssertionCondition.ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO
|
|
314
|
+
elif operator == models.AssertionStdOperatorClass.BETWEEN:
|
|
315
|
+
condition = VolumeAssertionCondition.ROW_COUNT_IS_WITHIN_A_RANGE
|
|
316
|
+
else:
|
|
317
|
+
raise SDKNotYetSupportedError(
|
|
318
|
+
f"Unsupported operator for row count total: {operator}"
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
return VolumeAssertionCriteria(condition=condition, parameters=parameters)
|
|
322
|
+
|
|
323
|
+
@staticmethod
|
|
324
|
+
def _extract_row_count_change_criteria(
|
|
325
|
+
assertion: Assertion,
|
|
326
|
+
) -> "VolumeAssertionCriteria":
|
|
327
|
+
"""Extract criteria from row count change assertion."""
|
|
328
|
+
# Type narrowing: we know assertion.info is VolumeAssertionInfoClass
|
|
329
|
+
assert isinstance(assertion.info, models.VolumeAssertionInfoClass)
|
|
330
|
+
|
|
331
|
+
if assertion.info.rowCountChange is None:
|
|
332
|
+
raise SDKNotYetSupportedError(
|
|
333
|
+
f"Volume assertion {assertion.urn} has ROW_COUNT_CHANGE type but no rowCountChange"
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
operator = assertion.info.rowCountChange.operator
|
|
337
|
+
change_type = assertion.info.rowCountChange.type
|
|
338
|
+
parameters = VolumeAssertionCriteria._extract_volume_parameters(
|
|
339
|
+
str(assertion.urn), str(operator), assertion.info.rowCountChange.parameters
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
# Determine condition based on operator and change type
|
|
343
|
+
if operator == models.AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO:
|
|
344
|
+
condition = (
|
|
345
|
+
VolumeAssertionCondition.ROW_COUNT_GROWS_BY_AT_MOST_ABSOLUTE
|
|
346
|
+
if change_type == models.AssertionValueChangeTypeClass.ABSOLUTE
|
|
347
|
+
else VolumeAssertionCondition.ROW_COUNT_GROWS_BY_AT_MOST_PERCENTAGE
|
|
348
|
+
)
|
|
349
|
+
elif operator == models.AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO:
|
|
350
|
+
condition = (
|
|
351
|
+
VolumeAssertionCondition.ROW_COUNT_GROWS_BY_AT_LEAST_ABSOLUTE
|
|
352
|
+
if change_type == models.AssertionValueChangeTypeClass.ABSOLUTE
|
|
353
|
+
else VolumeAssertionCondition.ROW_COUNT_GROWS_BY_AT_LEAST_PERCENTAGE
|
|
354
|
+
)
|
|
355
|
+
elif operator == models.AssertionStdOperatorClass.BETWEEN:
|
|
356
|
+
condition = (
|
|
357
|
+
VolumeAssertionCondition.ROW_COUNT_GROWS_WITHIN_A_RANGE_ABSOLUTE
|
|
358
|
+
if change_type == models.AssertionValueChangeTypeClass.ABSOLUTE
|
|
359
|
+
else VolumeAssertionCondition.ROW_COUNT_GROWS_WITHIN_A_RANGE_PERCENTAGE
|
|
360
|
+
)
|
|
361
|
+
else:
|
|
362
|
+
raise SDKNotYetSupportedError(
|
|
363
|
+
f"Unsupported operator for row count change: {operator}"
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
return VolumeAssertionCriteria(condition=condition, parameters=parameters)
|
|
367
|
+
|
|
368
|
+
@staticmethod
|
|
369
|
+
def _extract_volume_parameters(
|
|
370
|
+
assertion_urn: str,
|
|
371
|
+
operator: str,
|
|
372
|
+
parameters: models.AssertionStdParametersClass,
|
|
373
|
+
) -> VolumeAssertionDefinitionParameters:
|
|
374
|
+
"""Extract parameters from assertion based on operator type."""
|
|
375
|
+
if operator == "BETWEEN":
|
|
376
|
+
if parameters.minValue is None or parameters.maxValue is None:
|
|
377
|
+
raise SDKNotYetSupportedError(
|
|
378
|
+
f"Volume assertion {assertion_urn} has BETWEEN operator but missing min/max values"
|
|
379
|
+
)
|
|
380
|
+
return (float(parameters.minValue.value), float(parameters.maxValue.value))
|
|
381
|
+
else:
|
|
382
|
+
if parameters.value is None:
|
|
383
|
+
raise SDKNotYetSupportedError(
|
|
384
|
+
f"Volume assertion {assertion_urn} has {operator} operator but missing value"
|
|
385
|
+
)
|
|
386
|
+
return float(parameters.value.value)
|
|
387
|
+
|
|
388
|
+
@staticmethod
|
|
389
|
+
def _validate_between_parameters(
|
|
390
|
+
parameters: VolumeAssertionDefinitionParameters, condition: str
|
|
391
|
+
) -> None:
|
|
392
|
+
"""Validate parameters for WITHIN_A_RANGE conditions."""
|
|
393
|
+
if not isinstance(parameters, tuple) or len(parameters) != 2:
|
|
394
|
+
raise SDKUsageError(
|
|
395
|
+
f"For WITHIN_A_RANGE condition {condition}, parameters must be a tuple of two numbers (min_value, max_value)."
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
@staticmethod
|
|
399
|
+
def _validate_single_value_parameters(
|
|
400
|
+
parameters: VolumeAssertionDefinitionParameters,
|
|
401
|
+
condition: str,
|
|
402
|
+
) -> None:
|
|
403
|
+
"""Validate parameters for single-value conditions."""
|
|
404
|
+
if not isinstance(parameters, (int, float)):
|
|
405
|
+
if isinstance(parameters, tuple):
|
|
406
|
+
raise SDKUsageError(
|
|
407
|
+
f"For condition {condition}, parameters must be a single number, not a tuple."
|
|
408
|
+
)
|
|
409
|
+
else:
|
|
410
|
+
raise SDKUsageError(
|
|
411
|
+
f"For condition {condition}, parameters must be a single number."
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
@staticmethod
|
|
415
|
+
def _parse_condition(
|
|
416
|
+
condition: Union[str, VolumeAssertionCondition],
|
|
417
|
+
) -> VolumeAssertionCondition:
|
|
418
|
+
"""Parse and validate condition input, converting string to enum if needed."""
|
|
419
|
+
if isinstance(condition, str):
|
|
420
|
+
try:
|
|
421
|
+
return VolumeAssertionCondition(condition)
|
|
422
|
+
except ValueError as e:
|
|
423
|
+
valid_conditions = ", ".join(
|
|
424
|
+
[cond.value for cond in VolumeAssertionCondition]
|
|
425
|
+
)
|
|
426
|
+
raise SDKUsageError(
|
|
427
|
+
f"Invalid condition '{condition}'. Valid conditions: {valid_conditions}"
|
|
428
|
+
) from e
|
|
429
|
+
return condition
|
|
430
|
+
|
|
431
|
+
@staticmethod
|
|
432
|
+
def _validate_condition_and_parameters(
|
|
433
|
+
condition: Union[str, VolumeAssertionCondition],
|
|
434
|
+
parameters: VolumeAssertionDefinitionParameters,
|
|
435
|
+
) -> None:
|
|
436
|
+
"""Validate that condition and parameters are compatible for volume assertions."""
|
|
437
|
+
condition_enum = VolumeAssertionCriteria._parse_condition(condition)
|
|
438
|
+
|
|
439
|
+
# Validate parameter structure based on condition
|
|
440
|
+
if (
|
|
441
|
+
condition_enum.value.endswith("_WITHIN_A_RANGE")
|
|
442
|
+
or condition_enum.value.endswith("_WITHIN_A_RANGE_ABSOLUTE")
|
|
443
|
+
or condition_enum.value.endswith("_WITHIN_A_RANGE_PERCENTAGE")
|
|
444
|
+
):
|
|
445
|
+
VolumeAssertionCriteria._validate_between_parameters(
|
|
446
|
+
parameters, condition_enum.value
|
|
447
|
+
)
|
|
448
|
+
else:
|
|
449
|
+
VolumeAssertionCriteria._validate_single_value_parameters(
|
|
450
|
+
parameters, condition_enum.value
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
@staticmethod
|
|
454
|
+
def _format_number_value(value: Union[int, float]) -> str:
|
|
455
|
+
"""Format number value for DataHub parameter strings.
|
|
456
|
+
|
|
457
|
+
Converts whole numbers to integers (100.0 -> "100") and keeps decimals (100.5 -> "100.5").
|
|
458
|
+
"""
|
|
459
|
+
if isinstance(value, float) and value.is_integer():
|
|
460
|
+
return str(int(value))
|
|
461
|
+
return str(value)
|
|
462
|
+
|
|
463
|
+
@staticmethod
|
|
464
|
+
def _build_assertion_parameters(
|
|
465
|
+
operator: str,
|
|
466
|
+
parameters: VolumeAssertionDefinitionParameters,
|
|
467
|
+
) -> models.AssertionStdParametersClass:
|
|
468
|
+
"""Build assertion parameters for DataHub model classes.
|
|
469
|
+
|
|
470
|
+
Args:
|
|
471
|
+
operator: The assertion operator (from models.AssertionStdOperatorClass).
|
|
472
|
+
parameters: The parameters (int for single value, tuple for BETWEEN).
|
|
473
|
+
|
|
474
|
+
Returns:
|
|
475
|
+
AssertionStdParametersClass with appropriate parameter structure.
|
|
476
|
+
"""
|
|
477
|
+
if operator == models.AssertionStdOperatorClass.BETWEEN:
|
|
478
|
+
assert isinstance(parameters, tuple) and len(parameters) == 2, (
|
|
479
|
+
f"BETWEEN operator requires tuple of two numbers, got: {parameters}"
|
|
480
|
+
)
|
|
481
|
+
# Sort values to ensure minValue is actually the minimum and maxValue is the maximum
|
|
482
|
+
min_val, max_val = sorted(parameters)
|
|
483
|
+
return models.AssertionStdParametersClass(
|
|
484
|
+
minValue=models.AssertionStdParameterClass(
|
|
485
|
+
value=VolumeAssertionCriteria._format_number_value(min_val),
|
|
486
|
+
type=models.AssertionStdParameterTypeClass.NUMBER,
|
|
487
|
+
),
|
|
488
|
+
maxValue=models.AssertionStdParameterClass(
|
|
489
|
+
value=VolumeAssertionCriteria._format_number_value(max_val),
|
|
490
|
+
type=models.AssertionStdParameterTypeClass.NUMBER,
|
|
491
|
+
),
|
|
492
|
+
)
|
|
493
|
+
else:
|
|
494
|
+
# Single value operators
|
|
495
|
+
assert isinstance(parameters, (int, float)), (
|
|
496
|
+
f"Single value operator {operator} requires number parameter, got: {parameters}"
|
|
497
|
+
)
|
|
498
|
+
return models.AssertionStdParametersClass(
|
|
499
|
+
value=models.AssertionStdParameterClass(
|
|
500
|
+
value=VolumeAssertionCriteria._format_number_value(parameters),
|
|
501
|
+
type=models.AssertionStdParameterTypeClass.NUMBER,
|
|
502
|
+
),
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
class _VolumeAssertionInput(_AssertionInput):
|
|
507
|
+
def __init__(
|
|
508
|
+
self,
|
|
509
|
+
*,
|
|
510
|
+
dataset_urn: Union[str, DatasetUrn],
|
|
511
|
+
entity_client: EntityClient, # Needed to get the schema field spec for the detection mechanism if needed
|
|
512
|
+
criteria: VolumeAssertionCriteriaInputTypes,
|
|
513
|
+
urn: Optional[Union[str, AssertionUrn]] = None,
|
|
514
|
+
display_name: Optional[str] = None,
|
|
515
|
+
enabled: bool = True,
|
|
516
|
+
schedule: Optional[Union[str, models.CronScheduleClass]] = None,
|
|
517
|
+
detection_mechanism: DetectionMechanismInputTypes = None,
|
|
518
|
+
incident_behavior: Optional[AssertionIncidentBehaviorInputTypes] = None,
|
|
519
|
+
tags: Optional[TagsInputType] = None,
|
|
520
|
+
created_by: Union[str, CorpUserUrn],
|
|
521
|
+
created_at: datetime,
|
|
522
|
+
updated_by: Union[str, CorpUserUrn],
|
|
523
|
+
updated_at: datetime,
|
|
524
|
+
):
|
|
525
|
+
_AssertionInput.__init__(
|
|
526
|
+
self,
|
|
527
|
+
dataset_urn=dataset_urn,
|
|
528
|
+
entity_client=entity_client,
|
|
529
|
+
urn=urn,
|
|
530
|
+
display_name=display_name,
|
|
531
|
+
enabled=enabled,
|
|
532
|
+
schedule=schedule,
|
|
533
|
+
detection_mechanism=detection_mechanism,
|
|
534
|
+
incident_behavior=incident_behavior,
|
|
535
|
+
tags=tags,
|
|
536
|
+
source_type=models.AssertionSourceTypeClass.NATIVE, # Native assertions are of type native, not inferred
|
|
537
|
+
created_by=created_by,
|
|
538
|
+
created_at=created_at,
|
|
539
|
+
updated_by=updated_by,
|
|
540
|
+
updated_at=updated_at,
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
self.criteria = VolumeAssertionCriteria.parse(criteria)
|
|
544
|
+
|
|
545
|
+
def _assertion_type(self) -> str:
|
|
546
|
+
return models.AssertionTypeClass.VOLUME
|
|
547
|
+
|
|
548
|
+
def _create_assertion_info(
|
|
549
|
+
self, filter: Optional[models.DatasetFilterClass]
|
|
550
|
+
) -> AssertionInfoInputType:
|
|
551
|
+
"""
|
|
552
|
+
Create a VolumeAssertionInfoClass for a volume assertion.
|
|
553
|
+
|
|
554
|
+
Args:
|
|
555
|
+
filter: Optional filter to apply to the assertion.
|
|
556
|
+
|
|
557
|
+
Returns:
|
|
558
|
+
A VolumeAssertionInfoClass configured for volume assertions.
|
|
559
|
+
"""
|
|
560
|
+
return VolumeAssertionCriteria.build_model_volume_info(
|
|
561
|
+
self.criteria, str(self.dataset_urn), filter
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
def _create_monitor_info(
|
|
565
|
+
self,
|
|
566
|
+
assertion_urn: AssertionUrn,
|
|
567
|
+
status: models.MonitorStatusClass,
|
|
568
|
+
schedule: models.CronScheduleClass,
|
|
569
|
+
) -> models.MonitorInfoClass:
|
|
570
|
+
"""
|
|
571
|
+
Create a MonitorInfoClass with all the necessary components.
|
|
572
|
+
"""
|
|
573
|
+
source_type, field = self._convert_assertion_source_type_and_field()
|
|
574
|
+
return models.MonitorInfoClass(
|
|
575
|
+
type=models.MonitorTypeClass.ASSERTION,
|
|
576
|
+
status=status,
|
|
577
|
+
assertionMonitor=models.AssertionMonitorClass(
|
|
578
|
+
assertions=[
|
|
579
|
+
models.AssertionEvaluationSpecClass(
|
|
580
|
+
assertion=str(assertion_urn),
|
|
581
|
+
schedule=schedule,
|
|
582
|
+
parameters=self._get_assertion_evaluation_parameters(
|
|
583
|
+
str(source_type), field
|
|
584
|
+
),
|
|
585
|
+
)
|
|
586
|
+
]
|
|
587
|
+
),
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
def _convert_schedule(self) -> models.CronScheduleClass:
|
|
591
|
+
"""Create a schedule for a volume assertion.
|
|
592
|
+
|
|
593
|
+
Returns:
|
|
594
|
+
A CronScheduleClass with appropriate schedule settings.
|
|
595
|
+
"""
|
|
596
|
+
if self.schedule is None:
|
|
597
|
+
return DEFAULT_DAILY_SCHEDULE
|
|
598
|
+
|
|
599
|
+
return models.CronScheduleClass(
|
|
600
|
+
cron=self.schedule.cron,
|
|
601
|
+
timezone=self.schedule.timezone,
|
|
602
|
+
)
|
|
603
|
+
|
|
604
|
+
def _get_assertion_evaluation_parameters(
|
|
605
|
+
self, source_type: str, field: Optional[FieldSpecType]
|
|
606
|
+
) -> models.AssertionEvaluationParametersClass:
|
|
607
|
+
return models.AssertionEvaluationParametersClass(
|
|
608
|
+
type=models.AssertionEvaluationParametersTypeClass.DATASET_VOLUME,
|
|
609
|
+
datasetVolumeParameters=models.DatasetVolumeAssertionParametersClass(
|
|
610
|
+
sourceType=source_type
|
|
611
|
+
),
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
def _convert_assertion_source_type_and_field(
|
|
615
|
+
self,
|
|
616
|
+
) -> tuple[str, Optional[FieldSpecType]]:
|
|
617
|
+
"""Convert the detection mechanism to source type and field."""
|
|
618
|
+
source_type = models.DatasetVolumeSourceTypeClass.INFORMATION_SCHEMA
|
|
619
|
+
field = None
|
|
620
|
+
|
|
621
|
+
if self.detection_mechanism is None:
|
|
622
|
+
return source_type, field
|
|
623
|
+
|
|
624
|
+
if isinstance(self.detection_mechanism, _Query):
|
|
625
|
+
source_type = models.DatasetVolumeSourceTypeClass.QUERY
|
|
626
|
+
elif isinstance(self.detection_mechanism, _InformationSchema):
|
|
627
|
+
source_type = models.DatasetVolumeSourceTypeClass.INFORMATION_SCHEMA
|
|
628
|
+
elif isinstance(self.detection_mechanism, _DatasetProfile):
|
|
629
|
+
source_type = models.DatasetVolumeSourceTypeClass.DATAHUB_DATASET_PROFILE
|
|
630
|
+
else:
|
|
631
|
+
raise SDKNotYetSupportedError(
|
|
632
|
+
f"Detection mechanism {self.detection_mechanism} not yet supported for volume assertions"
|
|
633
|
+
)
|
|
634
|
+
|
|
635
|
+
return source_type, field
|