acryl-datahub-cloud 0.3.11rc0__py3-none-any.whl → 0.3.16.1rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub-cloud might be problematic. Click here for more details.
- acryl_datahub_cloud/_codegen_config.json +1 -1
- acryl_datahub_cloud/acryl_cs_issues/models.py +5 -3
- acryl_datahub_cloud/action_request/action_request_owner_source.py +36 -6
- acryl_datahub_cloud/datahub_forms_notifications/__init__.py +0 -0
- acryl_datahub_cloud/datahub_forms_notifications/forms_notifications_source.py +569 -0
- acryl_datahub_cloud/datahub_forms_notifications/get_feature_flag.gql +7 -0
- acryl_datahub_cloud/datahub_forms_notifications/get_search_results_total.gql +14 -0
- acryl_datahub_cloud/datahub_forms_notifications/query.py +17 -0
- acryl_datahub_cloud/datahub_forms_notifications/scroll_forms_for_notification.gql +29 -0
- acryl_datahub_cloud/datahub_forms_notifications/send_form_notification_request.gql +5 -0
- acryl_datahub_cloud/datahub_reporting/datahub_dataset.py +37 -13
- acryl_datahub_cloud/datahub_reporting/datahub_form_reporting.py +55 -24
- acryl_datahub_cloud/datahub_reporting/extract_graph.py +4 -3
- acryl_datahub_cloud/datahub_reporting/extract_sql.py +242 -51
- acryl_datahub_cloud/datahub_reporting/forms.py +1 -1
- acryl_datahub_cloud/datahub_reporting/forms_config.py +3 -2
- acryl_datahub_cloud/datahub_restore/source.py +3 -2
- acryl_datahub_cloud/datahub_usage_reporting/excluded.py +94 -0
- acryl_datahub_cloud/datahub_usage_reporting/query_builder.py +48 -8
- acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py +518 -77
- acryl_datahub_cloud/elasticsearch/graph_service.py +76 -14
- acryl_datahub_cloud/graphql_utils.py +64 -0
- acryl_datahub_cloud/lineage_features/source.py +555 -49
- acryl_datahub_cloud/metadata/_urns/urn_defs.py +2296 -1900
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/actionworkflow/__init__.py +53 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/anomaly/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/application/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/assertion/__init__.py +4 -2
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/common/__init__.py +6 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/conversation/__init__.py +29 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/event/notification/settings/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/execution/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/file/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/form/__init__.py +8 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/identity/__init__.py +8 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/knowledge/__init__.py +33 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/logical/__init__.py +15 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py +12 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/search/features/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/module/__init__.py +31 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/notification/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/platform/event/v1/__init__.py +4 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/role/__init__.py +2 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py +19 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +28 -0
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/template/__init__.py +31 -0
- acryl_datahub_cloud/metadata/schema.avsc +25091 -20557
- acryl_datahub_cloud/metadata/schema_classes.py +29269 -23863
- acryl_datahub_cloud/metadata/schemas/ActionRequestInfo.avsc +235 -2
- acryl_datahub_cloud/metadata/schemas/ActionWorkflowInfo.avsc +683 -0
- acryl_datahub_cloud/metadata/schemas/ActionWorkflowKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/Actors.avsc +38 -1
- acryl_datahub_cloud/metadata/schemas/ApplicationKey.avsc +31 -0
- acryl_datahub_cloud/metadata/schemas/ApplicationProperties.avsc +75 -0
- acryl_datahub_cloud/metadata/schemas/Applications.avsc +38 -0
- acryl_datahub_cloud/metadata/schemas/AssertionAnalyticsRunEvent.avsc +353 -215
- acryl_datahub_cloud/metadata/schemas/AssertionInfo.avsc +147 -20
- acryl_datahub_cloud/metadata/schemas/AssertionKey.avsc +1 -1
- acryl_datahub_cloud/metadata/schemas/AssertionRunEvent.avsc +166 -21
- acryl_datahub_cloud/metadata/schemas/{AssertionSummary.avsc → AssertionRunSummary.avsc} +15 -2
- acryl_datahub_cloud/metadata/schemas/AssertionsSummary.avsc +54 -0
- acryl_datahub_cloud/metadata/schemas/AssetSettings.avsc +63 -0
- acryl_datahub_cloud/metadata/schemas/BusinessAttributeInfo.avsc +7 -3
- acryl_datahub_cloud/metadata/schemas/ChartInfo.avsc +20 -6
- acryl_datahub_cloud/metadata/schemas/ChartKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/ConstraintInfo.avsc +12 -1
- acryl_datahub_cloud/metadata/schemas/ContainerKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/ContainerProperties.avsc +16 -5
- acryl_datahub_cloud/metadata/schemas/CorpGroupEditableInfo.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/CorpGroupInfo.avsc +7 -3
- acryl_datahub_cloud/metadata/schemas/CorpGroupKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/CorpGroupSettings.avsc +127 -2
- acryl_datahub_cloud/metadata/schemas/CorpUserEditableInfo.avsc +1 -1
- acryl_datahub_cloud/metadata/schemas/CorpUserInfo.avsc +18 -2
- acryl_datahub_cloud/metadata/schemas/CorpUserInvitationStatus.avsc +106 -0
- acryl_datahub_cloud/metadata/schemas/CorpUserKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/CorpUserSettings.avsc +304 -2
- acryl_datahub_cloud/metadata/schemas/CorpUserUsageFeatures.avsc +86 -0
- acryl_datahub_cloud/metadata/schemas/DashboardInfo.avsc +11 -5
- acryl_datahub_cloud/metadata/schemas/DashboardKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataFlowInfo.avsc +15 -5
- acryl_datahub_cloud/metadata/schemas/DataFlowKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataHubAiConversationInfo.avsc +256 -0
- acryl_datahub_cloud/metadata/schemas/DataHubAiConversationKey.avsc +22 -0
- acryl_datahub_cloud/metadata/schemas/DataHubFileInfo.avsc +234 -0
- acryl_datahub_cloud/metadata/schemas/DataHubFileKey.avsc +22 -0
- acryl_datahub_cloud/metadata/schemas/DataHubIngestionSourceKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/DataHubPageModuleKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageModuleProperties.avsc +308 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageTemplateKey.avsc +21 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPageTemplateProperties.avsc +251 -0
- acryl_datahub_cloud/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
- acryl_datahub_cloud/metadata/schemas/DataJobInfo.avsc +13 -4
- acryl_datahub_cloud/metadata/schemas/DataJobInputOutput.avsc +8 -0
- acryl_datahub_cloud/metadata/schemas/DataJobKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/DataPlatformInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/DataPlatformInstanceProperties.avsc +5 -2
- acryl_datahub_cloud/metadata/schemas/DataProcessKey.avsc +4 -0
- acryl_datahub_cloud/metadata/schemas/DataProductKey.avsc +2 -0
- acryl_datahub_cloud/metadata/schemas/DataProductProperties.avsc +6 -3
- acryl_datahub_cloud/metadata/schemas/DataTypeInfo.avsc +5 -0
- acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc +10 -2
- acryl_datahub_cloud/metadata/schemas/DatasetProperties.avsc +12 -5
- acryl_datahub_cloud/metadata/schemas/DatasetUsageStatistics.avsc +8 -0
- acryl_datahub_cloud/metadata/schemas/DocumentInfo.avsc +407 -0
- acryl_datahub_cloud/metadata/schemas/DocumentKey.avsc +35 -0
- acryl_datahub_cloud/metadata/schemas/DocumentSettings.avsc +79 -0
- acryl_datahub_cloud/metadata/schemas/DomainKey.avsc +2 -0
- acryl_datahub_cloud/metadata/schemas/DomainProperties.avsc +7 -3
- acryl_datahub_cloud/metadata/schemas/EditableContainerProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDashboardProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDataFlowProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDataJobProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableDatasetProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableERModelRelationshipProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLFeatureProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLFeatureTableProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLModelGroupProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableMLModelProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableNotebookProperties.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/EditableSchemaMetadata.avsc +4 -2
- acryl_datahub_cloud/metadata/schemas/EntityTypeInfo.avsc +5 -0
- acryl_datahub_cloud/metadata/schemas/ExecutionRequestArtifactsLocation.avsc +16 -0
- acryl_datahub_cloud/metadata/schemas/ExecutionRequestKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/FormAssignmentStatus.avsc +36 -0
- acryl_datahub_cloud/metadata/schemas/FormInfo.avsc +6 -0
- acryl_datahub_cloud/metadata/schemas/FormKey.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/FormNotifications.avsc +69 -0
- acryl_datahub_cloud/metadata/schemas/FormSettings.avsc +30 -0
- acryl_datahub_cloud/metadata/schemas/GlobalSettingsInfo.avsc +416 -0
- acryl_datahub_cloud/metadata/schemas/GlobalTags.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/GlossaryNodeInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/GlossaryNodeKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/GlossaryTermInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/GlossaryTermKey.avsc +2 -0
- acryl_datahub_cloud/metadata/schemas/IcebergWarehouseInfo.avsc +4 -0
- acryl_datahub_cloud/metadata/schemas/IncidentActivityEvent.avsc +3 -3
- acryl_datahub_cloud/metadata/schemas/IncidentInfo.avsc +3 -3
- acryl_datahub_cloud/metadata/schemas/InferredMetadata.avsc +71 -1
- acryl_datahub_cloud/metadata/schemas/InputFields.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/InviteToken.avsc +26 -0
- acryl_datahub_cloud/metadata/schemas/LineageFeatures.avsc +67 -42
- acryl_datahub_cloud/metadata/schemas/LogicalParent.avsc +145 -0
- acryl_datahub_cloud/metadata/schemas/MLFeatureKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/MLFeatureTableKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/MLModelDeploymentKey.avsc +7 -1
- acryl_datahub_cloud/metadata/schemas/MLModelGroupKey.avsc +9 -1
- acryl_datahub_cloud/metadata/schemas/MLModelKey.avsc +9 -1
- acryl_datahub_cloud/metadata/schemas/MLModelProperties.avsc +4 -2
- acryl_datahub_cloud/metadata/schemas/MLPrimaryKeyKey.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc +418 -97
- acryl_datahub_cloud/metadata/schemas/MetadataChangeLog.avsc +62 -44
- acryl_datahub_cloud/metadata/schemas/MetadataChangeProposal.avsc +61 -0
- acryl_datahub_cloud/metadata/schemas/MonitorAnomalyEvent.avsc +54 -9
- acryl_datahub_cloud/metadata/schemas/MonitorInfo.avsc +163 -23
- acryl_datahub_cloud/metadata/schemas/MonitorKey.avsc +9 -1
- acryl_datahub_cloud/metadata/schemas/MonitorSuiteInfo.avsc +128 -3
- acryl_datahub_cloud/metadata/schemas/NotebookInfo.avsc +5 -2
- acryl_datahub_cloud/metadata/schemas/NotebookKey.avsc +1 -0
- acryl_datahub_cloud/metadata/schemas/NotificationRequest.avsc +91 -4
- acryl_datahub_cloud/metadata/schemas/Operation.avsc +17 -0
- acryl_datahub_cloud/metadata/schemas/Ownership.avsc +71 -1
- acryl_datahub_cloud/metadata/schemas/QuerySubjects.avsc +2 -13
- acryl_datahub_cloud/metadata/schemas/RelationshipChangeEvent.avsc +215 -0
- acryl_datahub_cloud/metadata/schemas/RoleProperties.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/SchemaFieldInfo.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/SchemaFieldKey.avsc +3 -0
- acryl_datahub_cloud/metadata/schemas/SchemaMetadata.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/SemanticContent.avsc +123 -0
- acryl_datahub_cloud/metadata/schemas/StructuredProperties.avsc +69 -0
- acryl_datahub_cloud/metadata/schemas/StructuredPropertyDefinition.avsc +15 -4
- acryl_datahub_cloud/metadata/schemas/StructuredPropertySettings.avsc +9 -0
- acryl_datahub_cloud/metadata/schemas/SubscriptionInfo.avsc +136 -5
- acryl_datahub_cloud/metadata/schemas/SubscriptionKey.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/SystemMetadata.avsc +61 -0
- acryl_datahub_cloud/metadata/schemas/TagProperties.avsc +3 -1
- acryl_datahub_cloud/metadata/schemas/TestInfo.avsc +2 -1
- acryl_datahub_cloud/metadata/schemas/UpstreamLineage.avsc +9 -0
- acryl_datahub_cloud/metadata/schemas/UsageFeatures.avsc +10 -0
- acryl_datahub_cloud/notifications/__init__.py +0 -0
- acryl_datahub_cloud/notifications/notification_recipient_builder.py +399 -0
- acryl_datahub_cloud/sdk/__init__.py +69 -0
- acryl_datahub_cloud/sdk/assertion/__init__.py +58 -0
- acryl_datahub_cloud/sdk/assertion/assertion_base.py +779 -0
- acryl_datahub_cloud/sdk/assertion/column_metric_assertion.py +191 -0
- acryl_datahub_cloud/sdk/assertion/column_value_assertion.py +431 -0
- acryl_datahub_cloud/sdk/assertion/freshness_assertion.py +201 -0
- acryl_datahub_cloud/sdk/assertion/schema_assertion.py +268 -0
- acryl_datahub_cloud/sdk/assertion/smart_column_metric_assertion.py +212 -0
- acryl_datahub_cloud/sdk/assertion/smart_freshness_assertion.py +165 -0
- acryl_datahub_cloud/sdk/assertion/smart_sql_assertion.py +156 -0
- acryl_datahub_cloud/sdk/assertion/smart_volume_assertion.py +162 -0
- acryl_datahub_cloud/sdk/assertion/sql_assertion.py +273 -0
- acryl_datahub_cloud/sdk/assertion/types.py +20 -0
- acryl_datahub_cloud/sdk/assertion/volume_assertion.py +156 -0
- acryl_datahub_cloud/sdk/assertion_client/__init__.py +0 -0
- acryl_datahub_cloud/sdk/assertion_client/column_metric.py +545 -0
- acryl_datahub_cloud/sdk/assertion_client/column_value.py +617 -0
- acryl_datahub_cloud/sdk/assertion_client/freshness.py +371 -0
- acryl_datahub_cloud/sdk/assertion_client/helpers.py +166 -0
- acryl_datahub_cloud/sdk/assertion_client/schema.py +358 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_column_metric.py +540 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_freshness.py +373 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_sql.py +411 -0
- acryl_datahub_cloud/sdk/assertion_client/smart_volume.py +380 -0
- acryl_datahub_cloud/sdk/assertion_client/sql.py +410 -0
- acryl_datahub_cloud/sdk/assertion_client/volume.py +446 -0
- acryl_datahub_cloud/sdk/assertion_input/__init__.py +0 -0
- acryl_datahub_cloud/sdk/assertion_input/assertion_input.py +1470 -0
- acryl_datahub_cloud/sdk/assertion_input/column_assertion_constants.py +114 -0
- acryl_datahub_cloud/sdk/assertion_input/column_assertion_utils.py +284 -0
- acryl_datahub_cloud/sdk/assertion_input/column_metric_assertion_input.py +759 -0
- acryl_datahub_cloud/sdk/assertion_input/column_metric_constants.py +109 -0
- acryl_datahub_cloud/sdk/assertion_input/column_value_assertion_input.py +810 -0
- acryl_datahub_cloud/sdk/assertion_input/freshness_assertion_input.py +305 -0
- acryl_datahub_cloud/sdk/assertion_input/schema_assertion_input.py +413 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_column_metric_assertion_input.py +793 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_freshness_assertion_input.py +218 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_sql_assertion_input.py +181 -0
- acryl_datahub_cloud/sdk/assertion_input/smart_volume_assertion_input.py +189 -0
- acryl_datahub_cloud/sdk/assertion_input/sql_assertion_input.py +320 -0
- acryl_datahub_cloud/sdk/assertion_input/volume_assertion_input.py +635 -0
- acryl_datahub_cloud/sdk/assertions_client.py +1074 -0
- acryl_datahub_cloud/sdk/entities/__init__.py +0 -0
- acryl_datahub_cloud/sdk/entities/assertion.py +439 -0
- acryl_datahub_cloud/sdk/entities/monitor.py +291 -0
- acryl_datahub_cloud/sdk/entities/subscription.py +100 -0
- acryl_datahub_cloud/sdk/errors.py +34 -0
- acryl_datahub_cloud/sdk/resolver_client.py +42 -0
- acryl_datahub_cloud/sdk/subscription_client.py +737 -0
- {acryl_datahub_cloud-0.3.11rc0.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/METADATA +55 -49
- {acryl_datahub_cloud-0.3.11rc0.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/RECORD +235 -142
- {acryl_datahub_cloud-0.3.11rc0.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/WHEEL +1 -1
- {acryl_datahub_cloud-0.3.11rc0.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/entry_points.txt +1 -0
- acryl_datahub_cloud/_sdk_extras/__init__.py +0 -4
- acryl_datahub_cloud/_sdk_extras/assertion.py +0 -15
- acryl_datahub_cloud/_sdk_extras/assertions_client.py +0 -23
- {acryl_datahub_cloud-0.3.11rc0.dist-info → acryl_datahub_cloud-0.3.16.1rc0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,737 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
from typing import Dict, List, Optional, Sequence, Tuple, Union
|
|
4
|
+
|
|
5
|
+
from typing_extensions import TypeAlias
|
|
6
|
+
|
|
7
|
+
import datahub.metadata.schema_classes as models
|
|
8
|
+
from acryl_datahub_cloud.sdk.entities.assertion import Assertion
|
|
9
|
+
from acryl_datahub_cloud.sdk.entities.subscription import Subscription, SubscriptionKey
|
|
10
|
+
from datahub.emitter.enum_helpers import get_enum_options
|
|
11
|
+
from datahub.emitter.mce_builder import make_ts_millis
|
|
12
|
+
from datahub.emitter.rest_emitter import EmitMode
|
|
13
|
+
from datahub.errors import SdkUsageError
|
|
14
|
+
from datahub.metadata.urns import AssertionUrn, CorpGroupUrn, CorpUserUrn, DatasetUrn
|
|
15
|
+
from datahub.sdk._utils import DEFAULT_ACTOR_URN
|
|
16
|
+
from datahub.sdk.main_client import DataHubClient
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
SubscriberInputType: TypeAlias = Union[CorpUserUrn, CorpGroupUrn, str]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
ASSERTION_RELATED_ENTITY_CHANGE_TYPES = {
|
|
24
|
+
models.EntityChangeTypeClass.ASSERTION_PASSED,
|
|
25
|
+
models.EntityChangeTypeClass.ASSERTION_FAILED,
|
|
26
|
+
models.EntityChangeTypeClass.ASSERTION_ERROR,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
ALL_EXISTING_ENTITY_CHANGE_TYPES = {
|
|
30
|
+
getattr(models.EntityChangeTypeClass, attr)
|
|
31
|
+
for attr in dir(models.EntityChangeTypeClass)
|
|
32
|
+
if not attr.startswith("_")
|
|
33
|
+
and isinstance(getattr(models.EntityChangeTypeClass, attr), str)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SubscriptionClient:
|
|
38
|
+
"""
|
|
39
|
+
A client for managing subscriptions to entity changes in Acryl DataHub Cloud.
|
|
40
|
+
|
|
41
|
+
Subscriptions can be created at two granularity levels for different entity change types:
|
|
42
|
+
- Dataset level: Affects all assertions associated with the dataset
|
|
43
|
+
- Assertion level: Affects only the specific assertion and overrides any dataset-level subscriptions
|
|
44
|
+
|
|
45
|
+
Notes:
|
|
46
|
+
- This implementation currently returns low-level Subscription entities from the entities
|
|
47
|
+
submodule. In future versions, this may be replaced with a higher-level Subscription abstraction
|
|
48
|
+
for improved usability.
|
|
49
|
+
- The client is designed to work with both datasets and assertions, but currently only supports
|
|
50
|
+
datasets. Assertions will be supported in future versions.
|
|
51
|
+
- Only ENTITY_CHANGE subscription types is supported.
|
|
52
|
+
- No notificationConfig is set
|
|
53
|
+
- This client is experimental and under heavy development. Expect breaking changes.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, client: DataHubClient):
|
|
57
|
+
self.client = client
|
|
58
|
+
_print_experimental_warning()
|
|
59
|
+
|
|
60
|
+
def subscribe(
|
|
61
|
+
self,
|
|
62
|
+
*,
|
|
63
|
+
urn: Union[str, DatasetUrn, AssertionUrn],
|
|
64
|
+
subscriber_urn: SubscriberInputType,
|
|
65
|
+
entity_change_types: Optional[
|
|
66
|
+
Sequence[Union[str, models.EntityChangeTypeClass]]
|
|
67
|
+
] = None,
|
|
68
|
+
skip_actor_exists_check: bool = False,
|
|
69
|
+
) -> None:
|
|
70
|
+
"""
|
|
71
|
+
Create a subscription to receive notifications for entity changes.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
urn: The URN (string or URN object) of the dataset or assertion to subscribe to.
|
|
75
|
+
For datasets: subscription applies to all assertions on the dataset.
|
|
76
|
+
For assertions: subscription applies only to that specific assertion.
|
|
77
|
+
subscriber_urn: The URN of the user or group that will receive notifications.
|
|
78
|
+
Can be a string (valid corpuser or corpGroup URN) or URN object.
|
|
79
|
+
entity_change_types: Specific change types to subscribe to. If None, defaults are:
|
|
80
|
+
- Dataset: all existing change types
|
|
81
|
+
- Assertion: assertion-related types (ASSERTION_PASSED,
|
|
82
|
+
ASSERTION_FAILED, ASSERTION_ERROR)
|
|
83
|
+
skip_actor_exists_check: If True, skip validation that the subscriber entity exists.
|
|
84
|
+
This can be useful in cases where eventual consistency may
|
|
85
|
+
cause the subscriber to not be immediately available.
|
|
86
|
+
Defaults to False.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
None
|
|
90
|
+
|
|
91
|
+
Raises:
|
|
92
|
+
SdkUsageError: If URN format is invalid, entity not found, or empty change types list.
|
|
93
|
+
SdkUsageError: For assertion subscription - if non-assertion-related change types
|
|
94
|
+
are provided (only ASSERTION_PASSED, ASSERTION_FAILED, ASSERTION_ERROR allowed).
|
|
95
|
+
SdkUsageError: If skip_actor_exists_check is False and the subscriber does not exist.
|
|
96
|
+
"""
|
|
97
|
+
_print_experimental_warning()
|
|
98
|
+
|
|
99
|
+
# Parse URN string if needed
|
|
100
|
+
parsed_urn = self._maybe_parse_urn(urn)
|
|
101
|
+
|
|
102
|
+
# Parse subscriber URN string if needed
|
|
103
|
+
parsed_subscriber_urn = self._maybe_parse_subscriber_urn(subscriber_urn)
|
|
104
|
+
|
|
105
|
+
# Verify that the subscriber (user or group) exists (unless skipped)
|
|
106
|
+
if not skip_actor_exists_check and not self.client._graph.exists(
|
|
107
|
+
parsed_subscriber_urn.urn()
|
|
108
|
+
):
|
|
109
|
+
raise SdkUsageError(f"Subscriber not found: {parsed_subscriber_urn.urn()}")
|
|
110
|
+
|
|
111
|
+
dataset_urn: DatasetUrn
|
|
112
|
+
assertion_urn: Optional[AssertionUrn]
|
|
113
|
+
dataset_urn, assertion_urn = (
|
|
114
|
+
(parsed_urn, None)
|
|
115
|
+
if isinstance(parsed_urn, DatasetUrn)
|
|
116
|
+
else self._fetch_dataset_from_assertion(parsed_urn)
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
logger.info(
|
|
120
|
+
f"Subscribing to dataset={dataset_urn} assertion={assertion_urn} for subscriber={parsed_subscriber_urn} with change types: {entity_change_types}"
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
# Get entity change types (use all if none provided)
|
|
124
|
+
entity_change_type_strs = self._get_entity_change_types(
|
|
125
|
+
assertion_scope=assertion_urn is not None,
|
|
126
|
+
entity_change_types=entity_change_types,
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
existing_subscriptions = self.client.resolve.subscription( # type: ignore[attr-defined]
|
|
130
|
+
entity_urn=dataset_urn.urn(),
|
|
131
|
+
actor_urn=parsed_subscriber_urn.urn(),
|
|
132
|
+
skip_cache=True,
|
|
133
|
+
)
|
|
134
|
+
if not existing_subscriptions:
|
|
135
|
+
# new subscription - use stable ID generation
|
|
136
|
+
subscription_key = SubscriptionKey(
|
|
137
|
+
entity_urn=dataset_urn.urn(),
|
|
138
|
+
actor_urn=parsed_subscriber_urn.urn(),
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
subscription = Subscription(
|
|
142
|
+
id=f"urn:li:subscription:{subscription_key.guid()}",
|
|
143
|
+
info=models.SubscriptionInfoClass(
|
|
144
|
+
entityUrn=dataset_urn.urn(),
|
|
145
|
+
actorUrn=parsed_subscriber_urn.urn(),
|
|
146
|
+
actorType=CorpUserUrn.ENTITY_TYPE
|
|
147
|
+
if isinstance(parsed_subscriber_urn, CorpUserUrn)
|
|
148
|
+
else CorpGroupUrn.ENTITY_TYPE,
|
|
149
|
+
types=[
|
|
150
|
+
models.SubscriptionTypeClass.ENTITY_CHANGE,
|
|
151
|
+
],
|
|
152
|
+
entityChangeTypes=self._merge_entity_change_types(
|
|
153
|
+
existing_change_types=None,
|
|
154
|
+
new_change_type_strs=entity_change_type_strs,
|
|
155
|
+
new_assertion_urn=assertion_urn,
|
|
156
|
+
),
|
|
157
|
+
createdOn=self._create_audit_stamp(),
|
|
158
|
+
updatedOn=self._create_audit_stamp(),
|
|
159
|
+
),
|
|
160
|
+
)
|
|
161
|
+
self.client.entities.upsert(subscription, emit_mode=EmitMode.SYNC_WAIT)
|
|
162
|
+
logger.info(f"Subscription created: {subscription.urn}")
|
|
163
|
+
return
|
|
164
|
+
elif len(existing_subscriptions) == 1:
|
|
165
|
+
# update existing subscription
|
|
166
|
+
subscription_urn = existing_subscriptions[0]
|
|
167
|
+
existing_subscription_entity = self.client.entities.get(subscription_urn)
|
|
168
|
+
assert isinstance(existing_subscription_entity, Subscription), (
|
|
169
|
+
f"Expected Subscription entity type for subscription urn={subscription_urn}"
|
|
170
|
+
)
|
|
171
|
+
logger.info(
|
|
172
|
+
f"Found existing subscription to be updated: {existing_subscription_entity.urn}"
|
|
173
|
+
)
|
|
174
|
+
existing_subscription_entity.info.entityChangeTypes = self._merge_entity_change_types(
|
|
175
|
+
existing_change_types=existing_subscription_entity.info.entityChangeTypes,
|
|
176
|
+
new_change_type_strs=entity_change_type_strs,
|
|
177
|
+
new_assertion_urn=assertion_urn,
|
|
178
|
+
)
|
|
179
|
+
existing_subscription_entity.info.updatedOn = self._create_audit_stamp()
|
|
180
|
+
self.client.entities.upsert(
|
|
181
|
+
existing_subscription_entity, emit_mode=EmitMode.SYNC_WAIT
|
|
182
|
+
)
|
|
183
|
+
logger.info(f"Subscription updated: {existing_subscription_entity.urn}")
|
|
184
|
+
return
|
|
185
|
+
else:
|
|
186
|
+
raise SdkUsageError(
|
|
187
|
+
f"We have a mesh here - {len(existing_subscriptions)} subscriptions found for dataset={dataset_urn} assertion={assertion_urn} and subscriber={parsed_subscriber_urn}!"
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
def list_subscriptions(
|
|
191
|
+
self,
|
|
192
|
+
*,
|
|
193
|
+
urn: Union[str, DatasetUrn, AssertionUrn],
|
|
194
|
+
entity_change_types: Optional[
|
|
195
|
+
Sequence[Union[str, models.EntityChangeTypeClass]]
|
|
196
|
+
] = None,
|
|
197
|
+
subscriber_urn: Optional[SubscriberInputType] = None,
|
|
198
|
+
) -> List[Subscription]:
|
|
199
|
+
"""
|
|
200
|
+
Retrieve existing subscriptions for a dataset or assertion.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
urn: The URN of the dataset or assertion to query subscriptions for.
|
|
204
|
+
entity_change_types: Optional filter to return only subscriptions for specific
|
|
205
|
+
change types. If None, returns subscriptions for all change types.
|
|
206
|
+
subscriber_urn: Optional filter to return only subscriptions for a specific user
|
|
207
|
+
or group. Can be a string (valid corpuser or corpGroup URN) or URN object.
|
|
208
|
+
If None, returns subscriptions for all subscribers.
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
List[Subscription]: List of matching subscription objects.
|
|
212
|
+
"""
|
|
213
|
+
_print_experimental_warning()
|
|
214
|
+
logger.info(
|
|
215
|
+
f"Listing subscriptions for {urn} with change types: {entity_change_types} and subscriber: {subscriber_urn}"
|
|
216
|
+
)
|
|
217
|
+
# TODO: Implement the actual logic to retrieve subscriptions.
|
|
218
|
+
return [Subscription(**{})]
|
|
219
|
+
|
|
220
|
+
def unsubscribe(
|
|
221
|
+
self,
|
|
222
|
+
*,
|
|
223
|
+
urn: Union[str, DatasetUrn, AssertionUrn],
|
|
224
|
+
subscriber_urn: SubscriberInputType,
|
|
225
|
+
entity_change_types: Optional[
|
|
226
|
+
List[Union[str, models.EntityChangeTypeClass]]
|
|
227
|
+
] = None,
|
|
228
|
+
) -> None:
|
|
229
|
+
"""
|
|
230
|
+
Remove subscriptions for entity change notifications.
|
|
231
|
+
|
|
232
|
+
This method supports selective unsubscription based on subscriber and change types.
|
|
233
|
+
The behavior varies depending on whether the target is a dataset or assertion:
|
|
234
|
+
|
|
235
|
+
**Dataset unsubscription:**
|
|
236
|
+
- Removes specified change types from the subscription
|
|
237
|
+
- If no change types specified, removes all existing change types
|
|
238
|
+
- Deletes entire subscription if no change types remain
|
|
239
|
+
|
|
240
|
+
**Assertion unsubscription:**
|
|
241
|
+
- Removes assertion from specified change type filters
|
|
242
|
+
- If no change types specified, removes assertion from all assertion-related change types
|
|
243
|
+
(ASSERTION_PASSED, ASSERTION_FAILED, ASSERTION_ERROR)
|
|
244
|
+
- Deletes change type if no assertions remain in filter
|
|
245
|
+
(prevents assertion-level subscription from silently upgrading to dataset-level)
|
|
246
|
+
- Deletes entire subscription if no change types remain
|
|
247
|
+
|
|
248
|
+
**Warning behavior:**
|
|
249
|
+
- If entity_change_types is explicitly provided:
|
|
250
|
+
* Warns about change types that don't exist in the subscription
|
|
251
|
+
* For assertions: warns about change types that don't include the assertion in their filter
|
|
252
|
+
- If entity_change_types is None (using defaults), no warnings are logged
|
|
253
|
+
|
|
254
|
+
Args:
|
|
255
|
+
urn: URN (string or URN object) of the dataset or assertion to unsubscribe from.
|
|
256
|
+
subscriber_urn: User or group URN to unsubscribe.
|
|
257
|
+
Can be a string (valid corpuser or corpGroup URN) or URN object.
|
|
258
|
+
entity_change_types: Specific change types to remove. If None, defaults are:
|
|
259
|
+
- Dataset: all existing change types in the subscription
|
|
260
|
+
- Assertion: assertion-related types (ASSERTION_PASSED,
|
|
261
|
+
ASSERTION_FAILED, ASSERTION_ERROR)
|
|
262
|
+
|
|
263
|
+
Returns:
|
|
264
|
+
None
|
|
265
|
+
|
|
266
|
+
Raises:
|
|
267
|
+
SdkUsageError: If URN format is invalid, entity not found, or empty change types list.
|
|
268
|
+
SdkUsageError: For assertion unsubscription - if not assertion-related change types
|
|
269
|
+
(ASSERTION_PASSED, ASSERTION_FAILED, ASSERTION_ERROR) are provided.
|
|
270
|
+
|
|
271
|
+
Note:
|
|
272
|
+
This method is experimental and may change in future versions.
|
|
273
|
+
"""
|
|
274
|
+
_print_experimental_warning()
|
|
275
|
+
|
|
276
|
+
# Parse URN string if needed
|
|
277
|
+
parsed_urn = self._maybe_parse_urn(urn)
|
|
278
|
+
|
|
279
|
+
# Parse subscriber URN string if needed
|
|
280
|
+
parsed_subscriber_urn = self._maybe_parse_subscriber_urn(subscriber_urn)
|
|
281
|
+
|
|
282
|
+
dataset_urn: DatasetUrn
|
|
283
|
+
assertion_urn: Optional[AssertionUrn]
|
|
284
|
+
dataset_urn, assertion_urn = (
|
|
285
|
+
(parsed_urn, None)
|
|
286
|
+
if isinstance(parsed_urn, DatasetUrn)
|
|
287
|
+
else self._fetch_dataset_from_assertion(parsed_urn)
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
logger.info(
|
|
291
|
+
f"Unsubscribing from dataset={dataset_urn}{f' assertion={assertion_urn}' if assertion_urn else ''} for subscriber={parsed_subscriber_urn} with change types: {entity_change_types}"
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
# Find existing subscription
|
|
295
|
+
existing_subscription_urns = self.client.resolve.subscription( # type: ignore[attr-defined]
|
|
296
|
+
entity_urn=dataset_urn.urn(),
|
|
297
|
+
actor_urn=parsed_subscriber_urn.urn(),
|
|
298
|
+
skip_cache=True,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
if not existing_subscription_urns:
|
|
302
|
+
logger.info(
|
|
303
|
+
f"No subscription found for dataset={dataset_urn} and subscriber={parsed_subscriber_urn}"
|
|
304
|
+
)
|
|
305
|
+
return
|
|
306
|
+
elif len(existing_subscription_urns) > 1:
|
|
307
|
+
raise SdkUsageError(
|
|
308
|
+
f"Multiple subscriptions found for dataset={dataset_urn} and subscriber={parsed_subscriber_urn}. "
|
|
309
|
+
f"Expected at most 1, got {len(existing_subscription_urns)}"
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
subscription_urn = existing_subscription_urns[0]
|
|
313
|
+
subscription_entity = self.client.entities.get(subscription_urn)
|
|
314
|
+
assert isinstance(subscription_entity, Subscription), (
|
|
315
|
+
f"Expected Subscription entity type for subscription urn={subscription_urn}"
|
|
316
|
+
)
|
|
317
|
+
logger.info(
|
|
318
|
+
f"Found existing subscription to be updated: {subscription_entity.urn}"
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
# Get the change types to remove (validated input or defaults)
|
|
322
|
+
change_types_to_remove = self._get_entity_change_types(
|
|
323
|
+
assertion_scope=assertion_urn is not None,
|
|
324
|
+
entity_change_types=entity_change_types,
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
# Remove the specified change types
|
|
328
|
+
if subscription_entity.info.entityChangeTypes is None:
|
|
329
|
+
raise SdkUsageError(
|
|
330
|
+
f"Subscription {subscription_entity.urn} has no change types to remove"
|
|
331
|
+
)
|
|
332
|
+
# Determine if we should warn about missing items (only when user explicitly provided change types)
|
|
333
|
+
warn_if_missing = entity_change_types is not None
|
|
334
|
+
updated_change_types = self._remove_change_types(
|
|
335
|
+
subscription_entity.info.entityChangeTypes,
|
|
336
|
+
change_types_to_remove,
|
|
337
|
+
assertion_urn_to_remove=assertion_urn,
|
|
338
|
+
warn_if_missing=warn_if_missing,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
# If no change types remain, delete the subscription
|
|
342
|
+
if not updated_change_types:
|
|
343
|
+
logger.info(
|
|
344
|
+
f"No change types remain, deleting subscription: {subscription_entity.urn}"
|
|
345
|
+
)
|
|
346
|
+
self.client.entities.delete(subscription_entity.urn)
|
|
347
|
+
return
|
|
348
|
+
|
|
349
|
+
# Update the subscription with remaining change types
|
|
350
|
+
subscription_entity.info.entityChangeTypes = updated_change_types
|
|
351
|
+
subscription_entity.info.updatedOn = self._create_audit_stamp()
|
|
352
|
+
self.client.entities.upsert(subscription_entity, emit_mode=EmitMode.SYNC_WAIT)
|
|
353
|
+
logger.info(f"Subscription updated: {subscription_entity.urn}")
|
|
354
|
+
|
|
355
|
+
def _get_entity_change_types(
|
|
356
|
+
self,
|
|
357
|
+
assertion_scope: bool,
|
|
358
|
+
entity_change_types: Optional[
|
|
359
|
+
Sequence[Union[str, models.EntityChangeTypeClass]]
|
|
360
|
+
] = None,
|
|
361
|
+
) -> List[str]:
|
|
362
|
+
"""Get entity change types with validation and defaults.
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
assertion_scope: True for assertion subscriptions, False for dataset subscriptions.
|
|
366
|
+
entity_change_types: Specific change types to validate. If None, returns defaults.
|
|
367
|
+
|
|
368
|
+
Returns:
|
|
369
|
+
List of validated entity change types. Defaults are:
|
|
370
|
+
- Dataset: all available change types
|
|
371
|
+
- Assertion: assertion-related types (ASSERTION_PASSED, ASSERTION_FAILED, ASSERTION_ERROR)
|
|
372
|
+
|
|
373
|
+
Raises:
|
|
374
|
+
SdkUsageError: If entity_change_types is an empty list.
|
|
375
|
+
SdkUsageError: If invalid change types provided or assertion scope receives
|
|
376
|
+
non-assertion change types.
|
|
377
|
+
"""
|
|
378
|
+
if entity_change_types is not None:
|
|
379
|
+
if len(entity_change_types) == 0:
|
|
380
|
+
raise SdkUsageError("Entity change types cannot be an empty list.")
|
|
381
|
+
|
|
382
|
+
# Convert all entity change types to strings
|
|
383
|
+
entity_change_type_strs = [str(ect) for ect in entity_change_types]
|
|
384
|
+
|
|
385
|
+
all_options = get_enum_options(models.EntityChangeTypeClass)
|
|
386
|
+
if any([ect not in all_options for ect in entity_change_type_strs]):
|
|
387
|
+
raise SdkUsageError(
|
|
388
|
+
f"Invalid entity change types provided: {entity_change_type_strs}. "
|
|
389
|
+
f"Valid options are: {all_options}"
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
# For assertion scope, validate that only assertion-related change types are provided
|
|
393
|
+
if assertion_scope:
|
|
394
|
+
invalid_types = [
|
|
395
|
+
ect
|
|
396
|
+
for ect in entity_change_type_strs
|
|
397
|
+
if ect not in ASSERTION_RELATED_ENTITY_CHANGE_TYPES
|
|
398
|
+
]
|
|
399
|
+
if invalid_types:
|
|
400
|
+
raise SdkUsageError(
|
|
401
|
+
f"For assertion subscriptions, only assertion-related change types are allowed. "
|
|
402
|
+
f"Invalid types: {invalid_types}. "
|
|
403
|
+
f"Valid types: {list(ASSERTION_RELATED_ENTITY_CHANGE_TYPES)}"
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
return entity_change_type_strs
|
|
407
|
+
|
|
408
|
+
# If no specific change types are provided, return defaults based on scope
|
|
409
|
+
if assertion_scope:
|
|
410
|
+
return list(ASSERTION_RELATED_ENTITY_CHANGE_TYPES)
|
|
411
|
+
else:
|
|
412
|
+
return list(ALL_EXISTING_ENTITY_CHANGE_TYPES)
|
|
413
|
+
|
|
414
|
+
def _create_audit_stamp(self) -> models.AuditStampClass:
|
|
415
|
+
"""Create an audit stamp with current timestamp and default actor."""
|
|
416
|
+
return models.AuditStampClass(
|
|
417
|
+
make_ts_millis(datetime.now(tz=timezone.utc)),
|
|
418
|
+
actor=DEFAULT_ACTOR_URN, # TODO: Replace with actual actor URN from token if available
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
def _merge_entity_change_types(
|
|
422
|
+
self,
|
|
423
|
+
existing_change_types: Optional[List[models.EntityChangeDetailsClass]],
|
|
424
|
+
new_change_type_strs: List[str],
|
|
425
|
+
new_assertion_urn: Optional[AssertionUrn] = None,
|
|
426
|
+
) -> List[models.EntityChangeDetailsClass]:
|
|
427
|
+
"""Merge existing entity change types with new ones, avoiding duplicates.
|
|
428
|
+
|
|
429
|
+
Args:
|
|
430
|
+
existing_change_types: Existing entity change types from the subscription.
|
|
431
|
+
Can be None when creating a new subscription.
|
|
432
|
+
new_change_type_strs: New entity change type strings to add
|
|
433
|
+
new_assertion_urn: Optional Assertion URN to associate with the new change types
|
|
434
|
+
|
|
435
|
+
Returns:
|
|
436
|
+
List of EntityChangeDetailsClass with merged change types
|
|
437
|
+
|
|
438
|
+
Note:
|
|
439
|
+
This method does not modify existing_change_types in-place; it returns a new list.
|
|
440
|
+
"""
|
|
441
|
+
assert len(new_change_type_strs) > 0, (
|
|
442
|
+
"new_change_type_strs cannot be empty, worse case we have the default values"
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
existing_change_type_str_map_filters: Dict[
|
|
446
|
+
str, Optional[models.EntityChangeDetailsFilterClass]
|
|
447
|
+
] = (
|
|
448
|
+
{
|
|
449
|
+
# ect.entityChangeType: Union[str, EntityChangeTypeClass]; EntityChangeTypeClass is cosmetic, just a decorator
|
|
450
|
+
str(ect.entityChangeType): ect.filter
|
|
451
|
+
for ect in existing_change_types
|
|
452
|
+
}
|
|
453
|
+
if existing_change_types
|
|
454
|
+
else {}
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
# Combine existing and new change types (avoid duplicates)
|
|
458
|
+
all_change_types = set(existing_change_type_str_map_filters.keys()).union(
|
|
459
|
+
set(new_change_type_strs)
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
return [
|
|
463
|
+
models.EntityChangeDetailsClass(
|
|
464
|
+
entityChangeType=ect,
|
|
465
|
+
filter=self._merge_entity_change_types_filter(
|
|
466
|
+
existing_filter=existing_change_type_str_map_filters.get(ect),
|
|
467
|
+
# Apply new assertion URN to change types that are being newly added or re-specified
|
|
468
|
+
new_assertion_urn=new_assertion_urn
|
|
469
|
+
if ect in new_change_type_strs
|
|
470
|
+
else None,
|
|
471
|
+
),
|
|
472
|
+
)
|
|
473
|
+
for ect in all_change_types
|
|
474
|
+
]
|
|
475
|
+
|
|
476
|
+
def _merge_entity_change_types_filter(
|
|
477
|
+
self,
|
|
478
|
+
existing_filter: Optional[models.EntityChangeDetailsFilterClass],
|
|
479
|
+
new_assertion_urn: Optional[AssertionUrn] = None,
|
|
480
|
+
) -> Optional[models.EntityChangeDetailsFilterClass]:
|
|
481
|
+
"""Merge existing filter with new assertion URN if provided.
|
|
482
|
+
|
|
483
|
+
Args:
|
|
484
|
+
existing_filter: Existing filter from the subscription
|
|
485
|
+
new_assertion_urn: New assertion URN to add to the filter
|
|
486
|
+
|
|
487
|
+
Returns:
|
|
488
|
+
Merged filter with new assertion URN if provided, otherwise returns existing filter.
|
|
489
|
+
"""
|
|
490
|
+
if not existing_filter:
|
|
491
|
+
# if new assertion, create a new filter with it, otherwise return None
|
|
492
|
+
return (
|
|
493
|
+
models.EntityChangeDetailsFilterClass(
|
|
494
|
+
includeAssertions=[new_assertion_urn.urn()]
|
|
495
|
+
)
|
|
496
|
+
if new_assertion_urn
|
|
497
|
+
else None
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
if not new_assertion_urn:
|
|
501
|
+
# If no new assertion URN, just return the existing filter, None or whatever it is
|
|
502
|
+
return existing_filter
|
|
503
|
+
|
|
504
|
+
assert existing_filter is not None and new_assertion_urn is not None
|
|
505
|
+
|
|
506
|
+
if (
|
|
507
|
+
existing_filter.includeAssertions is None
|
|
508
|
+
or len(existing_filter.includeAssertions) == 0
|
|
509
|
+
):
|
|
510
|
+
# An existing filter with empty includeAssertions is weird, but we handle it just in case
|
|
511
|
+
existing_filter.includeAssertions = [new_assertion_urn.urn()]
|
|
512
|
+
return existing_filter
|
|
513
|
+
|
|
514
|
+
assert len(existing_filter.includeAssertions) > 0
|
|
515
|
+
|
|
516
|
+
if new_assertion_urn.urn() not in existing_filter.includeAssertions:
|
|
517
|
+
# Only added if not present already
|
|
518
|
+
existing_filter.includeAssertions.append(new_assertion_urn.urn())
|
|
519
|
+
|
|
520
|
+
return existing_filter
|
|
521
|
+
|
|
522
|
+
def _remove_change_types_filter(
|
|
523
|
+
self,
|
|
524
|
+
entity_change_type: str,
|
|
525
|
+
existing_filter: Optional[models.EntityChangeDetailsFilterClass],
|
|
526
|
+
assertion_urn_to_remove: Optional[AssertionUrn] = None,
|
|
527
|
+
warn_if_missing: bool = True,
|
|
528
|
+
) -> Optional[models.EntityChangeDetailsFilterClass]:
|
|
529
|
+
"""Remove assertion URN from existing filter.
|
|
530
|
+
|
|
531
|
+
Args:
|
|
532
|
+
entity_change_type: The entity change type, used only for better logging messages
|
|
533
|
+
existing_filter: Existing filter from the subscription
|
|
534
|
+
assertion_urn_to_remove: Assertion URN to remove from the filter
|
|
535
|
+
warn_if_missing: Whether to log warning if assertion is not found in filter
|
|
536
|
+
|
|
537
|
+
Returns:
|
|
538
|
+
Updated filter with assertion URN removed, or None if no assertions remain
|
|
539
|
+
(indicating the entire change type should be removed).
|
|
540
|
+
"""
|
|
541
|
+
if not existing_filter:
|
|
542
|
+
# No filter means dataset-level subscription, assertion is not included
|
|
543
|
+
if assertion_urn_to_remove and warn_if_missing:
|
|
544
|
+
logger.warning(
|
|
545
|
+
f"Assertion {assertion_urn_to_remove.urn()} is not included in entity change type '{entity_change_type}' and will be ignored"
|
|
546
|
+
)
|
|
547
|
+
return existing_filter
|
|
548
|
+
|
|
549
|
+
if not assertion_urn_to_remove:
|
|
550
|
+
# If no assertion URN to remove, just return the existing filter
|
|
551
|
+
return existing_filter
|
|
552
|
+
|
|
553
|
+
if (
|
|
554
|
+
existing_filter.includeAssertions is None
|
|
555
|
+
or len(existing_filter.includeAssertions) == 0
|
|
556
|
+
):
|
|
557
|
+
# Empty includeAssertions means dataset-level subscription, assertion is not included
|
|
558
|
+
if warn_if_missing:
|
|
559
|
+
logger.warning(
|
|
560
|
+
f"Assertion {assertion_urn_to_remove.urn()} is not included in entity change type '{entity_change_type}' and will be ignored"
|
|
561
|
+
)
|
|
562
|
+
return existing_filter
|
|
563
|
+
|
|
564
|
+
assertion_urn_str = assertion_urn_to_remove.urn()
|
|
565
|
+
if assertion_urn_str not in existing_filter.includeAssertions:
|
|
566
|
+
# Assertion not found in filter
|
|
567
|
+
if warn_if_missing:
|
|
568
|
+
logger.warning(
|
|
569
|
+
f"Assertion {assertion_urn_str} is not included in entity change type '{entity_change_type}' and will be ignored"
|
|
570
|
+
)
|
|
571
|
+
return existing_filter
|
|
572
|
+
|
|
573
|
+
# Remove the assertion from the list
|
|
574
|
+
updated_assertions = [
|
|
575
|
+
urn for urn in existing_filter.includeAssertions if urn != assertion_urn_str
|
|
576
|
+
]
|
|
577
|
+
|
|
578
|
+
if not updated_assertions:
|
|
579
|
+
# No assertions remain, return None to indicate change type should be removed
|
|
580
|
+
return None
|
|
581
|
+
|
|
582
|
+
# Return updated filter with remaining assertions
|
|
583
|
+
existing_filter.includeAssertions = updated_assertions
|
|
584
|
+
return existing_filter
|
|
585
|
+
|
|
586
|
+
def _remove_change_types(
|
|
587
|
+
self,
|
|
588
|
+
existing_change_types: List[models.EntityChangeDetailsClass],
|
|
589
|
+
change_types_to_remove: List[str],
|
|
590
|
+
assertion_urn_to_remove: Optional[AssertionUrn] = None,
|
|
591
|
+
warn_if_missing: bool = True,
|
|
592
|
+
) -> List[models.EntityChangeDetailsClass]:
|
|
593
|
+
"""Remove specified change types from subscription, returning a new list.
|
|
594
|
+
|
|
595
|
+
For dataset unsubscription, removes entire change types.
|
|
596
|
+
For assertion unsubscription, removes assertion from change type filters,
|
|
597
|
+
and removes entire change type if no assertions remain.
|
|
598
|
+
|
|
599
|
+
Args:
|
|
600
|
+
existing_change_types: Current entity change types from the subscription.
|
|
601
|
+
Never None since this method is only called for existing subscriptions.
|
|
602
|
+
change_types_to_remove: List of change type strings to remove (must not be empty)
|
|
603
|
+
assertion_urn_to_remove: Optional assertion URN to remove from filters.
|
|
604
|
+
If None, performs dataset-level removal (removes entire change types).
|
|
605
|
+
warn_if_missing: Whether to log warnings about missing change types or assertions.
|
|
606
|
+
|
|
607
|
+
Returns:
|
|
608
|
+
New list of EntityChangeDetailsClass with specified change types or assertions removed
|
|
609
|
+
|
|
610
|
+
Note:
|
|
611
|
+
This method does not modify existing_change_types in-place; it returns a new list.
|
|
612
|
+
"""
|
|
613
|
+
assert len(change_types_to_remove) > 0, (
|
|
614
|
+
"change_types_to_remove cannot be empty, worse case we have the default values"
|
|
615
|
+
)
|
|
616
|
+
assert len(existing_change_types) > 0, (
|
|
617
|
+
"Subscription must have at least one change type (no model restriction but business rule)"
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
change_types_to_remove_set = set(change_types_to_remove)
|
|
621
|
+
existing_change_types_set = {
|
|
622
|
+
str(ect.entityChangeType) for ect in existing_change_types
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
# Warn about change types that don't exist in the subscription
|
|
626
|
+
nonexistent_change_types = (
|
|
627
|
+
change_types_to_remove_set - existing_change_types_set
|
|
628
|
+
)
|
|
629
|
+
if nonexistent_change_types and warn_if_missing:
|
|
630
|
+
logger.warning(
|
|
631
|
+
f"The following change types do not exist in the subscription and will be ignored: {sorted(nonexistent_change_types)}"
|
|
632
|
+
)
|
|
633
|
+
|
|
634
|
+
if assertion_urn_to_remove is None:
|
|
635
|
+
# Dataset-level removal: remove entire change types
|
|
636
|
+
return [
|
|
637
|
+
ect
|
|
638
|
+
for ect in existing_change_types
|
|
639
|
+
if str(ect.entityChangeType) not in change_types_to_remove_set
|
|
640
|
+
]
|
|
641
|
+
else:
|
|
642
|
+
# Assertion-level removal: remove assertion from filters
|
|
643
|
+
result = []
|
|
644
|
+
for ect in existing_change_types:
|
|
645
|
+
if str(ect.entityChangeType) not in change_types_to_remove_set:
|
|
646
|
+
# Keep change types not being removed
|
|
647
|
+
result.append(ect)
|
|
648
|
+
else:
|
|
649
|
+
# Remove assertion from this change type's filter
|
|
650
|
+
updated_filter = self._remove_change_types_filter(
|
|
651
|
+
entity_change_type=str(ect.entityChangeType),
|
|
652
|
+
existing_filter=ect.filter,
|
|
653
|
+
assertion_urn_to_remove=assertion_urn_to_remove,
|
|
654
|
+
warn_if_missing=warn_if_missing,
|
|
655
|
+
)
|
|
656
|
+
if updated_filter is not None:
|
|
657
|
+
# Keep change type with updated filter
|
|
658
|
+
result.append(
|
|
659
|
+
models.EntityChangeDetailsClass(
|
|
660
|
+
entityChangeType=ect.entityChangeType,
|
|
661
|
+
filter=updated_filter,
|
|
662
|
+
)
|
|
663
|
+
)
|
|
664
|
+
# If updated_filter is None, the change type is removed entirely
|
|
665
|
+
return result
|
|
666
|
+
|
|
667
|
+
def _maybe_parse_urn(
|
|
668
|
+
self, urn: Union[str, DatasetUrn, AssertionUrn]
|
|
669
|
+
) -> Union[DatasetUrn, AssertionUrn]:
|
|
670
|
+
"""Parse URN string into appropriate URN object if needed.
|
|
671
|
+
|
|
672
|
+
Args:
|
|
673
|
+
urn: String URN or URN object (DatasetUrn or AssertionUrn)
|
|
674
|
+
|
|
675
|
+
Returns:
|
|
676
|
+
Parsed URN object (DatasetUrn or AssertionUrn)
|
|
677
|
+
|
|
678
|
+
Raises:
|
|
679
|
+
SdkUsageError: If the URN string format is unsupported entity type
|
|
680
|
+
"""
|
|
681
|
+
if isinstance(urn, (DatasetUrn, AssertionUrn)):
|
|
682
|
+
return urn
|
|
683
|
+
|
|
684
|
+
# Try to determine URN type from string format
|
|
685
|
+
if ":dataset:" in urn:
|
|
686
|
+
return DatasetUrn.from_string(urn)
|
|
687
|
+
elif ":assertion:" in urn:
|
|
688
|
+
return AssertionUrn.from_string(urn)
|
|
689
|
+
else:
|
|
690
|
+
raise SdkUsageError(
|
|
691
|
+
f"Unsupported URN type. Only dataset and assertion URNs are supported, got: {urn}"
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
def _maybe_parse_subscriber_urn(
|
|
695
|
+
self, subscriber_urn: SubscriberInputType
|
|
696
|
+
) -> Union[CorpUserUrn, CorpGroupUrn]:
|
|
697
|
+
"""Parse subscriber URN string into appropriate URN object if needed.
|
|
698
|
+
|
|
699
|
+
Args:
|
|
700
|
+
subscriber_urn: String URN or URN object (CorpUserUrn or CorpGroupUrn)
|
|
701
|
+
|
|
702
|
+
Returns:
|
|
703
|
+
Parsed URN object (CorpUserUrn or CorpGroupUrn)
|
|
704
|
+
|
|
705
|
+
Raises:
|
|
706
|
+
SdkUsageError: If the URN string format is invalid or unsupported
|
|
707
|
+
"""
|
|
708
|
+
if isinstance(subscriber_urn, (CorpUserUrn, CorpGroupUrn)):
|
|
709
|
+
return subscriber_urn
|
|
710
|
+
|
|
711
|
+
# Try to determine URN type from string format
|
|
712
|
+
if ":corpuser:" in subscriber_urn:
|
|
713
|
+
return CorpUserUrn.from_string(subscriber_urn)
|
|
714
|
+
elif ":corpGroup:" in subscriber_urn:
|
|
715
|
+
return CorpGroupUrn.from_string(subscriber_urn)
|
|
716
|
+
else:
|
|
717
|
+
raise SdkUsageError(
|
|
718
|
+
f"Unsupported subscriber URN type. Only corpuser and corpGroup URNs are supported, got: {subscriber_urn}"
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
def _fetch_dataset_from_assertion(
|
|
722
|
+
self, assertion_urn: AssertionUrn
|
|
723
|
+
) -> Tuple[DatasetUrn, AssertionUrn]:
|
|
724
|
+
assertion = self.client.entities.get(assertion_urn)
|
|
725
|
+
if assertion is None:
|
|
726
|
+
raise SdkUsageError(f"Assertion {assertion_urn} not found.")
|
|
727
|
+
|
|
728
|
+
assert isinstance(assertion, Assertion), (
|
|
729
|
+
f"Expected Assertion entity type for assertion urn={assertion_urn}"
|
|
730
|
+
)
|
|
731
|
+
return (assertion.dataset, assertion_urn)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _print_experimental_warning() -> None:
|
|
735
|
+
print(
|
|
736
|
+
"Warning: The subscriptions client is experimental and under heavy development. Expect breaking changes."
|
|
737
|
+
)
|