acryl-datahub 1.2.0.9rc1__py3-none-any.whl → 1.2.0.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.2.0.9rc1.dist-info → acryl_datahub-1.2.0.10.dist-info}/METADATA +2568 -2626
- {acryl_datahub-1.2.0.9rc1.dist-info → acryl_datahub-1.2.0.10.dist-info}/RECORD +120 -113
- {acryl_datahub-1.2.0.9rc1.dist-info → acryl_datahub-1.2.0.10.dist-info}/entry_points.txt +2 -0
- datahub/_version.py +1 -1
- datahub/api/entities/assertion/assertion.py +1 -1
- datahub/api/entities/corpgroup/corpgroup.py +1 -1
- datahub/api/entities/dataproduct/dataproduct.py +6 -3
- datahub/api/entities/dataset/dataset.py +9 -18
- datahub/api/entities/structuredproperties/structuredproperties.py +2 -2
- datahub/api/graphql/operation.py +10 -6
- datahub/cli/docker_check.py +2 -2
- datahub/configuration/common.py +29 -1
- datahub/configuration/connection_resolver.py +5 -2
- datahub/configuration/import_resolver.py +7 -4
- datahub/configuration/pydantic_migration_helpers.py +0 -9
- datahub/configuration/source_common.py +3 -2
- datahub/configuration/validate_field_deprecation.py +5 -2
- datahub/configuration/validate_field_removal.py +5 -2
- datahub/configuration/validate_field_rename.py +6 -5
- datahub/configuration/validate_multiline_string.py +5 -2
- datahub/ingestion/autogenerated/capability_summary.json +45 -1
- datahub/ingestion/run/pipeline_config.py +2 -2
- datahub/ingestion/source/azure/azure_common.py +1 -1
- datahub/ingestion/source/bigquery_v2/bigquery_config.py +28 -14
- datahub/ingestion/source/bigquery_v2/bigquery_schema.py +23 -16
- datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +11 -0
- datahub/ingestion/source/bigquery_v2/queries_extractor.py +4 -5
- datahub/ingestion/source/common/gcp_credentials_config.py +3 -1
- datahub/ingestion/source/data_lake_common/path_spec.py +16 -16
- datahub/ingestion/source/datahub/config.py +8 -9
- datahub/ingestion/source/dbt/dbt_common.py +65 -5
- datahub/ingestion/source/delta_lake/config.py +1 -1
- datahub/ingestion/source/dremio/dremio_config.py +3 -4
- datahub/ingestion/source/feast.py +8 -10
- datahub/ingestion/source/fivetran/config.py +1 -1
- datahub/ingestion/source/gcs/gcs_source.py +19 -2
- datahub/ingestion/source/ge_data_profiler.py +15 -2
- datahub/ingestion/source/ge_profiling_config.py +26 -22
- datahub/ingestion/source/grafana/grafana_config.py +2 -2
- datahub/ingestion/source/grafana/models.py +12 -14
- datahub/ingestion/source/hex/hex.py +6 -1
- datahub/ingestion/source/iceberg/iceberg_profiler.py +4 -2
- datahub/ingestion/source/kafka_connect/common.py +2 -2
- datahub/ingestion/source/looker/looker_common.py +76 -75
- datahub/ingestion/source/looker/looker_config.py +15 -4
- datahub/ingestion/source/looker/looker_source.py +493 -547
- datahub/ingestion/source/looker/lookml_config.py +1 -1
- datahub/ingestion/source/looker/lookml_source.py +46 -88
- datahub/ingestion/source/metabase.py +9 -2
- datahub/ingestion/source/metadata/business_glossary.py +7 -7
- datahub/ingestion/source/metadata/lineage.py +1 -1
- datahub/ingestion/source/mode.py +13 -5
- datahub/ingestion/source/nifi.py +1 -1
- datahub/ingestion/source/powerbi/config.py +14 -21
- datahub/ingestion/source/preset.py +1 -1
- datahub/ingestion/source/qlik_sense/data_classes.py +28 -8
- datahub/ingestion/source/redash.py +1 -1
- datahub/ingestion/source/redshift/config.py +6 -3
- datahub/ingestion/source/redshift/query.py +23 -19
- datahub/ingestion/source/s3/source.py +26 -24
- datahub/ingestion/source/salesforce.py +13 -9
- datahub/ingestion/source/schema/json_schema.py +14 -14
- datahub/ingestion/source/sigma/data_classes.py +3 -0
- datahub/ingestion/source/snaplogic/__init__.py +0 -0
- datahub/ingestion/source/snaplogic/snaplogic.py +355 -0
- datahub/ingestion/source/snaplogic/snaplogic_config.py +37 -0
- datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py +107 -0
- datahub/ingestion/source/snaplogic/snaplogic_parser.py +168 -0
- datahub/ingestion/source/snaplogic/snaplogic_utils.py +31 -0
- datahub/ingestion/source/snowflake/snowflake_config.py +12 -15
- datahub/ingestion/source/snowflake/snowflake_connection.py +8 -3
- datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +15 -2
- datahub/ingestion/source/snowflake/snowflake_queries.py +4 -5
- datahub/ingestion/source/sql/athena.py +2 -1
- datahub/ingestion/source/sql/clickhouse.py +12 -7
- datahub/ingestion/source/sql/cockroachdb.py +5 -3
- datahub/ingestion/source/sql/druid.py +2 -2
- datahub/ingestion/source/sql/hive.py +4 -3
- datahub/ingestion/source/sql/hive_metastore.py +7 -9
- datahub/ingestion/source/sql/mssql/source.py +2 -2
- datahub/ingestion/source/sql/mysql.py +2 -2
- datahub/ingestion/source/sql/oracle.py +3 -3
- datahub/ingestion/source/sql/presto.py +2 -1
- datahub/ingestion/source/sql/teradata.py +4 -4
- datahub/ingestion/source/sql/trino.py +2 -1
- datahub/ingestion/source/sql/two_tier_sql_source.py +2 -3
- datahub/ingestion/source/sql/vertica.py +1 -1
- datahub/ingestion/source/sql_queries.py +6 -6
- datahub/ingestion/source/state/checkpoint.py +5 -1
- datahub/ingestion/source/state/entity_removal_state.py +5 -2
- datahub/ingestion/source/state/stateful_ingestion_base.py +5 -8
- datahub/ingestion/source/superset.py +122 -15
- datahub/ingestion/source/tableau/tableau.py +68 -14
- datahub/ingestion/source/tableau/tableau_common.py +5 -0
- datahub/ingestion/source/tableau/tableau_constant.py +1 -0
- datahub/ingestion/source/tableau/tableau_server_wrapper.py +3 -0
- datahub/ingestion/source/unity/config.py +7 -3
- datahub/ingestion/source/usage/usage_common.py +3 -3
- datahub/ingestion/source_config/pulsar.py +3 -1
- datahub/ingestion/transformer/set_browse_path.py +112 -0
- datahub/metadata/_internal_schema_classes.py +728 -528
- datahub/metadata/_urns/urn_defs.py +1702 -1702
- datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +4 -0
- datahub/metadata/schema.avsc +17434 -17732
- datahub/metadata/schemas/GlobalSettingsInfo.avsc +72 -0
- datahub/metadata/schemas/InstitutionalMemory.avsc +22 -0
- datahub/metadata/schemas/LogicalParent.avsc +2 -1
- datahub/metadata/schemas/MLModelGroupKey.avsc +2 -1
- datahub/metadata/schemas/MetadataChangeEvent.avsc +22 -0
- datahub/sdk/_shared.py +126 -0
- datahub/sdk/chart.py +87 -30
- datahub/sdk/dashboard.py +79 -34
- datahub/sdk/entity_client.py +11 -4
- datahub/sdk/lineage_client.py +3 -3
- datahub/sdk/search_filters.py +1 -7
- datahub/sql_parsing/split_statements.py +13 -0
- {acryl_datahub-1.2.0.9rc1.dist-info → acryl_datahub-1.2.0.10.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.2.0.9rc1.dist-info → acryl_datahub-1.2.0.10.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.2.0.9rc1.dist-info → acryl_datahub-1.2.0.10.dist-info}/top_level.txt +0 -0
|
@@ -198,6 +198,78 @@
|
|
|
198
198
|
"default": null,
|
|
199
199
|
"doc": "SSO integrations between DataHub and identity providers"
|
|
200
200
|
},
|
|
201
|
+
{
|
|
202
|
+
"type": [
|
|
203
|
+
"null",
|
|
204
|
+
{
|
|
205
|
+
"type": "record",
|
|
206
|
+
"name": "OAuthSettings",
|
|
207
|
+
"namespace": "com.linkedin.pegasus2avro.settings.global",
|
|
208
|
+
"fields": [
|
|
209
|
+
{
|
|
210
|
+
"type": {
|
|
211
|
+
"type": "array",
|
|
212
|
+
"items": {
|
|
213
|
+
"type": "record",
|
|
214
|
+
"name": "OAuthProvider",
|
|
215
|
+
"namespace": "com.linkedin.pegasus2avro.settings.global",
|
|
216
|
+
"fields": [
|
|
217
|
+
{
|
|
218
|
+
"type": "boolean",
|
|
219
|
+
"name": "enabled",
|
|
220
|
+
"doc": "Whether this OAuth provider is enabled."
|
|
221
|
+
},
|
|
222
|
+
{
|
|
223
|
+
"type": "string",
|
|
224
|
+
"name": "name",
|
|
225
|
+
"doc": "The name of this OAuth provider. This is used for display purposes only."
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
"type": [
|
|
229
|
+
"null",
|
|
230
|
+
"string"
|
|
231
|
+
],
|
|
232
|
+
"name": "jwksUri",
|
|
233
|
+
"default": null,
|
|
234
|
+
"doc": "The URI of the JSON Web Key Set (JWKS) endpoint for this OAuth provider."
|
|
235
|
+
},
|
|
236
|
+
{
|
|
237
|
+
"type": "string",
|
|
238
|
+
"name": "issuer",
|
|
239
|
+
"doc": "The expected issuer (iss) claim in the JWTs issued by this OAuth provider."
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
"type": "string",
|
|
243
|
+
"name": "audience",
|
|
244
|
+
"doc": "The expected audience (aud) claim in the JWTs issued by this OAuth provider."
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
"type": "string",
|
|
248
|
+
"name": "algorithm",
|
|
249
|
+
"default": "RS256",
|
|
250
|
+
"doc": "The JWT signing algorithm required for this provider.\nPrevents algorithm confusion attacks. Common values: RS256, RS384, RS512, PS256, ES256"
|
|
251
|
+
},
|
|
252
|
+
{
|
|
253
|
+
"type": "string",
|
|
254
|
+
"name": "userIdClaim",
|
|
255
|
+
"default": "sub",
|
|
256
|
+
"doc": "The JWT claim to use as the user identifier for this provider.\nDifferent providers use different claims (sub, email, preferred_username, etc.)"
|
|
257
|
+
}
|
|
258
|
+
],
|
|
259
|
+
"doc": "An OAuth Provider. This provides information required to validate inbound\nrequests with OAuth 2.0 bearer tokens."
|
|
260
|
+
}
|
|
261
|
+
},
|
|
262
|
+
"name": "providers",
|
|
263
|
+
"doc": "Trusted OAuth Providers"
|
|
264
|
+
}
|
|
265
|
+
],
|
|
266
|
+
"doc": "Trust oauth providers to use for authentication."
|
|
267
|
+
}
|
|
268
|
+
],
|
|
269
|
+
"name": "oauth",
|
|
270
|
+
"default": null,
|
|
271
|
+
"doc": "Settings related to the oauth authentication provider"
|
|
272
|
+
},
|
|
201
273
|
{
|
|
202
274
|
"type": [
|
|
203
275
|
"null",
|
|
@@ -84,6 +84,28 @@
|
|
|
84
84
|
"name": "updateStamp",
|
|
85
85
|
"default": null,
|
|
86
86
|
"doc": "Audit stamp associated with updation of this record"
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"type": [
|
|
90
|
+
"null",
|
|
91
|
+
{
|
|
92
|
+
"type": "record",
|
|
93
|
+
"name": "InstitutionalMemoryMetadataSettings",
|
|
94
|
+
"namespace": "com.linkedin.pegasus2avro.common",
|
|
95
|
+
"fields": [
|
|
96
|
+
{
|
|
97
|
+
"type": "boolean",
|
|
98
|
+
"name": "showInAssetPreview",
|
|
99
|
+
"default": false,
|
|
100
|
+
"doc": "Show record in asset preview like on entity header and search previews"
|
|
101
|
+
}
|
|
102
|
+
],
|
|
103
|
+
"doc": "Settings related to a record of InstitutionalMemoryMetadata"
|
|
104
|
+
}
|
|
105
|
+
],
|
|
106
|
+
"name": "settings",
|
|
107
|
+
"default": null,
|
|
108
|
+
"doc": "Settings for this record"
|
|
87
109
|
}
|
|
88
110
|
],
|
|
89
111
|
"doc": "Metadata corresponding to a record of institutional memory."
|
|
@@ -1143,6 +1143,28 @@
|
|
|
1143
1143
|
"name": "updateStamp",
|
|
1144
1144
|
"default": null,
|
|
1145
1145
|
"doc": "Audit stamp associated with updation of this record"
|
|
1146
|
+
},
|
|
1147
|
+
{
|
|
1148
|
+
"type": [
|
|
1149
|
+
"null",
|
|
1150
|
+
{
|
|
1151
|
+
"type": "record",
|
|
1152
|
+
"name": "InstitutionalMemoryMetadataSettings",
|
|
1153
|
+
"namespace": "com.linkedin.pegasus2avro.common",
|
|
1154
|
+
"fields": [
|
|
1155
|
+
{
|
|
1156
|
+
"type": "boolean",
|
|
1157
|
+
"name": "showInAssetPreview",
|
|
1158
|
+
"default": false,
|
|
1159
|
+
"doc": "Show record in asset preview like on entity header and search previews"
|
|
1160
|
+
}
|
|
1161
|
+
],
|
|
1162
|
+
"doc": "Settings related to a record of InstitutionalMemoryMetadata"
|
|
1163
|
+
}
|
|
1164
|
+
],
|
|
1165
|
+
"name": "settings",
|
|
1166
|
+
"default": null,
|
|
1167
|
+
"doc": "Settings for this record"
|
|
1146
1168
|
}
|
|
1147
1169
|
],
|
|
1148
1170
|
"doc": "Metadata corresponding to a record of institutional memory."
|
datahub/sdk/_shared.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import warnings
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
4
5
|
from datetime import datetime
|
|
5
6
|
from typing import (
|
|
6
7
|
TYPE_CHECKING,
|
|
@@ -61,6 +62,7 @@ DataPlatformInstanceUrnOrStr: TypeAlias = Union[str, DataPlatformInstanceUrn]
|
|
|
61
62
|
DataPlatformUrnOrStr: TypeAlias = Union[str, DataPlatformUrn]
|
|
62
63
|
|
|
63
64
|
ActorUrn: TypeAlias = Union[CorpUserUrn, CorpGroupUrn]
|
|
65
|
+
ActorUrnOrStr: TypeAlias = Union[str, ActorUrn]
|
|
64
66
|
StructuredPropertyUrnOrStr: TypeAlias = Union[str, StructuredPropertyUrn]
|
|
65
67
|
StructuredPropertyValueType: TypeAlias = Union[str, float, int]
|
|
66
68
|
StructuredPropertyInputType: TypeAlias = Dict[
|
|
@@ -110,6 +112,130 @@ def parse_time_stamp(ts: Optional[models.TimeStampClass]) -> Optional[datetime]:
|
|
|
110
112
|
return parse_ts_millis(ts.time)
|
|
111
113
|
|
|
112
114
|
|
|
115
|
+
class ChangeAuditStampsMixin(ABC):
|
|
116
|
+
"""Mixin class for managing audit stamps on entities."""
|
|
117
|
+
|
|
118
|
+
__slots__ = ()
|
|
119
|
+
|
|
120
|
+
@abstractmethod
|
|
121
|
+
def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
|
|
122
|
+
"""Get the audit stamps from the entity properties."""
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
@abstractmethod
|
|
126
|
+
def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
|
|
127
|
+
"""Set the audit stamps on the entity properties."""
|
|
128
|
+
pass
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def last_modified(self) -> Optional[datetime]:
|
|
132
|
+
"""Get the last modification timestamp from audit stamps."""
|
|
133
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
134
|
+
if audit_stamps.lastModified.time == 0:
|
|
135
|
+
return None
|
|
136
|
+
return datetime.fromtimestamp(
|
|
137
|
+
audit_stamps.lastModified.time / 1000
|
|
138
|
+
) # supports only seconds precision
|
|
139
|
+
|
|
140
|
+
def set_last_modified(self, last_modified: datetime) -> None:
|
|
141
|
+
"""Set the last modification timestamp in audit stamps."""
|
|
142
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
143
|
+
audit_stamps.lastModified.time = make_ts_millis(last_modified)
|
|
144
|
+
self._set_audit_stamps(audit_stamps)
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def last_modified_by(self) -> Optional[str]:
|
|
148
|
+
"""Get the last modification actor from audit stamps."""
|
|
149
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
150
|
+
if audit_stamps.lastModified.actor == builder.UNKNOWN_USER:
|
|
151
|
+
return None
|
|
152
|
+
return audit_stamps.lastModified.actor
|
|
153
|
+
|
|
154
|
+
def set_last_modified_by(self, last_modified_by: ActorUrnOrStr) -> None:
|
|
155
|
+
"""Set the last modification actor in audit stamps."""
|
|
156
|
+
if isinstance(last_modified_by, str):
|
|
157
|
+
last_modified_by = make_user_urn(last_modified_by)
|
|
158
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
159
|
+
audit_stamps.lastModified.actor = str(last_modified_by)
|
|
160
|
+
self._set_audit_stamps(audit_stamps)
|
|
161
|
+
|
|
162
|
+
@property
|
|
163
|
+
def created_at(self) -> Optional[datetime]:
|
|
164
|
+
"""Get the creation timestamp from audit stamps."""
|
|
165
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
166
|
+
if audit_stamps.created.time == 0:
|
|
167
|
+
return None
|
|
168
|
+
return datetime.fromtimestamp(
|
|
169
|
+
audit_stamps.created.time / 1000
|
|
170
|
+
) # supports only seconds precision
|
|
171
|
+
|
|
172
|
+
def set_created_at(self, created_at: datetime) -> None:
|
|
173
|
+
"""Set the creation timestamp in audit stamps."""
|
|
174
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
175
|
+
audit_stamps.created.time = make_ts_millis(created_at)
|
|
176
|
+
self._set_audit_stamps(audit_stamps)
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def created_by(self) -> Optional[ActorUrnOrStr]:
|
|
180
|
+
"""Get the creation actor from audit stamps."""
|
|
181
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
182
|
+
if audit_stamps.created.actor == builder.UNKNOWN_USER:
|
|
183
|
+
return None
|
|
184
|
+
return audit_stamps.created.actor
|
|
185
|
+
|
|
186
|
+
def set_created_by(self, created_by: ActorUrnOrStr) -> None:
|
|
187
|
+
"""Set the creation actor in audit stamps."""
|
|
188
|
+
if isinstance(created_by, str):
|
|
189
|
+
created_by = make_user_urn(created_by)
|
|
190
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
191
|
+
audit_stamps.created.actor = str(created_by)
|
|
192
|
+
self._set_audit_stamps(audit_stamps)
|
|
193
|
+
|
|
194
|
+
@property
|
|
195
|
+
def deleted_on(self) -> Optional[datetime]:
|
|
196
|
+
"""Get the deletion timestamp from audit stamps."""
|
|
197
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
198
|
+
if audit_stamps.deleted is None or audit_stamps.deleted.time == 0:
|
|
199
|
+
return None
|
|
200
|
+
return datetime.fromtimestamp(
|
|
201
|
+
audit_stamps.deleted.time / 1000
|
|
202
|
+
) # supports only seconds precision
|
|
203
|
+
|
|
204
|
+
def set_deleted_on(self, deleted_on: datetime) -> None:
|
|
205
|
+
"""Set the deletion timestamp in audit stamps."""
|
|
206
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
207
|
+
# Default constructor sets deleted to None
|
|
208
|
+
if audit_stamps.deleted is None:
|
|
209
|
+
audit_stamps.deleted = models.AuditStampClass(
|
|
210
|
+
time=0, actor=builder.UNKNOWN_USER
|
|
211
|
+
)
|
|
212
|
+
audit_stamps.deleted.time = make_ts_millis(deleted_on)
|
|
213
|
+
self._set_audit_stamps(audit_stamps)
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def deleted_by(self) -> Optional[ActorUrnOrStr]:
|
|
217
|
+
"""Get the deletion actor from audit stamps."""
|
|
218
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
219
|
+
if (
|
|
220
|
+
audit_stamps.deleted is None
|
|
221
|
+
or audit_stamps.deleted.actor == builder.UNKNOWN_USER
|
|
222
|
+
):
|
|
223
|
+
return None
|
|
224
|
+
return audit_stamps.deleted.actor
|
|
225
|
+
|
|
226
|
+
def set_deleted_by(self, deleted_by: ActorUrnOrStr) -> None:
|
|
227
|
+
"""Set the deletion actor in audit stamps."""
|
|
228
|
+
if isinstance(deleted_by, str):
|
|
229
|
+
deleted_by = make_user_urn(deleted_by)
|
|
230
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
231
|
+
if audit_stamps.deleted is None:
|
|
232
|
+
audit_stamps.deleted = models.AuditStampClass(
|
|
233
|
+
time=0, actor=builder.UNKNOWN_USER
|
|
234
|
+
)
|
|
235
|
+
audit_stamps.deleted.actor = str(deleted_by)
|
|
236
|
+
self._set_audit_stamps(audit_stamps)
|
|
237
|
+
|
|
238
|
+
|
|
113
239
|
class HasPlatformInstance(Entity):
|
|
114
240
|
__slots__ = ()
|
|
115
241
|
|
datahub/sdk/chart.py
CHANGED
|
@@ -10,6 +10,8 @@ import datahub.metadata.schema_classes as models
|
|
|
10
10
|
from datahub.emitter.enum_helpers import get_enum_options
|
|
11
11
|
from datahub.metadata.urns import ChartUrn, DatasetUrn, Urn
|
|
12
12
|
from datahub.sdk._shared import (
|
|
13
|
+
ActorUrnOrStr,
|
|
14
|
+
ChangeAuditStampsMixin,
|
|
13
15
|
DataPlatformInstanceUrnOrStr,
|
|
14
16
|
DataPlatformUrnOrStr,
|
|
15
17
|
DatasetUrnOrStr,
|
|
@@ -34,6 +36,7 @@ from datahub.utilities.sentinels import Unset, unset
|
|
|
34
36
|
|
|
35
37
|
|
|
36
38
|
class Chart(
|
|
39
|
+
ChangeAuditStampsMixin,
|
|
37
40
|
HasPlatformInstance,
|
|
38
41
|
HasSubtype,
|
|
39
42
|
HasOwnership,
|
|
@@ -70,6 +73,11 @@ class Chart(
|
|
|
70
73
|
chart_url: Optional[str] = None,
|
|
71
74
|
custom_properties: Optional[Dict[str, str]] = None,
|
|
72
75
|
last_modified: Optional[datetime] = None,
|
|
76
|
+
last_modified_by: Optional[ActorUrnOrStr] = None,
|
|
77
|
+
created_at: Optional[datetime] = None,
|
|
78
|
+
created_by: Optional[ActorUrnOrStr] = None,
|
|
79
|
+
deleted_on: Optional[datetime] = None,
|
|
80
|
+
deleted_by: Optional[ActorUrnOrStr] = None,
|
|
73
81
|
last_refreshed: Optional[datetime] = None,
|
|
74
82
|
chart_type: Optional[Union[str, models.ChartTypeClass]] = None,
|
|
75
83
|
access: Optional[str] = None,
|
|
@@ -94,13 +102,60 @@ class Chart(
|
|
|
94
102
|
self._set_extra_aspects(extra_aspects)
|
|
95
103
|
|
|
96
104
|
self._set_platform_instance(platform, platform_instance)
|
|
97
|
-
|
|
98
105
|
self._ensure_chart_props(display_name=display_name)
|
|
106
|
+
self._init_chart_properties(
|
|
107
|
+
description,
|
|
108
|
+
display_name,
|
|
109
|
+
external_url,
|
|
110
|
+
chart_url,
|
|
111
|
+
custom_properties,
|
|
112
|
+
last_modified,
|
|
113
|
+
last_modified_by,
|
|
114
|
+
created_at,
|
|
115
|
+
created_by,
|
|
116
|
+
last_refreshed,
|
|
117
|
+
deleted_on,
|
|
118
|
+
deleted_by,
|
|
119
|
+
chart_type,
|
|
120
|
+
access,
|
|
121
|
+
input_datasets,
|
|
122
|
+
)
|
|
123
|
+
self._init_standard_aspects(
|
|
124
|
+
parent_container, subtype, owners, links, tags, terms, domain
|
|
125
|
+
)
|
|
99
126
|
|
|
100
|
-
|
|
101
|
-
|
|
127
|
+
@classmethod
|
|
128
|
+
def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
|
|
129
|
+
assert isinstance(urn, ChartUrn)
|
|
130
|
+
entity = cls(
|
|
131
|
+
platform=urn.dashboard_tool,
|
|
132
|
+
name=urn.chart_id,
|
|
133
|
+
)
|
|
134
|
+
return entity._init_from_graph(current_aspects)
|
|
135
|
+
|
|
136
|
+
def _init_chart_properties(
|
|
137
|
+
self,
|
|
138
|
+
description: Optional[str],
|
|
139
|
+
display_name: Optional[str],
|
|
140
|
+
external_url: Optional[str],
|
|
141
|
+
chart_url: Optional[str],
|
|
142
|
+
custom_properties: Optional[Dict[str, str]],
|
|
143
|
+
last_modified: Optional[datetime],
|
|
144
|
+
last_modified_by: Optional[ActorUrnOrStr],
|
|
145
|
+
created_at: Optional[datetime],
|
|
146
|
+
created_by: Optional[ActorUrnOrStr],
|
|
147
|
+
last_refreshed: Optional[datetime],
|
|
148
|
+
deleted_on: Optional[datetime],
|
|
149
|
+
deleted_by: Optional[ActorUrnOrStr],
|
|
150
|
+
chart_type: Optional[Union[str, models.ChartTypeClass]],
|
|
151
|
+
access: Optional[str],
|
|
152
|
+
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]],
|
|
153
|
+
) -> None:
|
|
154
|
+
"""Initialize chart-specific properties."""
|
|
102
155
|
if description is not None:
|
|
103
156
|
self.set_description(description)
|
|
157
|
+
if display_name is not None:
|
|
158
|
+
self.set_display_name(display_name)
|
|
104
159
|
if external_url is not None:
|
|
105
160
|
self.set_external_url(external_url)
|
|
106
161
|
if chart_url is not None:
|
|
@@ -109,6 +164,16 @@ class Chart(
|
|
|
109
164
|
self.set_custom_properties(custom_properties)
|
|
110
165
|
if last_modified is not None:
|
|
111
166
|
self.set_last_modified(last_modified)
|
|
167
|
+
if last_modified_by is not None:
|
|
168
|
+
self.set_last_modified_by(last_modified_by)
|
|
169
|
+
if created_at is not None:
|
|
170
|
+
self.set_created_at(created_at)
|
|
171
|
+
if created_by is not None:
|
|
172
|
+
self.set_created_by(created_by)
|
|
173
|
+
if deleted_on is not None:
|
|
174
|
+
self.set_deleted_on(deleted_on)
|
|
175
|
+
if deleted_by is not None:
|
|
176
|
+
self.set_deleted_by(deleted_by)
|
|
112
177
|
if last_refreshed is not None:
|
|
113
178
|
self.set_last_refreshed(last_refreshed)
|
|
114
179
|
if chart_type is not None:
|
|
@@ -118,6 +183,17 @@ class Chart(
|
|
|
118
183
|
if input_datasets is not None:
|
|
119
184
|
self.set_input_datasets(input_datasets)
|
|
120
185
|
|
|
186
|
+
def _init_standard_aspects(
|
|
187
|
+
self,
|
|
188
|
+
parent_container: ParentContainerInputType | Unset,
|
|
189
|
+
subtype: Optional[str],
|
|
190
|
+
owners: Optional[OwnersInputType],
|
|
191
|
+
links: Optional[LinksInputType],
|
|
192
|
+
tags: Optional[TagsInputType],
|
|
193
|
+
terms: Optional[TermsInputType],
|
|
194
|
+
domain: Optional[DomainInputType],
|
|
195
|
+
) -> None:
|
|
196
|
+
"""Initialize standard aspects."""
|
|
121
197
|
if parent_container is not unset:
|
|
122
198
|
self._set_container(parent_container)
|
|
123
199
|
if subtype is not None:
|
|
@@ -133,15 +209,6 @@ class Chart(
|
|
|
133
209
|
if domain is not None:
|
|
134
210
|
self.set_domain(domain)
|
|
135
211
|
|
|
136
|
-
@classmethod
|
|
137
|
-
def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
|
|
138
|
-
assert isinstance(urn, ChartUrn)
|
|
139
|
-
entity = cls(
|
|
140
|
-
platform=urn.dashboard_tool,
|
|
141
|
-
name=urn.chart_id,
|
|
142
|
-
)
|
|
143
|
-
return entity._init_from_graph(current_aspects)
|
|
144
|
-
|
|
145
212
|
@property
|
|
146
213
|
def urn(self) -> ChartUrn:
|
|
147
214
|
assert isinstance(self._urn, ChartUrn)
|
|
@@ -159,6 +226,14 @@ class Chart(
|
|
|
159
226
|
)
|
|
160
227
|
)
|
|
161
228
|
|
|
229
|
+
def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
|
|
230
|
+
"""Get the audit stamps from the chart properties."""
|
|
231
|
+
return self._ensure_chart_props().lastModified
|
|
232
|
+
|
|
233
|
+
def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
|
|
234
|
+
"""Set the audit stamps on the chart properties."""
|
|
235
|
+
self._ensure_chart_props().lastModified = audit_stamps
|
|
236
|
+
|
|
162
237
|
@property
|
|
163
238
|
def name(self) -> str:
|
|
164
239
|
"""Get the name of the chart."""
|
|
@@ -220,24 +295,6 @@ class Chart(
|
|
|
220
295
|
"""Set the custom properties of the chart."""
|
|
221
296
|
self._ensure_chart_props().customProperties = custom_properties
|
|
222
297
|
|
|
223
|
-
@property
|
|
224
|
-
def last_modified(self) -> Optional[datetime]:
|
|
225
|
-
"""Get the last modification timestamp of the chart."""
|
|
226
|
-
last_modified_time = self._ensure_chart_props().lastModified.lastModified.time
|
|
227
|
-
if not last_modified_time:
|
|
228
|
-
return None
|
|
229
|
-
return datetime.fromtimestamp(last_modified_time)
|
|
230
|
-
|
|
231
|
-
def set_last_modified(self, last_modified: datetime) -> None:
|
|
232
|
-
"""Set the last modification timestamp of the chart."""
|
|
233
|
-
chart_props = self._ensure_chart_props()
|
|
234
|
-
chart_props.lastModified = models.ChangeAuditStampsClass(
|
|
235
|
-
lastModified=models.AuditStampClass(
|
|
236
|
-
time=int(last_modified.timestamp()),
|
|
237
|
-
actor="urn:li:corpuser:datahub",
|
|
238
|
-
)
|
|
239
|
-
)
|
|
240
|
-
|
|
241
298
|
@property
|
|
242
299
|
def last_refreshed(self) -> Optional[datetime]:
|
|
243
300
|
"""Get the last refresh timestamp of the chart."""
|
datahub/sdk/dashboard.py
CHANGED
|
@@ -9,6 +9,8 @@ from typing_extensions import Self
|
|
|
9
9
|
import datahub.metadata.schema_classes as models
|
|
10
10
|
from datahub.metadata.urns import ChartUrn, DashboardUrn, DatasetUrn, Urn
|
|
11
11
|
from datahub.sdk._shared import (
|
|
12
|
+
ActorUrnOrStr,
|
|
13
|
+
ChangeAuditStampsMixin,
|
|
12
14
|
ChartUrnOrStr,
|
|
13
15
|
DashboardUrnOrStr,
|
|
14
16
|
DataPlatformInstanceUrnOrStr,
|
|
@@ -36,6 +38,7 @@ from datahub.utilities.sentinels import Unset, unset
|
|
|
36
38
|
|
|
37
39
|
|
|
38
40
|
class Dashboard(
|
|
41
|
+
ChangeAuditStampsMixin,
|
|
39
42
|
HasPlatformInstance,
|
|
40
43
|
HasSubtype,
|
|
41
44
|
HasOwnership,
|
|
@@ -72,6 +75,11 @@ class Dashboard(
|
|
|
72
75
|
dashboard_url: Optional[str] = None,
|
|
73
76
|
custom_properties: Optional[Dict[str, str]] = None,
|
|
74
77
|
last_modified: Optional[datetime] = None,
|
|
78
|
+
last_modified_by: Optional[ActorUrnOrStr] = None,
|
|
79
|
+
created_at: Optional[datetime] = None,
|
|
80
|
+
created_by: Optional[ActorUrnOrStr] = None,
|
|
81
|
+
deleted_on: Optional[datetime] = None,
|
|
82
|
+
deleted_by: Optional[ActorUrnOrStr] = None,
|
|
75
83
|
last_refreshed: Optional[datetime] = None,
|
|
76
84
|
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]] = None,
|
|
77
85
|
charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]] = None,
|
|
@@ -96,17 +104,48 @@ class Dashboard(
|
|
|
96
104
|
self._set_extra_aspects(extra_aspects)
|
|
97
105
|
|
|
98
106
|
self._set_platform_instance(platform, platform_instance)
|
|
107
|
+
self._ensure_dashboard_props(display_name=display_name)
|
|
108
|
+
|
|
109
|
+
self._init_dashboard_properties(
|
|
110
|
+
description,
|
|
111
|
+
display_name,
|
|
112
|
+
external_url,
|
|
113
|
+
dashboard_url,
|
|
114
|
+
custom_properties,
|
|
115
|
+
last_modified,
|
|
116
|
+
last_modified_by,
|
|
117
|
+
created_at,
|
|
118
|
+
created_by,
|
|
119
|
+
last_refreshed,
|
|
120
|
+
deleted_on,
|
|
121
|
+
deleted_by,
|
|
122
|
+
input_datasets,
|
|
123
|
+
charts,
|
|
124
|
+
dashboards,
|
|
125
|
+
)
|
|
126
|
+
self._init_standard_aspects(
|
|
127
|
+
parent_container, subtype, owners, links, tags, terms, domain
|
|
128
|
+
)
|
|
99
129
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
130
|
+
def _init_dashboard_properties(
|
|
131
|
+
self,
|
|
132
|
+
description: Optional[str],
|
|
133
|
+
display_name: Optional[str],
|
|
134
|
+
external_url: Optional[str],
|
|
135
|
+
dashboard_url: Optional[str],
|
|
136
|
+
custom_properties: Optional[Dict[str, str]],
|
|
137
|
+
last_modified: Optional[datetime],
|
|
138
|
+
last_modified_by: Optional[ActorUrnOrStr],
|
|
139
|
+
created_at: Optional[datetime],
|
|
140
|
+
created_by: Optional[ActorUrnOrStr],
|
|
141
|
+
last_refreshed: Optional[datetime],
|
|
142
|
+
deleted_on: Optional[datetime],
|
|
143
|
+
deleted_by: Optional[ActorUrnOrStr],
|
|
144
|
+
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]],
|
|
145
|
+
charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]],
|
|
146
|
+
dashboards: Optional[Sequence[Union[DashboardUrnOrStr, Dashboard]]],
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Initialize dashboard-specific properties."""
|
|
110
149
|
if description is not None:
|
|
111
150
|
self.set_description(description)
|
|
112
151
|
if display_name is not None:
|
|
@@ -119,6 +158,16 @@ class Dashboard(
|
|
|
119
158
|
self.set_custom_properties(custom_properties)
|
|
120
159
|
if last_modified is not None:
|
|
121
160
|
self.set_last_modified(last_modified)
|
|
161
|
+
if last_modified_by is not None:
|
|
162
|
+
self.set_last_modified_by(last_modified_by)
|
|
163
|
+
if created_at is not None:
|
|
164
|
+
self.set_created_at(created_at)
|
|
165
|
+
if created_by is not None:
|
|
166
|
+
self.set_created_by(created_by)
|
|
167
|
+
if deleted_on is not None:
|
|
168
|
+
self.set_deleted_on(deleted_on)
|
|
169
|
+
if deleted_by is not None:
|
|
170
|
+
self.set_deleted_by(deleted_by)
|
|
122
171
|
if last_refreshed is not None:
|
|
123
172
|
self.set_last_refreshed(last_refreshed)
|
|
124
173
|
if input_datasets is not None:
|
|
@@ -128,6 +177,17 @@ class Dashboard(
|
|
|
128
177
|
if dashboards is not None:
|
|
129
178
|
self.set_dashboards(dashboards)
|
|
130
179
|
|
|
180
|
+
def _init_standard_aspects(
|
|
181
|
+
self,
|
|
182
|
+
parent_container: ParentContainerInputType | Unset,
|
|
183
|
+
subtype: Optional[str],
|
|
184
|
+
owners: Optional[OwnersInputType],
|
|
185
|
+
links: Optional[LinksInputType],
|
|
186
|
+
tags: Optional[TagsInputType],
|
|
187
|
+
terms: Optional[TermsInputType],
|
|
188
|
+
domain: Optional[DomainInputType],
|
|
189
|
+
) -> None:
|
|
190
|
+
"""Initialize standard aspects."""
|
|
131
191
|
if parent_container is not unset:
|
|
132
192
|
self._set_container(parent_container)
|
|
133
193
|
if subtype is not None:
|
|
@@ -165,18 +225,20 @@ class Dashboard(
|
|
|
165
225
|
models.DashboardInfoClass(
|
|
166
226
|
title=display_name or self.urn.dashboard_id,
|
|
167
227
|
description="",
|
|
168
|
-
lastModified=models.ChangeAuditStampsClass(
|
|
169
|
-
lastModified=models.AuditStampClass(
|
|
170
|
-
time=0, actor="urn:li:corpuser:unknown"
|
|
171
|
-
)
|
|
172
|
-
),
|
|
228
|
+
lastModified=models.ChangeAuditStampsClass(),
|
|
173
229
|
customProperties={},
|
|
174
|
-
chartEdges=[],
|
|
175
|
-
datasetEdges=[],
|
|
176
230
|
dashboards=[],
|
|
177
231
|
)
|
|
178
232
|
)
|
|
179
233
|
|
|
234
|
+
def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
|
|
235
|
+
"""Get the audit stamps from the dashboard properties."""
|
|
236
|
+
return self._ensure_dashboard_props().lastModified
|
|
237
|
+
|
|
238
|
+
def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
|
|
239
|
+
"""Set the audit stamps on the dashboard properties."""
|
|
240
|
+
self._ensure_dashboard_props().lastModified = audit_stamps
|
|
241
|
+
|
|
180
242
|
@property
|
|
181
243
|
def name(self) -> str:
|
|
182
244
|
"""Get the name of the dashboard."""
|
|
@@ -240,23 +302,6 @@ class Dashboard(
|
|
|
240
302
|
"""Set the custom properties of the dashboard."""
|
|
241
303
|
self._ensure_dashboard_props().customProperties = custom_properties
|
|
242
304
|
|
|
243
|
-
@property
|
|
244
|
-
def last_modified(self) -> Optional[datetime]:
|
|
245
|
-
"""Get the last modification timestamp of the dashboard."""
|
|
246
|
-
props = self._ensure_dashboard_props()
|
|
247
|
-
if props.lastModified.lastModified.time == 0:
|
|
248
|
-
return None
|
|
249
|
-
return datetime.fromtimestamp(props.lastModified.lastModified.time)
|
|
250
|
-
|
|
251
|
-
def set_last_modified(self, last_modified: datetime) -> None:
|
|
252
|
-
"""Set the last modification timestamp of the dashboard."""
|
|
253
|
-
self._ensure_dashboard_props().lastModified = models.ChangeAuditStampsClass(
|
|
254
|
-
lastModified=models.AuditStampClass(
|
|
255
|
-
time=int(last_modified.timestamp()),
|
|
256
|
-
actor="urn:li:corpuser:datahub",
|
|
257
|
-
),
|
|
258
|
-
)
|
|
259
|
-
|
|
260
305
|
@property
|
|
261
306
|
def last_refreshed(self) -> Optional[datetime]:
|
|
262
307
|
"""Get the last refresh timestamp of the dashboard."""
|