acryl-datahub 0.15.0.5rc7__py3-none-any.whl → 0.15.0.5rc9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/METADATA +2493 -2463
- {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/RECORD +38 -35
- datahub/_version.py +1 -1
- datahub/cli/iceberg_cli.py +707 -0
- datahub/entrypoints.py +21 -0
- datahub/ingestion/api/incremental_lineage_helper.py +4 -0
- datahub/ingestion/glossary/classification_mixin.py +6 -0
- datahub/ingestion/glossary/classifier.py +3 -2
- datahub/ingestion/source/aws/glue.py +3 -2
- datahub/ingestion/source/identity/azure_ad.py +6 -14
- datahub/ingestion/source/mode.py +2 -4
- datahub/ingestion/source/snowflake/snowflake_config.py +13 -0
- datahub/ingestion/source/snowflake/snowflake_query.py +11 -0
- datahub/ingestion/source/snowflake/snowflake_report.py +1 -0
- datahub/ingestion/source/snowflake/snowflake_schema.py +17 -0
- datahub/ingestion/source/snowflake/snowflake_schema_gen.py +18 -36
- datahub/ingestion/source/snowflake/snowflake_tag.py +57 -3
- datahub/ingestion/source/snowflake/snowflake_v2.py +1 -0
- datahub/ingestion/source/sql/mssql/job_models.py +37 -8
- datahub/ingestion/source/sql/mssql/source.py +17 -0
- datahub/ingestion/source/tableau/tableau.py +14 -12
- datahub/ingestion/source/tableau/tableau_common.py +1 -1
- datahub/metadata/_schema_classes.py +160 -2
- datahub/metadata/com/linkedin/pegasus2avro/dataplatforminstance/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/dataset/__init__.py +2 -0
- datahub/metadata/schema.avsc +96 -7
- datahub/metadata/schemas/DashboardInfo.avsc +5 -5
- datahub/metadata/schemas/DataPlatformInstanceKey.avsc +2 -1
- datahub/metadata/schemas/DatasetKey.avsc +2 -1
- datahub/metadata/schemas/IcebergCatalogInfo.avsc +28 -0
- datahub/metadata/schemas/IcebergWarehouseInfo.avsc +92 -0
- datahub/metadata/schemas/MetadataChangeEvent.avsc +5 -5
- datahub/specific/dashboard.py +43 -1
- datahub/upgrade/upgrade.py +13 -5
- {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/LICENSE +0 -0
- {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/WHEEL +0 -0
- {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/top_level.txt +0 -0
|
@@ -9149,6 +9149,113 @@ class DataPlatformInstancePropertiesClass(_Aspect):
|
|
|
9149
9149
|
self._inner_dict['description'] = value
|
|
9150
9150
|
|
|
9151
9151
|
|
|
9152
|
+
class IcebergWarehouseInfoClass(_Aspect):
|
|
9153
|
+
"""An Iceberg warehouse location and credentails whose read/writes are governed by datahub catalog."""
|
|
9154
|
+
|
|
9155
|
+
|
|
9156
|
+
ASPECT_NAME = 'icebergWarehouseInfo'
|
|
9157
|
+
ASPECT_INFO = {}
|
|
9158
|
+
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.dataplatforminstance.IcebergWarehouseInfo")
|
|
9159
|
+
|
|
9160
|
+
def __init__(self,
|
|
9161
|
+
dataRoot: str,
|
|
9162
|
+
clientId: str,
|
|
9163
|
+
clientSecret: str,
|
|
9164
|
+
region: str,
|
|
9165
|
+
env: Union[str, "FabricTypeClass"],
|
|
9166
|
+
role: Union[None, str]=None,
|
|
9167
|
+
tempCredentialExpirationSeconds: Union[None, int]=None,
|
|
9168
|
+
):
|
|
9169
|
+
super().__init__()
|
|
9170
|
+
|
|
9171
|
+
self.dataRoot = dataRoot
|
|
9172
|
+
self.clientId = clientId
|
|
9173
|
+
self.clientSecret = clientSecret
|
|
9174
|
+
self.region = region
|
|
9175
|
+
self.role = role
|
|
9176
|
+
self.tempCredentialExpirationSeconds = tempCredentialExpirationSeconds
|
|
9177
|
+
self.env = env
|
|
9178
|
+
|
|
9179
|
+
def _restore_defaults(self) -> None:
|
|
9180
|
+
self.dataRoot = str()
|
|
9181
|
+
self.clientId = str()
|
|
9182
|
+
self.clientSecret = str()
|
|
9183
|
+
self.region = str()
|
|
9184
|
+
self.role = self.RECORD_SCHEMA.fields_dict["role"].default
|
|
9185
|
+
self.tempCredentialExpirationSeconds = self.RECORD_SCHEMA.fields_dict["tempCredentialExpirationSeconds"].default
|
|
9186
|
+
self.env = FabricTypeClass.DEV
|
|
9187
|
+
|
|
9188
|
+
|
|
9189
|
+
@property
|
|
9190
|
+
def dataRoot(self) -> str:
|
|
9191
|
+
"""Path of the root for the backing store of the tables in the warehouse."""
|
|
9192
|
+
return self._inner_dict.get('dataRoot') # type: ignore
|
|
9193
|
+
|
|
9194
|
+
@dataRoot.setter
|
|
9195
|
+
def dataRoot(self, value: str) -> None:
|
|
9196
|
+
self._inner_dict['dataRoot'] = value
|
|
9197
|
+
|
|
9198
|
+
|
|
9199
|
+
@property
|
|
9200
|
+
def clientId(self) -> str:
|
|
9201
|
+
"""clientId to be used to authenticate with storage hosting this warehouse"""
|
|
9202
|
+
return self._inner_dict.get('clientId') # type: ignore
|
|
9203
|
+
|
|
9204
|
+
@clientId.setter
|
|
9205
|
+
def clientId(self, value: str) -> None:
|
|
9206
|
+
self._inner_dict['clientId'] = value
|
|
9207
|
+
|
|
9208
|
+
|
|
9209
|
+
@property
|
|
9210
|
+
def clientSecret(self) -> str:
|
|
9211
|
+
"""client secret to authenticate with storage hosting this warehouse"""
|
|
9212
|
+
return self._inner_dict.get('clientSecret') # type: ignore
|
|
9213
|
+
|
|
9214
|
+
@clientSecret.setter
|
|
9215
|
+
def clientSecret(self, value: str) -> None:
|
|
9216
|
+
self._inner_dict['clientSecret'] = value
|
|
9217
|
+
|
|
9218
|
+
|
|
9219
|
+
@property
|
|
9220
|
+
def region(self) -> str:
|
|
9221
|
+
"""region where the warehouse is located."""
|
|
9222
|
+
return self._inner_dict.get('region') # type: ignore
|
|
9223
|
+
|
|
9224
|
+
@region.setter
|
|
9225
|
+
def region(self, value: str) -> None:
|
|
9226
|
+
self._inner_dict['region'] = value
|
|
9227
|
+
|
|
9228
|
+
|
|
9229
|
+
@property
|
|
9230
|
+
def role(self) -> Union[None, str]:
|
|
9231
|
+
# No docs available.
|
|
9232
|
+
return self._inner_dict.get('role') # type: ignore
|
|
9233
|
+
|
|
9234
|
+
@role.setter
|
|
9235
|
+
def role(self, value: Union[None, str]) -> None:
|
|
9236
|
+
self._inner_dict['role'] = value
|
|
9237
|
+
|
|
9238
|
+
|
|
9239
|
+
@property
|
|
9240
|
+
def tempCredentialExpirationSeconds(self) -> Union[None, int]:
|
|
9241
|
+
# No docs available.
|
|
9242
|
+
return self._inner_dict.get('tempCredentialExpirationSeconds') # type: ignore
|
|
9243
|
+
|
|
9244
|
+
@tempCredentialExpirationSeconds.setter
|
|
9245
|
+
def tempCredentialExpirationSeconds(self, value: Union[None, int]) -> None:
|
|
9246
|
+
self._inner_dict['tempCredentialExpirationSeconds'] = value
|
|
9247
|
+
|
|
9248
|
+
|
|
9249
|
+
@property
|
|
9250
|
+
def env(self) -> Union[str, "FabricTypeClass"]:
|
|
9251
|
+
# No docs available.
|
|
9252
|
+
return self._inner_dict.get('env') # type: ignore
|
|
9253
|
+
|
|
9254
|
+
@env.setter
|
|
9255
|
+
def env(self, value: Union[str, "FabricTypeClass"]) -> None:
|
|
9256
|
+
self._inner_dict['env'] = value
|
|
9257
|
+
|
|
9258
|
+
|
|
9152
9259
|
class DataProcessInfoClass(_Aspect):
|
|
9153
9260
|
"""The inputs and outputs of this data process"""
|
|
9154
9261
|
|
|
@@ -10992,6 +11099,49 @@ class HistogramClass(DictWrapper):
|
|
|
10992
11099
|
self._inner_dict['heights'] = value
|
|
10993
11100
|
|
|
10994
11101
|
|
|
11102
|
+
class IcebergCatalogInfoClass(_Aspect):
|
|
11103
|
+
"""Iceberg Catalog metadata associated with an Iceberg table/view"""
|
|
11104
|
+
|
|
11105
|
+
|
|
11106
|
+
ASPECT_NAME = 'icebergCatalogInfo'
|
|
11107
|
+
ASPECT_INFO = {}
|
|
11108
|
+
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.dataset.IcebergCatalogInfo")
|
|
11109
|
+
|
|
11110
|
+
def __init__(self,
|
|
11111
|
+
metadataPointer: Union[None, str]=None,
|
|
11112
|
+
view: Union[None, bool]=None,
|
|
11113
|
+
):
|
|
11114
|
+
super().__init__()
|
|
11115
|
+
|
|
11116
|
+
self.metadataPointer = metadataPointer
|
|
11117
|
+
self.view = view
|
|
11118
|
+
|
|
11119
|
+
def _restore_defaults(self) -> None:
|
|
11120
|
+
self.metadataPointer = self.RECORD_SCHEMA.fields_dict["metadataPointer"].default
|
|
11121
|
+
self.view = self.RECORD_SCHEMA.fields_dict["view"].default
|
|
11122
|
+
|
|
11123
|
+
|
|
11124
|
+
@property
|
|
11125
|
+
def metadataPointer(self) -> Union[None, str]:
|
|
11126
|
+
"""When Datahub is the REST Catalog for an Iceberg Table, stores the current metadata pointer.
|
|
11127
|
+
If the Iceberg table is managed by an external catalog, the metadata pointer is not set."""
|
|
11128
|
+
return self._inner_dict.get('metadataPointer') # type: ignore
|
|
11129
|
+
|
|
11130
|
+
@metadataPointer.setter
|
|
11131
|
+
def metadataPointer(self, value: Union[None, str]) -> None:
|
|
11132
|
+
self._inner_dict['metadataPointer'] = value
|
|
11133
|
+
|
|
11134
|
+
|
|
11135
|
+
@property
|
|
11136
|
+
def view(self) -> Union[None, bool]:
|
|
11137
|
+
# No docs available.
|
|
11138
|
+
return self._inner_dict.get('view') # type: ignore
|
|
11139
|
+
|
|
11140
|
+
@view.setter
|
|
11141
|
+
def view(self, value: Union[None, bool]) -> None:
|
|
11142
|
+
self._inner_dict['view'] = value
|
|
11143
|
+
|
|
11144
|
+
|
|
10995
11145
|
class PartitionSummaryClass(DictWrapper):
|
|
10996
11146
|
"""Defines how the data is partitioned"""
|
|
10997
11147
|
|
|
@@ -14987,7 +15137,7 @@ class DataPlatformInstanceKeyClass(_Aspect):
|
|
|
14987
15137
|
|
|
14988
15138
|
|
|
14989
15139
|
ASPECT_NAME = 'dataPlatformInstanceKey'
|
|
14990
|
-
ASPECT_INFO = {'keyForEntity': 'dataPlatformInstance', 'entityCategory': 'internal', 'entityAspects': ['dataPlatformInstanceProperties', 'ownership', 'globalTags', 'institutionalMemory', 'deprecation', 'status']}
|
|
15140
|
+
ASPECT_INFO = {'keyForEntity': 'dataPlatformInstance', 'entityCategory': 'internal', 'entityAspects': ['dataPlatformInstanceProperties', 'ownership', 'globalTags', 'institutionalMemory', 'deprecation', 'status', 'icebergWarehouseInfo']}
|
|
14991
15141
|
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DataPlatformInstanceKey")
|
|
14992
15142
|
|
|
14993
15143
|
def __init__(self,
|
|
@@ -15143,7 +15293,7 @@ class DatasetKeyClass(_Aspect):
|
|
|
15143
15293
|
|
|
15144
15294
|
|
|
15145
15295
|
ASPECT_NAME = 'datasetKey'
|
|
15146
|
-
ASPECT_INFO = {'keyForEntity': 'dataset', 'entityCategory': 'core', 'entityAspects': ['viewProperties', 'subTypes', 'datasetProfile', 'datasetUsageStatistics', 'operation', 'domains', 'schemaMetadata', 'status', 'container', 'deprecation', 'testResults', 'siblings', 'embed', 'incidentsSummary', 'datasetProperties', 'editableDatasetProperties', 'datasetDeprecation', 'datasetUpstreamLineage', 'upstreamLineage', 'institutionalMemory', 'ownership', 'editableSchemaMetadata', 'globalTags', 'glossaryTerms', 'browsePaths', 'dataPlatformInstance', 'browsePathsV2', 'access', 'structuredProperties', 'forms', 'partitionsSummary', 'versionProperties'], 'entityDoc': 'Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets.'}
|
|
15296
|
+
ASPECT_INFO = {'keyForEntity': 'dataset', 'entityCategory': 'core', 'entityAspects': ['viewProperties', 'subTypes', 'datasetProfile', 'datasetUsageStatistics', 'operation', 'domains', 'schemaMetadata', 'status', 'container', 'deprecation', 'testResults', 'siblings', 'embed', 'incidentsSummary', 'datasetProperties', 'editableDatasetProperties', 'datasetDeprecation', 'datasetUpstreamLineage', 'upstreamLineage', 'institutionalMemory', 'ownership', 'editableSchemaMetadata', 'globalTags', 'glossaryTerms', 'browsePaths', 'dataPlatformInstance', 'browsePathsV2', 'access', 'structuredProperties', 'forms', 'partitionsSummary', 'versionProperties', 'icebergCatalogInfo'], 'entityDoc': 'Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets.'}
|
|
15147
15297
|
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DatasetKey")
|
|
15148
15298
|
|
|
15149
15299
|
def __init__(self,
|
|
@@ -25245,6 +25395,7 @@ __SCHEMA_TYPES = {
|
|
|
25245
25395
|
'com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo': DataPlatformInfoClass,
|
|
25246
25396
|
'com.linkedin.pegasus2avro.dataplatform.PlatformType': PlatformTypeClass,
|
|
25247
25397
|
'com.linkedin.pegasus2avro.dataplatforminstance.DataPlatformInstanceProperties': DataPlatformInstancePropertiesClass,
|
|
25398
|
+
'com.linkedin.pegasus2avro.dataplatforminstance.IcebergWarehouseInfo': IcebergWarehouseInfoClass,
|
|
25248
25399
|
'com.linkedin.pegasus2avro.dataprocess.DataProcessInfo': DataProcessInfoClass,
|
|
25249
25400
|
'com.linkedin.pegasus2avro.dataprocess.DataProcessInstanceInput': DataProcessInstanceInputClass,
|
|
25250
25401
|
'com.linkedin.pegasus2avro.dataprocess.DataProcessInstanceOutput': DataProcessInstanceOutputClass,
|
|
@@ -25275,6 +25426,7 @@ __SCHEMA_TYPES = {
|
|
|
25275
25426
|
'com.linkedin.pegasus2avro.dataset.FineGrainedLineageDownstreamType': FineGrainedLineageDownstreamTypeClass,
|
|
25276
25427
|
'com.linkedin.pegasus2avro.dataset.FineGrainedLineageUpstreamType': FineGrainedLineageUpstreamTypeClass,
|
|
25277
25428
|
'com.linkedin.pegasus2avro.dataset.Histogram': HistogramClass,
|
|
25429
|
+
'com.linkedin.pegasus2avro.dataset.IcebergCatalogInfo': IcebergCatalogInfoClass,
|
|
25278
25430
|
'com.linkedin.pegasus2avro.dataset.PartitionSummary': PartitionSummaryClass,
|
|
25279
25431
|
'com.linkedin.pegasus2avro.dataset.PartitionsSummary': PartitionsSummaryClass,
|
|
25280
25432
|
'com.linkedin.pegasus2avro.dataset.Quantile': QuantileClass,
|
|
@@ -25721,6 +25873,7 @@ __SCHEMA_TYPES = {
|
|
|
25721
25873
|
'DataPlatformInfo': DataPlatformInfoClass,
|
|
25722
25874
|
'PlatformType': PlatformTypeClass,
|
|
25723
25875
|
'DataPlatformInstanceProperties': DataPlatformInstancePropertiesClass,
|
|
25876
|
+
'IcebergWarehouseInfo': IcebergWarehouseInfoClass,
|
|
25724
25877
|
'DataProcessInfo': DataProcessInfoClass,
|
|
25725
25878
|
'DataProcessInstanceInput': DataProcessInstanceInputClass,
|
|
25726
25879
|
'DataProcessInstanceOutput': DataProcessInstanceOutputClass,
|
|
@@ -25751,6 +25904,7 @@ __SCHEMA_TYPES = {
|
|
|
25751
25904
|
'FineGrainedLineageDownstreamType': FineGrainedLineageDownstreamTypeClass,
|
|
25752
25905
|
'FineGrainedLineageUpstreamType': FineGrainedLineageUpstreamTypeClass,
|
|
25753
25906
|
'Histogram': HistogramClass,
|
|
25907
|
+
'IcebergCatalogInfo': IcebergCatalogInfoClass,
|
|
25754
25908
|
'PartitionSummary': PartitionSummaryClass,
|
|
25755
25909
|
'PartitionsSummary': PartitionsSummaryClass,
|
|
25756
25910
|
'Quantile': QuantileClass,
|
|
@@ -26164,6 +26318,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
|
|
|
26164
26318
|
QuerySubjectsClass,
|
|
26165
26319
|
QueryUsageStatisticsClass,
|
|
26166
26320
|
DataHubSecretValueClass,
|
|
26321
|
+
IcebergCatalogInfoClass,
|
|
26167
26322
|
DatasetUpstreamLineageClass,
|
|
26168
26323
|
UpstreamLineageClass,
|
|
26169
26324
|
DatasetProfileClass,
|
|
@@ -26252,6 +26407,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
|
|
|
26252
26407
|
EntityTypeKeyClass,
|
|
26253
26408
|
OwnershipTypeInfoClass,
|
|
26254
26409
|
DataPlatformInstancePropertiesClass,
|
|
26410
|
+
IcebergWarehouseInfoClass,
|
|
26255
26411
|
DataHubAccessTokenInfoClass
|
|
26256
26412
|
]
|
|
26257
26413
|
|
|
@@ -26383,6 +26539,7 @@ class AspectBag(TypedDict, total=False):
|
|
|
26383
26539
|
querySubjects: QuerySubjectsClass
|
|
26384
26540
|
queryUsageStatistics: QueryUsageStatisticsClass
|
|
26385
26541
|
dataHubSecretValue: DataHubSecretValueClass
|
|
26542
|
+
icebergCatalogInfo: IcebergCatalogInfoClass
|
|
26386
26543
|
datasetUpstreamLineage: DatasetUpstreamLineageClass
|
|
26387
26544
|
upstreamLineage: UpstreamLineageClass
|
|
26388
26545
|
datasetProfile: DatasetProfileClass
|
|
@@ -26471,6 +26628,7 @@ class AspectBag(TypedDict, total=False):
|
|
|
26471
26628
|
entityTypeKey: EntityTypeKeyClass
|
|
26472
26629
|
ownershipTypeInfo: OwnershipTypeInfoClass
|
|
26473
26630
|
dataPlatformInstanceProperties: DataPlatformInstancePropertiesClass
|
|
26631
|
+
icebergWarehouseInfo: IcebergWarehouseInfoClass
|
|
26474
26632
|
dataHubAccessTokenInfo: DataHubAccessTokenInfoClass
|
|
26475
26633
|
|
|
26476
26634
|
|
|
@@ -8,8 +8,10 @@
|
|
|
8
8
|
# fmt: off
|
|
9
9
|
# isort: skip_file
|
|
10
10
|
from .....schema_classes import DataPlatformInstancePropertiesClass
|
|
11
|
+
from .....schema_classes import IcebergWarehouseInfoClass
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
DataPlatformInstanceProperties = DataPlatformInstancePropertiesClass
|
|
15
|
+
IcebergWarehouseInfo = IcebergWarehouseInfoClass
|
|
14
16
|
|
|
15
17
|
# fmt: on
|
|
@@ -24,6 +24,7 @@ from .....schema_classes import FineGrainedLineageClass
|
|
|
24
24
|
from .....schema_classes import FineGrainedLineageDownstreamTypeClass
|
|
25
25
|
from .....schema_classes import FineGrainedLineageUpstreamTypeClass
|
|
26
26
|
from .....schema_classes import HistogramClass
|
|
27
|
+
from .....schema_classes import IcebergCatalogInfoClass
|
|
27
28
|
from .....schema_classes import PartitionSummaryClass
|
|
28
29
|
from .....schema_classes import PartitionsSummaryClass
|
|
29
30
|
from .....schema_classes import QuantileClass
|
|
@@ -50,6 +51,7 @@ FineGrainedLineage = FineGrainedLineageClass
|
|
|
50
51
|
FineGrainedLineageDownstreamType = FineGrainedLineageDownstreamTypeClass
|
|
51
52
|
FineGrainedLineageUpstreamType = FineGrainedLineageUpstreamTypeClass
|
|
52
53
|
Histogram = HistogramClass
|
|
54
|
+
IcebergCatalogInfo = IcebergCatalogInfoClass
|
|
53
55
|
PartitionSummary = PartitionSummaryClass
|
|
54
56
|
PartitionsSummary = PartitionsSummaryClass
|
|
55
57
|
Quantile = QuantileClass
|
datahub/metadata/schema.avsc
CHANGED
|
@@ -4730,16 +4730,16 @@
|
|
|
4730
4730
|
{
|
|
4731
4731
|
"Relationship": {
|
|
4732
4732
|
"/*/destinationUrn": {
|
|
4733
|
-
"createdActor": "
|
|
4734
|
-
"createdOn": "
|
|
4733
|
+
"createdActor": "dashboards/*/created/actor",
|
|
4734
|
+
"createdOn": "dashboards/*/created/time",
|
|
4735
4735
|
"entityTypes": [
|
|
4736
4736
|
"dashboard"
|
|
4737
4737
|
],
|
|
4738
4738
|
"isLineage": true,
|
|
4739
4739
|
"name": "DashboardContainsDashboard",
|
|
4740
|
-
"properties": "
|
|
4741
|
-
"updatedActor": "
|
|
4742
|
-
"updatedOn": "
|
|
4740
|
+
"properties": "dashboards/*/properties",
|
|
4741
|
+
"updatedActor": "dashboards/*/lastModified/actor",
|
|
4742
|
+
"updatedOn": "dashboards/*/lastModified/time"
|
|
4743
4743
|
}
|
|
4744
4744
|
},
|
|
4745
4745
|
"type": {
|
|
@@ -13109,6 +13109,34 @@
|
|
|
13109
13109
|
],
|
|
13110
13110
|
"doc": "The value of a DataHub Secret"
|
|
13111
13111
|
},
|
|
13112
|
+
{
|
|
13113
|
+
"type": "record",
|
|
13114
|
+
"Aspect": {
|
|
13115
|
+
"name": "icebergCatalogInfo"
|
|
13116
|
+
},
|
|
13117
|
+
"name": "IcebergCatalogInfo",
|
|
13118
|
+
"namespace": "com.linkedin.pegasus2avro.dataset",
|
|
13119
|
+
"fields": [
|
|
13120
|
+
{
|
|
13121
|
+
"type": [
|
|
13122
|
+
"null",
|
|
13123
|
+
"string"
|
|
13124
|
+
],
|
|
13125
|
+
"name": "metadataPointer",
|
|
13126
|
+
"default": null,
|
|
13127
|
+
"doc": "When Datahub is the REST Catalog for an Iceberg Table, stores the current metadata pointer.\nIf the Iceberg table is managed by an external catalog, the metadata pointer is not set."
|
|
13128
|
+
},
|
|
13129
|
+
{
|
|
13130
|
+
"type": [
|
|
13131
|
+
"null",
|
|
13132
|
+
"boolean"
|
|
13133
|
+
],
|
|
13134
|
+
"name": "view",
|
|
13135
|
+
"default": null
|
|
13136
|
+
}
|
|
13137
|
+
],
|
|
13138
|
+
"doc": "Iceberg Catalog metadata associated with an Iceberg table/view"
|
|
13139
|
+
},
|
|
13112
13140
|
{
|
|
13113
13141
|
"type": "record",
|
|
13114
13142
|
"Aspect": {
|
|
@@ -16692,7 +16720,8 @@
|
|
|
16692
16720
|
"structuredProperties",
|
|
16693
16721
|
"forms",
|
|
16694
16722
|
"partitionsSummary",
|
|
16695
|
-
"versionProperties"
|
|
16723
|
+
"versionProperties",
|
|
16724
|
+
"icebergCatalogInfo"
|
|
16696
16725
|
],
|
|
16697
16726
|
"entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
|
|
16698
16727
|
},
|
|
@@ -17393,7 +17422,8 @@
|
|
|
17393
17422
|
"globalTags",
|
|
17394
17423
|
"institutionalMemory",
|
|
17395
17424
|
"deprecation",
|
|
17396
|
-
"status"
|
|
17425
|
+
"status",
|
|
17426
|
+
"icebergWarehouseInfo"
|
|
17397
17427
|
]
|
|
17398
17428
|
},
|
|
17399
17429
|
"name": "DataPlatformInstanceKey",
|
|
@@ -18144,6 +18174,65 @@
|
|
|
18144
18174
|
],
|
|
18145
18175
|
"doc": "Properties associated with a Data Platform Instance"
|
|
18146
18176
|
},
|
|
18177
|
+
{
|
|
18178
|
+
"type": "record",
|
|
18179
|
+
"Aspect": {
|
|
18180
|
+
"name": "icebergWarehouseInfo"
|
|
18181
|
+
},
|
|
18182
|
+
"name": "IcebergWarehouseInfo",
|
|
18183
|
+
"namespace": "com.linkedin.pegasus2avro.dataplatforminstance",
|
|
18184
|
+
"fields": [
|
|
18185
|
+
{
|
|
18186
|
+
"type": "string",
|
|
18187
|
+
"name": "dataRoot",
|
|
18188
|
+
"doc": "Path of the root for the backing store of the tables in the warehouse."
|
|
18189
|
+
},
|
|
18190
|
+
{
|
|
18191
|
+
"java": {
|
|
18192
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
18193
|
+
},
|
|
18194
|
+
"Urn": "Urn",
|
|
18195
|
+
"type": "string",
|
|
18196
|
+
"name": "clientId",
|
|
18197
|
+
"doc": "clientId to be used to authenticate with storage hosting this warehouse"
|
|
18198
|
+
},
|
|
18199
|
+
{
|
|
18200
|
+
"java": {
|
|
18201
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
18202
|
+
},
|
|
18203
|
+
"Urn": "Urn",
|
|
18204
|
+
"type": "string",
|
|
18205
|
+
"name": "clientSecret",
|
|
18206
|
+
"doc": "client secret to authenticate with storage hosting this warehouse"
|
|
18207
|
+
},
|
|
18208
|
+
{
|
|
18209
|
+
"type": "string",
|
|
18210
|
+
"name": "region",
|
|
18211
|
+
"doc": "region where the warehouse is located."
|
|
18212
|
+
},
|
|
18213
|
+
{
|
|
18214
|
+
"type": [
|
|
18215
|
+
"null",
|
|
18216
|
+
"string"
|
|
18217
|
+
],
|
|
18218
|
+
"name": "role",
|
|
18219
|
+
"default": null
|
|
18220
|
+
},
|
|
18221
|
+
{
|
|
18222
|
+
"type": [
|
|
18223
|
+
"null",
|
|
18224
|
+
"int"
|
|
18225
|
+
],
|
|
18226
|
+
"name": "tempCredentialExpirationSeconds",
|
|
18227
|
+
"default": null
|
|
18228
|
+
},
|
|
18229
|
+
{
|
|
18230
|
+
"type": "com.linkedin.pegasus2avro.common.FabricType",
|
|
18231
|
+
"name": "env"
|
|
18232
|
+
}
|
|
18233
|
+
],
|
|
18234
|
+
"doc": "An Iceberg warehouse location and credentails whose read/writes are governed by datahub catalog."
|
|
18235
|
+
},
|
|
18147
18236
|
{
|
|
18148
18237
|
"type": "record",
|
|
18149
18238
|
"deprecated": "Use DatasetUsageStatistics, or other UsageStatistics records, instead",
|
|
@@ -258,16 +258,16 @@
|
|
|
258
258
|
{
|
|
259
259
|
"Relationship": {
|
|
260
260
|
"/*/destinationUrn": {
|
|
261
|
-
"createdActor": "
|
|
262
|
-
"createdOn": "
|
|
261
|
+
"createdActor": "dashboards/*/created/actor",
|
|
262
|
+
"createdOn": "dashboards/*/created/time",
|
|
263
263
|
"entityTypes": [
|
|
264
264
|
"dashboard"
|
|
265
265
|
],
|
|
266
266
|
"isLineage": true,
|
|
267
267
|
"name": "DashboardContainsDashboard",
|
|
268
|
-
"properties": "
|
|
269
|
-
"updatedActor": "
|
|
270
|
-
"updatedOn": "
|
|
268
|
+
"properties": "dashboards/*/properties",
|
|
269
|
+
"updatedActor": "dashboards/*/lastModified/actor",
|
|
270
|
+
"updatedOn": "dashboards/*/lastModified/time"
|
|
271
271
|
}
|
|
272
272
|
},
|
|
273
273
|
"type": {
|
|
@@ -36,7 +36,8 @@
|
|
|
36
36
|
"structuredProperties",
|
|
37
37
|
"forms",
|
|
38
38
|
"partitionsSummary",
|
|
39
|
-
"versionProperties"
|
|
39
|
+
"versionProperties",
|
|
40
|
+
"icebergCatalogInfo"
|
|
40
41
|
],
|
|
41
42
|
"entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
|
|
42
43
|
},
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "record",
|
|
3
|
+
"Aspect": {
|
|
4
|
+
"name": "icebergCatalogInfo"
|
|
5
|
+
},
|
|
6
|
+
"name": "IcebergCatalogInfo",
|
|
7
|
+
"namespace": "com.linkedin.pegasus2avro.dataset",
|
|
8
|
+
"fields": [
|
|
9
|
+
{
|
|
10
|
+
"type": [
|
|
11
|
+
"null",
|
|
12
|
+
"string"
|
|
13
|
+
],
|
|
14
|
+
"name": "metadataPointer",
|
|
15
|
+
"default": null,
|
|
16
|
+
"doc": "When Datahub is the REST Catalog for an Iceberg Table, stores the current metadata pointer.\nIf the Iceberg table is managed by an external catalog, the metadata pointer is not set."
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"type": [
|
|
20
|
+
"null",
|
|
21
|
+
"boolean"
|
|
22
|
+
],
|
|
23
|
+
"name": "view",
|
|
24
|
+
"default": null
|
|
25
|
+
}
|
|
26
|
+
],
|
|
27
|
+
"doc": "Iceberg Catalog metadata associated with an Iceberg table/view"
|
|
28
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "record",
|
|
3
|
+
"Aspect": {
|
|
4
|
+
"name": "icebergWarehouseInfo"
|
|
5
|
+
},
|
|
6
|
+
"name": "IcebergWarehouseInfo",
|
|
7
|
+
"namespace": "com.linkedin.pegasus2avro.dataplatforminstance",
|
|
8
|
+
"fields": [
|
|
9
|
+
{
|
|
10
|
+
"type": "string",
|
|
11
|
+
"name": "dataRoot",
|
|
12
|
+
"doc": "Path of the root for the backing store of the tables in the warehouse."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"java": {
|
|
16
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
17
|
+
},
|
|
18
|
+
"type": "string",
|
|
19
|
+
"name": "clientId",
|
|
20
|
+
"doc": "clientId to be used to authenticate with storage hosting this warehouse",
|
|
21
|
+
"Urn": "Urn"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"java": {
|
|
25
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
26
|
+
},
|
|
27
|
+
"type": "string",
|
|
28
|
+
"name": "clientSecret",
|
|
29
|
+
"doc": "client secret to authenticate with storage hosting this warehouse",
|
|
30
|
+
"Urn": "Urn"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"type": "string",
|
|
34
|
+
"name": "region",
|
|
35
|
+
"doc": "region where the warehouse is located."
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"type": [
|
|
39
|
+
"null",
|
|
40
|
+
"string"
|
|
41
|
+
],
|
|
42
|
+
"name": "role",
|
|
43
|
+
"default": null
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"type": [
|
|
47
|
+
"null",
|
|
48
|
+
"int"
|
|
49
|
+
],
|
|
50
|
+
"name": "tempCredentialExpirationSeconds",
|
|
51
|
+
"default": null
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"type": {
|
|
55
|
+
"type": "enum",
|
|
56
|
+
"symbolDocs": {
|
|
57
|
+
"CORP": "Designates corporation fabrics",
|
|
58
|
+
"DEV": "Designates development fabrics",
|
|
59
|
+
"EI": "Designates early-integration fabrics",
|
|
60
|
+
"NON_PROD": "Designates non-production fabrics",
|
|
61
|
+
"PRE": "Designates pre-production fabrics",
|
|
62
|
+
"PROD": "Designates production fabrics",
|
|
63
|
+
"QA": "Designates quality assurance fabrics",
|
|
64
|
+
"RVW": "Designates review fabrics",
|
|
65
|
+
"SANDBOX": "Designates sandbox fabrics",
|
|
66
|
+
"STG": "Designates staging fabrics",
|
|
67
|
+
"TEST": "Designates testing fabrics",
|
|
68
|
+
"UAT": "Designates user acceptance testing fabrics"
|
|
69
|
+
},
|
|
70
|
+
"name": "FabricType",
|
|
71
|
+
"namespace": "com.linkedin.pegasus2avro.common",
|
|
72
|
+
"symbols": [
|
|
73
|
+
"DEV",
|
|
74
|
+
"TEST",
|
|
75
|
+
"QA",
|
|
76
|
+
"UAT",
|
|
77
|
+
"EI",
|
|
78
|
+
"PRE",
|
|
79
|
+
"STG",
|
|
80
|
+
"NON_PROD",
|
|
81
|
+
"PROD",
|
|
82
|
+
"CORP",
|
|
83
|
+
"RVW",
|
|
84
|
+
"SANDBOX"
|
|
85
|
+
],
|
|
86
|
+
"doc": "Fabric group type"
|
|
87
|
+
},
|
|
88
|
+
"name": "env"
|
|
89
|
+
}
|
|
90
|
+
],
|
|
91
|
+
"doc": "An Iceberg warehouse location and credentails whose read/writes are governed by datahub catalog."
|
|
92
|
+
}
|
|
@@ -2049,16 +2049,16 @@
|
|
|
2049
2049
|
{
|
|
2050
2050
|
"Relationship": {
|
|
2051
2051
|
"/*/destinationUrn": {
|
|
2052
|
-
"createdActor": "
|
|
2053
|
-
"createdOn": "
|
|
2052
|
+
"createdActor": "dashboards/*/created/actor",
|
|
2053
|
+
"createdOn": "dashboards/*/created/time",
|
|
2054
2054
|
"entityTypes": [
|
|
2055
2055
|
"dashboard"
|
|
2056
2056
|
],
|
|
2057
2057
|
"isLineage": true,
|
|
2058
2058
|
"name": "DashboardContainsDashboard",
|
|
2059
|
-
"properties": "
|
|
2060
|
-
"updatedActor": "
|
|
2061
|
-
"updatedOn": "
|
|
2059
|
+
"properties": "dashboards/*/properties",
|
|
2060
|
+
"updatedActor": "dashboards/*/lastModified/actor",
|
|
2061
|
+
"updatedOn": "dashboards/*/lastModified/time"
|
|
2062
2062
|
}
|
|
2063
2063
|
},
|
|
2064
2064
|
"type": {
|
datahub/specific/dashboard.py
CHANGED
|
@@ -161,7 +161,7 @@ class DashboardPatchBuilder(
|
|
|
161
161
|
lastModified=self._mint_auditstamp(),
|
|
162
162
|
)
|
|
163
163
|
|
|
164
|
-
self._ensure_urn_type("
|
|
164
|
+
self._ensure_urn_type("chart", [chart_edge], "add_chart_edge")
|
|
165
165
|
self._add_patch(
|
|
166
166
|
DashboardInfo.ASPECT_NAME,
|
|
167
167
|
"add",
|
|
@@ -271,6 +271,48 @@ class DashboardPatchBuilder(
|
|
|
271
271
|
|
|
272
272
|
return self
|
|
273
273
|
|
|
274
|
+
def add_dashboard(
|
|
275
|
+
self, dashboard: Union[Edge, Urn, str]
|
|
276
|
+
) -> "DashboardPatchBuilder":
|
|
277
|
+
"""
|
|
278
|
+
Adds an dashboard to the DashboardPatchBuilder.
|
|
279
|
+
|
|
280
|
+
Args:
|
|
281
|
+
dashboard: The dashboard, which can be an Edge object, Urn object, or a string.
|
|
282
|
+
|
|
283
|
+
Returns:
|
|
284
|
+
The DashboardPatchBuilder instance.
|
|
285
|
+
|
|
286
|
+
Raises:
|
|
287
|
+
ValueError: If the dashboard is not a Dashboard urn.
|
|
288
|
+
|
|
289
|
+
Notes:
|
|
290
|
+
If `dashboard` is an Edge object, it is used directly. If `dashboard` is a Urn object or string,
|
|
291
|
+
it is converted to an Edge object and added with default audit stamps.
|
|
292
|
+
"""
|
|
293
|
+
if isinstance(dashboard, Edge):
|
|
294
|
+
dashboard_urn: str = dashboard.destinationUrn
|
|
295
|
+
dashboard_edge: Edge = dashboard
|
|
296
|
+
elif isinstance(dashboard, (Urn, str)):
|
|
297
|
+
dashboard_urn = str(dashboard)
|
|
298
|
+
if not dashboard_urn.startswith("urn:li:dashboard:"):
|
|
299
|
+
raise ValueError(f"Input {dashboard} is not a Dashboard urn")
|
|
300
|
+
|
|
301
|
+
dashboard_edge = Edge(
|
|
302
|
+
destinationUrn=dashboard_urn,
|
|
303
|
+
created=self._mint_auditstamp(),
|
|
304
|
+
lastModified=self._mint_auditstamp(),
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
self._ensure_urn_type("dashboard", [dashboard_edge], "add_dashboard")
|
|
308
|
+
self._add_patch(
|
|
309
|
+
DashboardInfo.ASPECT_NAME,
|
|
310
|
+
"add",
|
|
311
|
+
path=("dashboards", dashboard_urn),
|
|
312
|
+
value=dashboard_edge,
|
|
313
|
+
)
|
|
314
|
+
return self
|
|
315
|
+
|
|
274
316
|
def set_dashboard_url(
|
|
275
317
|
self, dashboard_url: Optional[str]
|
|
276
318
|
) -> "DashboardPatchBuilder":
|