acryl-datahub 0.15.0.5rc7__py3-none-any.whl → 0.15.0.5rc9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (38) hide show
  1. {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/METADATA +2493 -2463
  2. {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/RECORD +38 -35
  3. datahub/_version.py +1 -1
  4. datahub/cli/iceberg_cli.py +707 -0
  5. datahub/entrypoints.py +21 -0
  6. datahub/ingestion/api/incremental_lineage_helper.py +4 -0
  7. datahub/ingestion/glossary/classification_mixin.py +6 -0
  8. datahub/ingestion/glossary/classifier.py +3 -2
  9. datahub/ingestion/source/aws/glue.py +3 -2
  10. datahub/ingestion/source/identity/azure_ad.py +6 -14
  11. datahub/ingestion/source/mode.py +2 -4
  12. datahub/ingestion/source/snowflake/snowflake_config.py +13 -0
  13. datahub/ingestion/source/snowflake/snowflake_query.py +11 -0
  14. datahub/ingestion/source/snowflake/snowflake_report.py +1 -0
  15. datahub/ingestion/source/snowflake/snowflake_schema.py +17 -0
  16. datahub/ingestion/source/snowflake/snowflake_schema_gen.py +18 -36
  17. datahub/ingestion/source/snowflake/snowflake_tag.py +57 -3
  18. datahub/ingestion/source/snowflake/snowflake_v2.py +1 -0
  19. datahub/ingestion/source/sql/mssql/job_models.py +37 -8
  20. datahub/ingestion/source/sql/mssql/source.py +17 -0
  21. datahub/ingestion/source/tableau/tableau.py +14 -12
  22. datahub/ingestion/source/tableau/tableau_common.py +1 -1
  23. datahub/metadata/_schema_classes.py +160 -2
  24. datahub/metadata/com/linkedin/pegasus2avro/dataplatforminstance/__init__.py +2 -0
  25. datahub/metadata/com/linkedin/pegasus2avro/dataset/__init__.py +2 -0
  26. datahub/metadata/schema.avsc +96 -7
  27. datahub/metadata/schemas/DashboardInfo.avsc +5 -5
  28. datahub/metadata/schemas/DataPlatformInstanceKey.avsc +2 -1
  29. datahub/metadata/schemas/DatasetKey.avsc +2 -1
  30. datahub/metadata/schemas/IcebergCatalogInfo.avsc +28 -0
  31. datahub/metadata/schemas/IcebergWarehouseInfo.avsc +92 -0
  32. datahub/metadata/schemas/MetadataChangeEvent.avsc +5 -5
  33. datahub/specific/dashboard.py +43 -1
  34. datahub/upgrade/upgrade.py +13 -5
  35. {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/LICENSE +0 -0
  36. {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/WHEEL +0 -0
  37. {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/entry_points.txt +0 -0
  38. {acryl_datahub-0.15.0.5rc7.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/top_level.txt +0 -0
@@ -9149,6 +9149,113 @@ class DataPlatformInstancePropertiesClass(_Aspect):
9149
9149
  self._inner_dict['description'] = value
9150
9150
 
9151
9151
 
9152
+ class IcebergWarehouseInfoClass(_Aspect):
9153
+ """An Iceberg warehouse location and credentails whose read/writes are governed by datahub catalog."""
9154
+
9155
+
9156
+ ASPECT_NAME = 'icebergWarehouseInfo'
9157
+ ASPECT_INFO = {}
9158
+ RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.dataplatforminstance.IcebergWarehouseInfo")
9159
+
9160
+ def __init__(self,
9161
+ dataRoot: str,
9162
+ clientId: str,
9163
+ clientSecret: str,
9164
+ region: str,
9165
+ env: Union[str, "FabricTypeClass"],
9166
+ role: Union[None, str]=None,
9167
+ tempCredentialExpirationSeconds: Union[None, int]=None,
9168
+ ):
9169
+ super().__init__()
9170
+
9171
+ self.dataRoot = dataRoot
9172
+ self.clientId = clientId
9173
+ self.clientSecret = clientSecret
9174
+ self.region = region
9175
+ self.role = role
9176
+ self.tempCredentialExpirationSeconds = tempCredentialExpirationSeconds
9177
+ self.env = env
9178
+
9179
+ def _restore_defaults(self) -> None:
9180
+ self.dataRoot = str()
9181
+ self.clientId = str()
9182
+ self.clientSecret = str()
9183
+ self.region = str()
9184
+ self.role = self.RECORD_SCHEMA.fields_dict["role"].default
9185
+ self.tempCredentialExpirationSeconds = self.RECORD_SCHEMA.fields_dict["tempCredentialExpirationSeconds"].default
9186
+ self.env = FabricTypeClass.DEV
9187
+
9188
+
9189
+ @property
9190
+ def dataRoot(self) -> str:
9191
+ """Path of the root for the backing store of the tables in the warehouse."""
9192
+ return self._inner_dict.get('dataRoot') # type: ignore
9193
+
9194
+ @dataRoot.setter
9195
+ def dataRoot(self, value: str) -> None:
9196
+ self._inner_dict['dataRoot'] = value
9197
+
9198
+
9199
+ @property
9200
+ def clientId(self) -> str:
9201
+ """clientId to be used to authenticate with storage hosting this warehouse"""
9202
+ return self._inner_dict.get('clientId') # type: ignore
9203
+
9204
+ @clientId.setter
9205
+ def clientId(self, value: str) -> None:
9206
+ self._inner_dict['clientId'] = value
9207
+
9208
+
9209
+ @property
9210
+ def clientSecret(self) -> str:
9211
+ """client secret to authenticate with storage hosting this warehouse"""
9212
+ return self._inner_dict.get('clientSecret') # type: ignore
9213
+
9214
+ @clientSecret.setter
9215
+ def clientSecret(self, value: str) -> None:
9216
+ self._inner_dict['clientSecret'] = value
9217
+
9218
+
9219
+ @property
9220
+ def region(self) -> str:
9221
+ """region where the warehouse is located."""
9222
+ return self._inner_dict.get('region') # type: ignore
9223
+
9224
+ @region.setter
9225
+ def region(self, value: str) -> None:
9226
+ self._inner_dict['region'] = value
9227
+
9228
+
9229
+ @property
9230
+ def role(self) -> Union[None, str]:
9231
+ # No docs available.
9232
+ return self._inner_dict.get('role') # type: ignore
9233
+
9234
+ @role.setter
9235
+ def role(self, value: Union[None, str]) -> None:
9236
+ self._inner_dict['role'] = value
9237
+
9238
+
9239
+ @property
9240
+ def tempCredentialExpirationSeconds(self) -> Union[None, int]:
9241
+ # No docs available.
9242
+ return self._inner_dict.get('tempCredentialExpirationSeconds') # type: ignore
9243
+
9244
+ @tempCredentialExpirationSeconds.setter
9245
+ def tempCredentialExpirationSeconds(self, value: Union[None, int]) -> None:
9246
+ self._inner_dict['tempCredentialExpirationSeconds'] = value
9247
+
9248
+
9249
+ @property
9250
+ def env(self) -> Union[str, "FabricTypeClass"]:
9251
+ # No docs available.
9252
+ return self._inner_dict.get('env') # type: ignore
9253
+
9254
+ @env.setter
9255
+ def env(self, value: Union[str, "FabricTypeClass"]) -> None:
9256
+ self._inner_dict['env'] = value
9257
+
9258
+
9152
9259
  class DataProcessInfoClass(_Aspect):
9153
9260
  """The inputs and outputs of this data process"""
9154
9261
 
@@ -10992,6 +11099,49 @@ class HistogramClass(DictWrapper):
10992
11099
  self._inner_dict['heights'] = value
10993
11100
 
10994
11101
 
11102
+ class IcebergCatalogInfoClass(_Aspect):
11103
+ """Iceberg Catalog metadata associated with an Iceberg table/view"""
11104
+
11105
+
11106
+ ASPECT_NAME = 'icebergCatalogInfo'
11107
+ ASPECT_INFO = {}
11108
+ RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.dataset.IcebergCatalogInfo")
11109
+
11110
+ def __init__(self,
11111
+ metadataPointer: Union[None, str]=None,
11112
+ view: Union[None, bool]=None,
11113
+ ):
11114
+ super().__init__()
11115
+
11116
+ self.metadataPointer = metadataPointer
11117
+ self.view = view
11118
+
11119
+ def _restore_defaults(self) -> None:
11120
+ self.metadataPointer = self.RECORD_SCHEMA.fields_dict["metadataPointer"].default
11121
+ self.view = self.RECORD_SCHEMA.fields_dict["view"].default
11122
+
11123
+
11124
+ @property
11125
+ def metadataPointer(self) -> Union[None, str]:
11126
+ """When Datahub is the REST Catalog for an Iceberg Table, stores the current metadata pointer.
11127
+ If the Iceberg table is managed by an external catalog, the metadata pointer is not set."""
11128
+ return self._inner_dict.get('metadataPointer') # type: ignore
11129
+
11130
+ @metadataPointer.setter
11131
+ def metadataPointer(self, value: Union[None, str]) -> None:
11132
+ self._inner_dict['metadataPointer'] = value
11133
+
11134
+
11135
+ @property
11136
+ def view(self) -> Union[None, bool]:
11137
+ # No docs available.
11138
+ return self._inner_dict.get('view') # type: ignore
11139
+
11140
+ @view.setter
11141
+ def view(self, value: Union[None, bool]) -> None:
11142
+ self._inner_dict['view'] = value
11143
+
11144
+
10995
11145
  class PartitionSummaryClass(DictWrapper):
10996
11146
  """Defines how the data is partitioned"""
10997
11147
 
@@ -14987,7 +15137,7 @@ class DataPlatformInstanceKeyClass(_Aspect):
14987
15137
 
14988
15138
 
14989
15139
  ASPECT_NAME = 'dataPlatformInstanceKey'
14990
- ASPECT_INFO = {'keyForEntity': 'dataPlatformInstance', 'entityCategory': 'internal', 'entityAspects': ['dataPlatformInstanceProperties', 'ownership', 'globalTags', 'institutionalMemory', 'deprecation', 'status']}
15140
+ ASPECT_INFO = {'keyForEntity': 'dataPlatformInstance', 'entityCategory': 'internal', 'entityAspects': ['dataPlatformInstanceProperties', 'ownership', 'globalTags', 'institutionalMemory', 'deprecation', 'status', 'icebergWarehouseInfo']}
14991
15141
  RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DataPlatformInstanceKey")
14992
15142
 
14993
15143
  def __init__(self,
@@ -15143,7 +15293,7 @@ class DatasetKeyClass(_Aspect):
15143
15293
 
15144
15294
 
15145
15295
  ASPECT_NAME = 'datasetKey'
15146
- ASPECT_INFO = {'keyForEntity': 'dataset', 'entityCategory': 'core', 'entityAspects': ['viewProperties', 'subTypes', 'datasetProfile', 'datasetUsageStatistics', 'operation', 'domains', 'schemaMetadata', 'status', 'container', 'deprecation', 'testResults', 'siblings', 'embed', 'incidentsSummary', 'datasetProperties', 'editableDatasetProperties', 'datasetDeprecation', 'datasetUpstreamLineage', 'upstreamLineage', 'institutionalMemory', 'ownership', 'editableSchemaMetadata', 'globalTags', 'glossaryTerms', 'browsePaths', 'dataPlatformInstance', 'browsePathsV2', 'access', 'structuredProperties', 'forms', 'partitionsSummary', 'versionProperties'], 'entityDoc': 'Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets.'}
15296
+ ASPECT_INFO = {'keyForEntity': 'dataset', 'entityCategory': 'core', 'entityAspects': ['viewProperties', 'subTypes', 'datasetProfile', 'datasetUsageStatistics', 'operation', 'domains', 'schemaMetadata', 'status', 'container', 'deprecation', 'testResults', 'siblings', 'embed', 'incidentsSummary', 'datasetProperties', 'editableDatasetProperties', 'datasetDeprecation', 'datasetUpstreamLineage', 'upstreamLineage', 'institutionalMemory', 'ownership', 'editableSchemaMetadata', 'globalTags', 'glossaryTerms', 'browsePaths', 'dataPlatformInstance', 'browsePathsV2', 'access', 'structuredProperties', 'forms', 'partitionsSummary', 'versionProperties', 'icebergCatalogInfo'], 'entityDoc': 'Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets.'}
15147
15297
  RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DatasetKey")
15148
15298
 
15149
15299
  def __init__(self,
@@ -25245,6 +25395,7 @@ __SCHEMA_TYPES = {
25245
25395
  'com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo': DataPlatformInfoClass,
25246
25396
  'com.linkedin.pegasus2avro.dataplatform.PlatformType': PlatformTypeClass,
25247
25397
  'com.linkedin.pegasus2avro.dataplatforminstance.DataPlatformInstanceProperties': DataPlatformInstancePropertiesClass,
25398
+ 'com.linkedin.pegasus2avro.dataplatforminstance.IcebergWarehouseInfo': IcebergWarehouseInfoClass,
25248
25399
  'com.linkedin.pegasus2avro.dataprocess.DataProcessInfo': DataProcessInfoClass,
25249
25400
  'com.linkedin.pegasus2avro.dataprocess.DataProcessInstanceInput': DataProcessInstanceInputClass,
25250
25401
  'com.linkedin.pegasus2avro.dataprocess.DataProcessInstanceOutput': DataProcessInstanceOutputClass,
@@ -25275,6 +25426,7 @@ __SCHEMA_TYPES = {
25275
25426
  'com.linkedin.pegasus2avro.dataset.FineGrainedLineageDownstreamType': FineGrainedLineageDownstreamTypeClass,
25276
25427
  'com.linkedin.pegasus2avro.dataset.FineGrainedLineageUpstreamType': FineGrainedLineageUpstreamTypeClass,
25277
25428
  'com.linkedin.pegasus2avro.dataset.Histogram': HistogramClass,
25429
+ 'com.linkedin.pegasus2avro.dataset.IcebergCatalogInfo': IcebergCatalogInfoClass,
25278
25430
  'com.linkedin.pegasus2avro.dataset.PartitionSummary': PartitionSummaryClass,
25279
25431
  'com.linkedin.pegasus2avro.dataset.PartitionsSummary': PartitionsSummaryClass,
25280
25432
  'com.linkedin.pegasus2avro.dataset.Quantile': QuantileClass,
@@ -25721,6 +25873,7 @@ __SCHEMA_TYPES = {
25721
25873
  'DataPlatformInfo': DataPlatformInfoClass,
25722
25874
  'PlatformType': PlatformTypeClass,
25723
25875
  'DataPlatformInstanceProperties': DataPlatformInstancePropertiesClass,
25876
+ 'IcebergWarehouseInfo': IcebergWarehouseInfoClass,
25724
25877
  'DataProcessInfo': DataProcessInfoClass,
25725
25878
  'DataProcessInstanceInput': DataProcessInstanceInputClass,
25726
25879
  'DataProcessInstanceOutput': DataProcessInstanceOutputClass,
@@ -25751,6 +25904,7 @@ __SCHEMA_TYPES = {
25751
25904
  'FineGrainedLineageDownstreamType': FineGrainedLineageDownstreamTypeClass,
25752
25905
  'FineGrainedLineageUpstreamType': FineGrainedLineageUpstreamTypeClass,
25753
25906
  'Histogram': HistogramClass,
25907
+ 'IcebergCatalogInfo': IcebergCatalogInfoClass,
25754
25908
  'PartitionSummary': PartitionSummaryClass,
25755
25909
  'PartitionsSummary': PartitionsSummaryClass,
25756
25910
  'Quantile': QuantileClass,
@@ -26164,6 +26318,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
26164
26318
  QuerySubjectsClass,
26165
26319
  QueryUsageStatisticsClass,
26166
26320
  DataHubSecretValueClass,
26321
+ IcebergCatalogInfoClass,
26167
26322
  DatasetUpstreamLineageClass,
26168
26323
  UpstreamLineageClass,
26169
26324
  DatasetProfileClass,
@@ -26252,6 +26407,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
26252
26407
  EntityTypeKeyClass,
26253
26408
  OwnershipTypeInfoClass,
26254
26409
  DataPlatformInstancePropertiesClass,
26410
+ IcebergWarehouseInfoClass,
26255
26411
  DataHubAccessTokenInfoClass
26256
26412
  ]
26257
26413
 
@@ -26383,6 +26539,7 @@ class AspectBag(TypedDict, total=False):
26383
26539
  querySubjects: QuerySubjectsClass
26384
26540
  queryUsageStatistics: QueryUsageStatisticsClass
26385
26541
  dataHubSecretValue: DataHubSecretValueClass
26542
+ icebergCatalogInfo: IcebergCatalogInfoClass
26386
26543
  datasetUpstreamLineage: DatasetUpstreamLineageClass
26387
26544
  upstreamLineage: UpstreamLineageClass
26388
26545
  datasetProfile: DatasetProfileClass
@@ -26471,6 +26628,7 @@ class AspectBag(TypedDict, total=False):
26471
26628
  entityTypeKey: EntityTypeKeyClass
26472
26629
  ownershipTypeInfo: OwnershipTypeInfoClass
26473
26630
  dataPlatformInstanceProperties: DataPlatformInstancePropertiesClass
26631
+ icebergWarehouseInfo: IcebergWarehouseInfoClass
26474
26632
  dataHubAccessTokenInfo: DataHubAccessTokenInfoClass
26475
26633
 
26476
26634
 
@@ -8,8 +8,10 @@
8
8
  # fmt: off
9
9
  # isort: skip_file
10
10
  from .....schema_classes import DataPlatformInstancePropertiesClass
11
+ from .....schema_classes import IcebergWarehouseInfoClass
11
12
 
12
13
 
13
14
  DataPlatformInstanceProperties = DataPlatformInstancePropertiesClass
15
+ IcebergWarehouseInfo = IcebergWarehouseInfoClass
14
16
 
15
17
  # fmt: on
@@ -24,6 +24,7 @@ from .....schema_classes import FineGrainedLineageClass
24
24
  from .....schema_classes import FineGrainedLineageDownstreamTypeClass
25
25
  from .....schema_classes import FineGrainedLineageUpstreamTypeClass
26
26
  from .....schema_classes import HistogramClass
27
+ from .....schema_classes import IcebergCatalogInfoClass
27
28
  from .....schema_classes import PartitionSummaryClass
28
29
  from .....schema_classes import PartitionsSummaryClass
29
30
  from .....schema_classes import QuantileClass
@@ -50,6 +51,7 @@ FineGrainedLineage = FineGrainedLineageClass
50
51
  FineGrainedLineageDownstreamType = FineGrainedLineageDownstreamTypeClass
51
52
  FineGrainedLineageUpstreamType = FineGrainedLineageUpstreamTypeClass
52
53
  Histogram = HistogramClass
54
+ IcebergCatalogInfo = IcebergCatalogInfoClass
53
55
  PartitionSummary = PartitionSummaryClass
54
56
  PartitionsSummary = PartitionsSummaryClass
55
57
  Quantile = QuantileClass
@@ -4730,16 +4730,16 @@
4730
4730
  {
4731
4731
  "Relationship": {
4732
4732
  "/*/destinationUrn": {
4733
- "createdActor": "datasetEdges/*/created/actor",
4734
- "createdOn": "datasetEdges/*/created/time",
4733
+ "createdActor": "dashboards/*/created/actor",
4734
+ "createdOn": "dashboards/*/created/time",
4735
4735
  "entityTypes": [
4736
4736
  "dashboard"
4737
4737
  ],
4738
4738
  "isLineage": true,
4739
4739
  "name": "DashboardContainsDashboard",
4740
- "properties": "datasetEdges/*/properties",
4741
- "updatedActor": "datasetEdges/*/lastModified/actor",
4742
- "updatedOn": "datasetEdges/*/lastModified/time"
4740
+ "properties": "dashboards/*/properties",
4741
+ "updatedActor": "dashboards/*/lastModified/actor",
4742
+ "updatedOn": "dashboards/*/lastModified/time"
4743
4743
  }
4744
4744
  },
4745
4745
  "type": {
@@ -13109,6 +13109,34 @@
13109
13109
  ],
13110
13110
  "doc": "The value of a DataHub Secret"
13111
13111
  },
13112
+ {
13113
+ "type": "record",
13114
+ "Aspect": {
13115
+ "name": "icebergCatalogInfo"
13116
+ },
13117
+ "name": "IcebergCatalogInfo",
13118
+ "namespace": "com.linkedin.pegasus2avro.dataset",
13119
+ "fields": [
13120
+ {
13121
+ "type": [
13122
+ "null",
13123
+ "string"
13124
+ ],
13125
+ "name": "metadataPointer",
13126
+ "default": null,
13127
+ "doc": "When Datahub is the REST Catalog for an Iceberg Table, stores the current metadata pointer.\nIf the Iceberg table is managed by an external catalog, the metadata pointer is not set."
13128
+ },
13129
+ {
13130
+ "type": [
13131
+ "null",
13132
+ "boolean"
13133
+ ],
13134
+ "name": "view",
13135
+ "default": null
13136
+ }
13137
+ ],
13138
+ "doc": "Iceberg Catalog metadata associated with an Iceberg table/view"
13139
+ },
13112
13140
  {
13113
13141
  "type": "record",
13114
13142
  "Aspect": {
@@ -16692,7 +16720,8 @@
16692
16720
  "structuredProperties",
16693
16721
  "forms",
16694
16722
  "partitionsSummary",
16695
- "versionProperties"
16723
+ "versionProperties",
16724
+ "icebergCatalogInfo"
16696
16725
  ],
16697
16726
  "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
16698
16727
  },
@@ -17393,7 +17422,8 @@
17393
17422
  "globalTags",
17394
17423
  "institutionalMemory",
17395
17424
  "deprecation",
17396
- "status"
17425
+ "status",
17426
+ "icebergWarehouseInfo"
17397
17427
  ]
17398
17428
  },
17399
17429
  "name": "DataPlatformInstanceKey",
@@ -18144,6 +18174,65 @@
18144
18174
  ],
18145
18175
  "doc": "Properties associated with a Data Platform Instance"
18146
18176
  },
18177
+ {
18178
+ "type": "record",
18179
+ "Aspect": {
18180
+ "name": "icebergWarehouseInfo"
18181
+ },
18182
+ "name": "IcebergWarehouseInfo",
18183
+ "namespace": "com.linkedin.pegasus2avro.dataplatforminstance",
18184
+ "fields": [
18185
+ {
18186
+ "type": "string",
18187
+ "name": "dataRoot",
18188
+ "doc": "Path of the root for the backing store of the tables in the warehouse."
18189
+ },
18190
+ {
18191
+ "java": {
18192
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
18193
+ },
18194
+ "Urn": "Urn",
18195
+ "type": "string",
18196
+ "name": "clientId",
18197
+ "doc": "clientId to be used to authenticate with storage hosting this warehouse"
18198
+ },
18199
+ {
18200
+ "java": {
18201
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
18202
+ },
18203
+ "Urn": "Urn",
18204
+ "type": "string",
18205
+ "name": "clientSecret",
18206
+ "doc": "client secret to authenticate with storage hosting this warehouse"
18207
+ },
18208
+ {
18209
+ "type": "string",
18210
+ "name": "region",
18211
+ "doc": "region where the warehouse is located."
18212
+ },
18213
+ {
18214
+ "type": [
18215
+ "null",
18216
+ "string"
18217
+ ],
18218
+ "name": "role",
18219
+ "default": null
18220
+ },
18221
+ {
18222
+ "type": [
18223
+ "null",
18224
+ "int"
18225
+ ],
18226
+ "name": "tempCredentialExpirationSeconds",
18227
+ "default": null
18228
+ },
18229
+ {
18230
+ "type": "com.linkedin.pegasus2avro.common.FabricType",
18231
+ "name": "env"
18232
+ }
18233
+ ],
18234
+ "doc": "An Iceberg warehouse location and credentails whose read/writes are governed by datahub catalog."
18235
+ },
18147
18236
  {
18148
18237
  "type": "record",
18149
18238
  "deprecated": "Use DatasetUsageStatistics, or other UsageStatistics records, instead",
@@ -258,16 +258,16 @@
258
258
  {
259
259
  "Relationship": {
260
260
  "/*/destinationUrn": {
261
- "createdActor": "datasetEdges/*/created/actor",
262
- "createdOn": "datasetEdges/*/created/time",
261
+ "createdActor": "dashboards/*/created/actor",
262
+ "createdOn": "dashboards/*/created/time",
263
263
  "entityTypes": [
264
264
  "dashboard"
265
265
  ],
266
266
  "isLineage": true,
267
267
  "name": "DashboardContainsDashboard",
268
- "properties": "datasetEdges/*/properties",
269
- "updatedActor": "datasetEdges/*/lastModified/actor",
270
- "updatedOn": "datasetEdges/*/lastModified/time"
268
+ "properties": "dashboards/*/properties",
269
+ "updatedActor": "dashboards/*/lastModified/actor",
270
+ "updatedOn": "dashboards/*/lastModified/time"
271
271
  }
272
272
  },
273
273
  "type": {
@@ -10,7 +10,8 @@
10
10
  "globalTags",
11
11
  "institutionalMemory",
12
12
  "deprecation",
13
- "status"
13
+ "status",
14
+ "icebergWarehouseInfo"
14
15
  ]
15
16
  },
16
17
  "name": "DataPlatformInstanceKey",
@@ -36,7 +36,8 @@
36
36
  "structuredProperties",
37
37
  "forms",
38
38
  "partitionsSummary",
39
- "versionProperties"
39
+ "versionProperties",
40
+ "icebergCatalogInfo"
40
41
  ],
41
42
  "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
42
43
  },
@@ -0,0 +1,28 @@
1
+ {
2
+ "type": "record",
3
+ "Aspect": {
4
+ "name": "icebergCatalogInfo"
5
+ },
6
+ "name": "IcebergCatalogInfo",
7
+ "namespace": "com.linkedin.pegasus2avro.dataset",
8
+ "fields": [
9
+ {
10
+ "type": [
11
+ "null",
12
+ "string"
13
+ ],
14
+ "name": "metadataPointer",
15
+ "default": null,
16
+ "doc": "When Datahub is the REST Catalog for an Iceberg Table, stores the current metadata pointer.\nIf the Iceberg table is managed by an external catalog, the metadata pointer is not set."
17
+ },
18
+ {
19
+ "type": [
20
+ "null",
21
+ "boolean"
22
+ ],
23
+ "name": "view",
24
+ "default": null
25
+ }
26
+ ],
27
+ "doc": "Iceberg Catalog metadata associated with an Iceberg table/view"
28
+ }
@@ -0,0 +1,92 @@
1
+ {
2
+ "type": "record",
3
+ "Aspect": {
4
+ "name": "icebergWarehouseInfo"
5
+ },
6
+ "name": "IcebergWarehouseInfo",
7
+ "namespace": "com.linkedin.pegasus2avro.dataplatforminstance",
8
+ "fields": [
9
+ {
10
+ "type": "string",
11
+ "name": "dataRoot",
12
+ "doc": "Path of the root for the backing store of the tables in the warehouse."
13
+ },
14
+ {
15
+ "java": {
16
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
17
+ },
18
+ "type": "string",
19
+ "name": "clientId",
20
+ "doc": "clientId to be used to authenticate with storage hosting this warehouse",
21
+ "Urn": "Urn"
22
+ },
23
+ {
24
+ "java": {
25
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
26
+ },
27
+ "type": "string",
28
+ "name": "clientSecret",
29
+ "doc": "client secret to authenticate with storage hosting this warehouse",
30
+ "Urn": "Urn"
31
+ },
32
+ {
33
+ "type": "string",
34
+ "name": "region",
35
+ "doc": "region where the warehouse is located."
36
+ },
37
+ {
38
+ "type": [
39
+ "null",
40
+ "string"
41
+ ],
42
+ "name": "role",
43
+ "default": null
44
+ },
45
+ {
46
+ "type": [
47
+ "null",
48
+ "int"
49
+ ],
50
+ "name": "tempCredentialExpirationSeconds",
51
+ "default": null
52
+ },
53
+ {
54
+ "type": {
55
+ "type": "enum",
56
+ "symbolDocs": {
57
+ "CORP": "Designates corporation fabrics",
58
+ "DEV": "Designates development fabrics",
59
+ "EI": "Designates early-integration fabrics",
60
+ "NON_PROD": "Designates non-production fabrics",
61
+ "PRE": "Designates pre-production fabrics",
62
+ "PROD": "Designates production fabrics",
63
+ "QA": "Designates quality assurance fabrics",
64
+ "RVW": "Designates review fabrics",
65
+ "SANDBOX": "Designates sandbox fabrics",
66
+ "STG": "Designates staging fabrics",
67
+ "TEST": "Designates testing fabrics",
68
+ "UAT": "Designates user acceptance testing fabrics"
69
+ },
70
+ "name": "FabricType",
71
+ "namespace": "com.linkedin.pegasus2avro.common",
72
+ "symbols": [
73
+ "DEV",
74
+ "TEST",
75
+ "QA",
76
+ "UAT",
77
+ "EI",
78
+ "PRE",
79
+ "STG",
80
+ "NON_PROD",
81
+ "PROD",
82
+ "CORP",
83
+ "RVW",
84
+ "SANDBOX"
85
+ ],
86
+ "doc": "Fabric group type"
87
+ },
88
+ "name": "env"
89
+ }
90
+ ],
91
+ "doc": "An Iceberg warehouse location and credentails whose read/writes are governed by datahub catalog."
92
+ }
@@ -2049,16 +2049,16 @@
2049
2049
  {
2050
2050
  "Relationship": {
2051
2051
  "/*/destinationUrn": {
2052
- "createdActor": "datasetEdges/*/created/actor",
2053
- "createdOn": "datasetEdges/*/created/time",
2052
+ "createdActor": "dashboards/*/created/actor",
2053
+ "createdOn": "dashboards/*/created/time",
2054
2054
  "entityTypes": [
2055
2055
  "dashboard"
2056
2056
  ],
2057
2057
  "isLineage": true,
2058
2058
  "name": "DashboardContainsDashboard",
2059
- "properties": "datasetEdges/*/properties",
2060
- "updatedActor": "datasetEdges/*/lastModified/actor",
2061
- "updatedOn": "datasetEdges/*/lastModified/time"
2059
+ "properties": "dashboards/*/properties",
2060
+ "updatedActor": "dashboards/*/lastModified/actor",
2061
+ "updatedOn": "dashboards/*/lastModified/time"
2062
2062
  }
2063
2063
  },
2064
2064
  "type": {
@@ -161,7 +161,7 @@ class DashboardPatchBuilder(
161
161
  lastModified=self._mint_auditstamp(),
162
162
  )
163
163
 
164
- self._ensure_urn_type("dataset", [chart_edge], "add_chart_edge")
164
+ self._ensure_urn_type("chart", [chart_edge], "add_chart_edge")
165
165
  self._add_patch(
166
166
  DashboardInfo.ASPECT_NAME,
167
167
  "add",
@@ -271,6 +271,48 @@ class DashboardPatchBuilder(
271
271
 
272
272
  return self
273
273
 
274
+ def add_dashboard(
275
+ self, dashboard: Union[Edge, Urn, str]
276
+ ) -> "DashboardPatchBuilder":
277
+ """
278
+ Adds an dashboard to the DashboardPatchBuilder.
279
+
280
+ Args:
281
+ dashboard: The dashboard, which can be an Edge object, Urn object, or a string.
282
+
283
+ Returns:
284
+ The DashboardPatchBuilder instance.
285
+
286
+ Raises:
287
+ ValueError: If the dashboard is not a Dashboard urn.
288
+
289
+ Notes:
290
+ If `dashboard` is an Edge object, it is used directly. If `dashboard` is a Urn object or string,
291
+ it is converted to an Edge object and added with default audit stamps.
292
+ """
293
+ if isinstance(dashboard, Edge):
294
+ dashboard_urn: str = dashboard.destinationUrn
295
+ dashboard_edge: Edge = dashboard
296
+ elif isinstance(dashboard, (Urn, str)):
297
+ dashboard_urn = str(dashboard)
298
+ if not dashboard_urn.startswith("urn:li:dashboard:"):
299
+ raise ValueError(f"Input {dashboard} is not a Dashboard urn")
300
+
301
+ dashboard_edge = Edge(
302
+ destinationUrn=dashboard_urn,
303
+ created=self._mint_auditstamp(),
304
+ lastModified=self._mint_auditstamp(),
305
+ )
306
+
307
+ self._ensure_urn_type("dashboard", [dashboard_edge], "add_dashboard")
308
+ self._add_patch(
309
+ DashboardInfo.ASPECT_NAME,
310
+ "add",
311
+ path=("dashboards", dashboard_urn),
312
+ value=dashboard_edge,
313
+ )
314
+ return self
315
+
274
316
  def set_dashboard_url(
275
317
  self, dashboard_url: Optional[str]
276
318
  ) -> "DashboardPatchBuilder":