acryl-datahub 1.2.0.7rc4__py3-none-any.whl → 1.2.0.8rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.2.0.7rc4.dist-info → acryl_datahub-1.2.0.8rc1.dist-info}/METADATA +2674 -2674
- {acryl_datahub-1.2.0.7rc4.dist-info → acryl_datahub-1.2.0.8rc1.dist-info}/RECORD +34 -32
- datahub/_version.py +1 -1
- datahub/cli/delete_cli.py +1 -0
- datahub/ingestion/api/report.py +4 -0
- datahub/ingestion/autogenerated/capability_summary.json +1 -1
- datahub/ingestion/graph/client.py +8 -1
- datahub/ingestion/source/datahub/config.py +4 -0
- datahub/ingestion/source/datahub/datahub_database_reader.py +6 -1
- datahub/ingestion/source/metadata/lineage.py +8 -8
- datahub/ingestion/source/redshift/redshift.py +1 -1
- datahub/ingestion/source/sql/athena.py +95 -18
- datahub/ingestion/source/sql/athena_properties_extractor.py +43 -25
- datahub/ingestion/source/superset.py +3 -2
- datahub/ingestion/source/tableau/tableau.py +8 -5
- datahub/metadata/_internal_schema_classes.py +207 -12
- datahub/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/template/__init__.py +6 -0
- datahub/metadata/schema.avsc +160 -12
- datahub/metadata/schemas/AssetSettings.avsc +63 -0
- datahub/metadata/schemas/DataHubPageModuleProperties.avsc +9 -1
- datahub/metadata/schemas/DataHubPageTemplateProperties.avsc +77 -1
- datahub/metadata/schemas/DataProductKey.avsc +2 -1
- datahub/metadata/schemas/DomainKey.avsc +2 -1
- datahub/metadata/schemas/GlossaryNodeKey.avsc +2 -1
- datahub/metadata/schemas/GlossaryTermKey.avsc +2 -1
- datahub/metadata/schemas/IncidentInfo.avsc +3 -3
- datahub/metadata/schemas/StructuredPropertyDefinition.avsc +0 -3
- datahub/sql_parsing/sqlglot_lineage.py +121 -28
- datahub/sql_parsing/sqlglot_utils.py +12 -1
- {acryl_datahub-1.2.0.7rc4.dist-info → acryl_datahub-1.2.0.8rc1.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.2.0.7rc4.dist-info → acryl_datahub-1.2.0.8rc1.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.2.0.7rc4.dist-info → acryl_datahub-1.2.0.8rc1.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.2.0.7rc4.dist-info → acryl_datahub-1.2.0.8rc1.dist-info}/top_level.txt +0 -0
|
@@ -174,20 +174,16 @@ class AthenaPropertiesExtractor:
|
|
|
174
174
|
def format_column_definition(line):
|
|
175
175
|
# Use regex to parse the line more accurately
|
|
176
176
|
# Pattern: column_name data_type [COMMENT comment_text] [,]
|
|
177
|
-
#
|
|
178
|
-
pattern = r"^\s*(
|
|
179
|
-
match = re.match(pattern, line, re.IGNORECASE)
|
|
177
|
+
# Improved pattern to better separate column name, data type, and comment
|
|
178
|
+
pattern = r"^\s*([`\w']+)\s+([\w<>\[\](),\s]+?)(\s+COMMENT\s+(.+?))?(,?)\s*$"
|
|
179
|
+
match = re.match(pattern, line.strip(), re.IGNORECASE)
|
|
180
180
|
|
|
181
181
|
if not match:
|
|
182
182
|
return line
|
|
183
|
-
column_name = match.group(1)
|
|
184
|
-
data_type = match.group(2)
|
|
185
|
-
comment_part = match.group(
|
|
186
|
-
|
|
187
|
-
if comment_part:
|
|
188
|
-
trailing_comma = match.group(6) if match.group(6) else ""
|
|
189
|
-
else:
|
|
190
|
-
trailing_comma = match.group(7) if match.group(7) else ""
|
|
183
|
+
column_name = match.group(1).strip()
|
|
184
|
+
data_type = match.group(2).strip()
|
|
185
|
+
comment_part = match.group(4) # COMMENT part
|
|
186
|
+
trailing_comma = match.group(5) if match.group(5) else ""
|
|
191
187
|
|
|
192
188
|
# Add backticks to column name if not already present
|
|
193
189
|
if not (column_name.startswith("`") and column_name.endswith("`")):
|
|
@@ -201,17 +197,19 @@ class AthenaPropertiesExtractor:
|
|
|
201
197
|
|
|
202
198
|
# Handle comment quoting and escaping
|
|
203
199
|
if comment_part.startswith("'") and comment_part.endswith("'"):
|
|
204
|
-
# Already
|
|
205
|
-
|
|
200
|
+
# Already single quoted - but check for proper escaping
|
|
201
|
+
inner_content = comment_part[1:-1]
|
|
202
|
+
# Re-escape any single quotes that aren't properly escaped
|
|
203
|
+
escaped_content = inner_content.replace("'", "''")
|
|
204
|
+
formatted_comment = f"'{escaped_content}'"
|
|
206
205
|
elif comment_part.startswith('"') and comment_part.endswith('"'):
|
|
207
206
|
# Double quoted - convert to single quotes and escape internal single quotes
|
|
208
207
|
inner_content = comment_part[1:-1]
|
|
209
208
|
escaped_content = inner_content.replace("'", "''")
|
|
210
209
|
formatted_comment = f"'{escaped_content}'"
|
|
211
210
|
else:
|
|
212
|
-
# Not quoted -
|
|
213
|
-
|
|
214
|
-
formatted_comment = f"'{escaped_content}'"
|
|
211
|
+
# Not quoted - use double quotes to avoid escaping issues with single quotes
|
|
212
|
+
formatted_comment = f'"{comment_part}"'
|
|
215
213
|
|
|
216
214
|
result_parts.extend(["COMMENT", formatted_comment])
|
|
217
215
|
|
|
@@ -240,19 +238,39 @@ class AthenaPropertiesExtractor:
|
|
|
240
238
|
formatted_lines.append(line)
|
|
241
239
|
continue
|
|
242
240
|
|
|
243
|
-
#
|
|
244
|
-
if in_column_definition and "
|
|
245
|
-
in_column_definition = False
|
|
241
|
+
# Skip processing PARTITIONED BY clauses as column definitions
|
|
242
|
+
if in_column_definition and "PARTITIONED BY" in line.upper():
|
|
246
243
|
formatted_lines.append(line)
|
|
247
244
|
continue
|
|
248
245
|
|
|
249
|
-
# Process
|
|
246
|
+
# Process column definitions first, then check for exit condition
|
|
250
247
|
if in_column_definition and stripped_line:
|
|
251
|
-
#
|
|
252
|
-
|
|
253
|
-
line
|
|
254
|
-
|
|
255
|
-
|
|
248
|
+
# Check if this line contains a column definition (before the closing paren)
|
|
249
|
+
if ")" in line:
|
|
250
|
+
# Split the line at the closing parenthesis
|
|
251
|
+
paren_index = line.find(")")
|
|
252
|
+
column_part = line[:paren_index].strip()
|
|
253
|
+
closing_part = line[paren_index:]
|
|
254
|
+
|
|
255
|
+
if column_part:
|
|
256
|
+
# Format the column part
|
|
257
|
+
formatted_column = (
|
|
258
|
+
AthenaPropertiesExtractor.format_column_definition(
|
|
259
|
+
column_part
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
# Reconstruct the line
|
|
263
|
+
formatted_line = formatted_column.rstrip() + closing_part
|
|
264
|
+
formatted_lines.append(formatted_line)
|
|
265
|
+
else:
|
|
266
|
+
formatted_lines.append(line)
|
|
267
|
+
in_column_definition = False
|
|
268
|
+
else:
|
|
269
|
+
# Regular column definition line
|
|
270
|
+
formatted_line = AthenaPropertiesExtractor.format_column_definition(
|
|
271
|
+
line
|
|
272
|
+
)
|
|
273
|
+
formatted_lines.append(formatted_line)
|
|
256
274
|
else:
|
|
257
275
|
# For all other lines, keep as-is
|
|
258
276
|
formatted_lines.append(line)
|
|
@@ -154,6 +154,7 @@ class SupersetDataset(BaseModel):
|
|
|
154
154
|
table_name: str
|
|
155
155
|
changed_on_utc: Optional[str] = None
|
|
156
156
|
explore_url: Optional[str] = ""
|
|
157
|
+
description: Optional[str] = ""
|
|
157
158
|
|
|
158
159
|
@property
|
|
159
160
|
def modified_dt(self) -> Optional[datetime]:
|
|
@@ -1062,7 +1063,7 @@ class SupersetSource(StatefulIngestionSourceBase):
|
|
|
1062
1063
|
fieldPath=col.get("column_name", ""),
|
|
1063
1064
|
type=SchemaFieldDataType(data_type),
|
|
1064
1065
|
nativeDataType="",
|
|
1065
|
-
description=col.get("column_name", ""),
|
|
1066
|
+
description=col.get("description") or col.get("column_name", ""),
|
|
1066
1067
|
nullable=True,
|
|
1067
1068
|
)
|
|
1068
1069
|
schema_fields.append(field)
|
|
@@ -1283,7 +1284,7 @@ class SupersetSource(StatefulIngestionSourceBase):
|
|
|
1283
1284
|
|
|
1284
1285
|
dataset_info = DatasetPropertiesClass(
|
|
1285
1286
|
name=dataset.table_name,
|
|
1286
|
-
description="",
|
|
1287
|
+
description=dataset.description or "",
|
|
1287
1288
|
externalUrl=dataset_url,
|
|
1288
1289
|
lastModified=TimeStamp(time=modified_ts),
|
|
1289
1290
|
)
|
|
@@ -1561,12 +1561,15 @@ class TableauSiteSource:
|
|
|
1561
1561
|
}}""",
|
|
1562
1562
|
)
|
|
1563
1563
|
else:
|
|
1564
|
-
# As of Tableau Server 2024.2, the metadata API sporadically returns a 30-second
|
|
1565
|
-
# timeout error.
|
|
1566
|
-
# It doesn't reliably happen, so retrying a couple of times makes sense.
|
|
1567
1564
|
if all(
|
|
1565
|
+
# As of Tableau Server 2024.2, the metadata API sporadically returns a 30-second
|
|
1566
|
+
# timeout error.
|
|
1567
|
+
# It doesn't reliably happen, so retrying a couple of times makes sense.
|
|
1568
1568
|
error.get("message")
|
|
1569
1569
|
== "Execution canceled because timeout of 30000 millis was reached"
|
|
1570
|
+
# The Metadata API sometimes returns an 'unexpected error' message when querying
|
|
1571
|
+
# embeddedDatasourcesConnection. Try retrying a couple of times.
|
|
1572
|
+
or error.get("message") == "Unexpected error occurred"
|
|
1570
1573
|
for error in errors
|
|
1571
1574
|
):
|
|
1572
1575
|
# If it was only a timeout error, we can retry.
|
|
@@ -1578,8 +1581,8 @@ class TableauSiteSource:
|
|
|
1578
1581
|
(self.config.max_retries - retries_remaining + 1) ** 2, 60
|
|
1579
1582
|
)
|
|
1580
1583
|
logger.info(
|
|
1581
|
-
f"Query {connection_type} received a
|
|
1582
|
-
f"
|
|
1584
|
+
f"Query {connection_type} received a retryable error with {retries_remaining} retries remaining, "
|
|
1585
|
+
f"will retry in {backoff_time} seconds: {errors}"
|
|
1583
1586
|
)
|
|
1584
1587
|
time.sleep(backoff_time)
|
|
1585
1588
|
return self.get_connection_object_page(
|
|
@@ -10315,7 +10315,7 @@ class DataProductKeyClass(_Aspect):
|
|
|
10315
10315
|
|
|
10316
10316
|
|
|
10317
10317
|
ASPECT_NAME = 'dataProductKey'
|
|
10318
|
-
ASPECT_INFO = {'keyForEntity': 'dataProduct', 'entityCategory': 'core', 'entityAspects': ['ownership', 'glossaryTerms', 'globalTags', 'domains', 'applications', 'dataProductProperties', 'institutionalMemory', 'status', 'structuredProperties', 'forms', 'testResults', 'subTypes']}
|
|
10318
|
+
ASPECT_INFO = {'keyForEntity': 'dataProduct', 'entityCategory': 'core', 'entityAspects': ['ownership', 'glossaryTerms', 'globalTags', 'domains', 'applications', 'dataProductProperties', 'institutionalMemory', 'status', 'structuredProperties', 'forms', 'testResults', 'subTypes', 'assetSettings']}
|
|
10319
10319
|
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.dataproduct.DataProductKey")
|
|
10320
10320
|
|
|
10321
10321
|
def __init__(self,
|
|
@@ -14700,7 +14700,7 @@ class IncidentInfoClass(_Aspect):
|
|
|
14700
14700
|
customType: Union[None, str]=None,
|
|
14701
14701
|
title: Union[None, str]=None,
|
|
14702
14702
|
description: Union[None, str]=None,
|
|
14703
|
-
priority:
|
|
14703
|
+
priority: Union[None, int]=None,
|
|
14704
14704
|
assignees: Union[None, List["IncidentAssigneeClass"]]=None,
|
|
14705
14705
|
source: Union[None, "IncidentSourceClass"]=None,
|
|
14706
14706
|
startedAt: Union[None, int]=None,
|
|
@@ -14712,11 +14712,7 @@ class IncidentInfoClass(_Aspect):
|
|
|
14712
14712
|
self.title = title
|
|
14713
14713
|
self.description = description
|
|
14714
14714
|
self.entities = entities
|
|
14715
|
-
|
|
14716
|
-
# default: 0
|
|
14717
|
-
self.priority = self.RECORD_SCHEMA.fields_dict["priority"].default
|
|
14718
|
-
else:
|
|
14719
|
-
self.priority = priority
|
|
14715
|
+
self.priority = priority
|
|
14720
14716
|
self.assignees = assignees
|
|
14721
14717
|
self.status = status
|
|
14722
14718
|
self.source = source
|
|
@@ -14788,14 +14784,14 @@ class IncidentInfoClass(_Aspect):
|
|
|
14788
14784
|
|
|
14789
14785
|
|
|
14790
14786
|
@property
|
|
14791
|
-
def priority(self) -> Union[
|
|
14787
|
+
def priority(self) -> Union[None, int]:
|
|
14792
14788
|
"""A numeric severity or priority for the incident. On the UI we will translate this into something easy to understand.
|
|
14793
14789
|
Currently supported: 0 - CRITICAL, 1 - HIGH, 2 - MED, 3 - LOW
|
|
14794
14790
|
(We probably should have modeled as an enum)"""
|
|
14795
14791
|
return self._inner_dict.get('priority') # type: ignore
|
|
14796
14792
|
|
|
14797
14793
|
@priority.setter
|
|
14798
|
-
def priority(self, value: Union[
|
|
14794
|
+
def priority(self, value: Union[None, int]) -> None:
|
|
14799
14795
|
self._inner_dict['priority'] = value
|
|
14800
14796
|
|
|
14801
14797
|
|
|
@@ -16293,7 +16289,7 @@ class DomainKeyClass(_Aspect):
|
|
|
16293
16289
|
|
|
16294
16290
|
|
|
16295
16291
|
ASPECT_NAME = 'domainKey'
|
|
16296
|
-
ASPECT_INFO = {'keyForEntity': 'domain', 'entityCategory': 'core', 'entityAspects': ['domainProperties', 'institutionalMemory', 'ownership', 'structuredProperties', 'forms', 'testResults', 'displayProperties'], 'entityDoc': 'A data domain within an organization.'}
|
|
16292
|
+
ASPECT_INFO = {'keyForEntity': 'domain', 'entityCategory': 'core', 'entityAspects': ['domainProperties', 'institutionalMemory', 'ownership', 'structuredProperties', 'forms', 'testResults', 'displayProperties', 'assetSettings'], 'entityDoc': 'A data domain within an organization.'}
|
|
16297
16293
|
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DomainKey")
|
|
16298
16294
|
|
|
16299
16295
|
def __init__(self,
|
|
@@ -16438,7 +16434,7 @@ class GlossaryNodeKeyClass(_Aspect):
|
|
|
16438
16434
|
|
|
16439
16435
|
|
|
16440
16436
|
ASPECT_NAME = 'glossaryNodeKey'
|
|
16441
|
-
ASPECT_INFO = {'keyForEntity': 'glossaryNode', 'entityCategory': 'core', 'entityAspects': ['glossaryNodeInfo', 'institutionalMemory', 'ownership', 'status', 'structuredProperties', 'forms', 'testResults', 'subTypes', 'displayProperties']}
|
|
16437
|
+
ASPECT_INFO = {'keyForEntity': 'glossaryNode', 'entityCategory': 'core', 'entityAspects': ['glossaryNodeInfo', 'institutionalMemory', 'ownership', 'status', 'structuredProperties', 'forms', 'testResults', 'subTypes', 'displayProperties', 'assetSettings']}
|
|
16442
16438
|
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.GlossaryNodeKey")
|
|
16443
16439
|
|
|
16444
16440
|
def __init__(self,
|
|
@@ -16467,7 +16463,7 @@ class GlossaryTermKeyClass(_Aspect):
|
|
|
16467
16463
|
|
|
16468
16464
|
|
|
16469
16465
|
ASPECT_NAME = 'glossaryTermKey'
|
|
16470
|
-
ASPECT_INFO = {'keyForEntity': 'glossaryTerm', 'entityCategory': 'core', 'entityAspects': ['glossaryTermInfo', 'glossaryRelatedTerms', 'institutionalMemory', 'schemaMetadata', 'ownership', 'deprecation', 'domains', 'applications', 'status', 'browsePaths', 'structuredProperties', 'forms', 'testResults', 'subTypes']}
|
|
16466
|
+
ASPECT_INFO = {'keyForEntity': 'glossaryTerm', 'entityCategory': 'core', 'entityAspects': ['glossaryTermInfo', 'glossaryRelatedTerms', 'institutionalMemory', 'schemaMetadata', 'ownership', 'deprecation', 'domains', 'applications', 'status', 'browsePaths', 'structuredProperties', 'forms', 'testResults', 'subTypes', 'assetSettings']}
|
|
16471
16467
|
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.GlossaryTermKey")
|
|
16472
16468
|
|
|
16473
16469
|
def __init__(self,
|
|
@@ -20178,6 +20174,18 @@ class DataHubPageModuleTypeClass(object):
|
|
|
20178
20174
|
DOMAINS = "DOMAINS"
|
|
20179
20175
|
"""Module displaying the top domains"""
|
|
20180
20176
|
|
|
20177
|
+
ASSETS = "ASSETS"
|
|
20178
|
+
"""Module displaying the assets of parent entity"""
|
|
20179
|
+
|
|
20180
|
+
CHILD_HIERARCHY = "CHILD_HIERARCHY"
|
|
20181
|
+
"""Module displaying the hierarchy of the children of a given entity. Glossary or Domains."""
|
|
20182
|
+
|
|
20183
|
+
DATA_PRODUCTS = "DATA_PRODUCTS"
|
|
20184
|
+
"""Module displaying child data products of a given domain"""
|
|
20185
|
+
|
|
20186
|
+
RELATED_TERMS = "RELATED_TERMS"
|
|
20187
|
+
"""Module displaying the related terms of a given glossary term"""
|
|
20188
|
+
|
|
20181
20189
|
|
|
20182
20190
|
|
|
20183
20191
|
class DataHubPageModuleVisibilityClass(DictWrapper):
|
|
@@ -24634,6 +24642,89 @@ class DataHubSecretValueClass(_Aspect):
|
|
|
24634
24642
|
self._inner_dict['created'] = value
|
|
24635
24643
|
|
|
24636
24644
|
|
|
24645
|
+
class AssetSettingsClass(_Aspect):
|
|
24646
|
+
"""Settings associated with this asset"""
|
|
24647
|
+
|
|
24648
|
+
|
|
24649
|
+
ASPECT_NAME = 'assetSettings'
|
|
24650
|
+
ASPECT_INFO = {}
|
|
24651
|
+
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.settings.asset.AssetSettings")
|
|
24652
|
+
|
|
24653
|
+
def __init__(self,
|
|
24654
|
+
assetSummary: Union[None, "AssetSummarySettingsClass"]=None,
|
|
24655
|
+
):
|
|
24656
|
+
super().__init__()
|
|
24657
|
+
|
|
24658
|
+
self.assetSummary = assetSummary
|
|
24659
|
+
|
|
24660
|
+
def _restore_defaults(self) -> None:
|
|
24661
|
+
self.assetSummary = self.RECORD_SCHEMA.fields_dict["assetSummary"].default
|
|
24662
|
+
|
|
24663
|
+
|
|
24664
|
+
@property
|
|
24665
|
+
def assetSummary(self) -> Union[None, "AssetSummarySettingsClass"]:
|
|
24666
|
+
"""Information related to the asset summary for this asset"""
|
|
24667
|
+
return self._inner_dict.get('assetSummary') # type: ignore
|
|
24668
|
+
|
|
24669
|
+
@assetSummary.setter
|
|
24670
|
+
def assetSummary(self, value: Union[None, "AssetSummarySettingsClass"]) -> None:
|
|
24671
|
+
self._inner_dict['assetSummary'] = value
|
|
24672
|
+
|
|
24673
|
+
|
|
24674
|
+
class AssetSummarySettingsClass(DictWrapper):
|
|
24675
|
+
"""Information related to the asset summary for this asset"""
|
|
24676
|
+
|
|
24677
|
+
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.settings.asset.AssetSummarySettings")
|
|
24678
|
+
def __init__(self,
|
|
24679
|
+
templates: Optional[Union[List["AssetSummarySettingsTemplateClass"], None]]=None,
|
|
24680
|
+
):
|
|
24681
|
+
super().__init__()
|
|
24682
|
+
|
|
24683
|
+
if templates is None:
|
|
24684
|
+
# default: []
|
|
24685
|
+
self.templates = list()
|
|
24686
|
+
else:
|
|
24687
|
+
self.templates = templates
|
|
24688
|
+
|
|
24689
|
+
def _restore_defaults(self) -> None:
|
|
24690
|
+
self.templates = list()
|
|
24691
|
+
|
|
24692
|
+
|
|
24693
|
+
@property
|
|
24694
|
+
def templates(self) -> Union[List["AssetSummarySettingsTemplateClass"], None]:
|
|
24695
|
+
"""The list of templates applied to this asset in order. Right now we only expect one."""
|
|
24696
|
+
return self._inner_dict.get('templates') # type: ignore
|
|
24697
|
+
|
|
24698
|
+
@templates.setter
|
|
24699
|
+
def templates(self, value: Union[List["AssetSummarySettingsTemplateClass"], None]) -> None:
|
|
24700
|
+
self._inner_dict['templates'] = value
|
|
24701
|
+
|
|
24702
|
+
|
|
24703
|
+
class AssetSummarySettingsTemplateClass(DictWrapper):
|
|
24704
|
+
"""Object containing the template and any additional info for asset summary settings"""
|
|
24705
|
+
|
|
24706
|
+
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.settings.asset.AssetSummarySettingsTemplate")
|
|
24707
|
+
def __init__(self,
|
|
24708
|
+
template: str,
|
|
24709
|
+
):
|
|
24710
|
+
super().__init__()
|
|
24711
|
+
|
|
24712
|
+
self.template = template
|
|
24713
|
+
|
|
24714
|
+
def _restore_defaults(self) -> None:
|
|
24715
|
+
self.template = str()
|
|
24716
|
+
|
|
24717
|
+
|
|
24718
|
+
@property
|
|
24719
|
+
def template(self) -> str:
|
|
24720
|
+
"""The urn of the template"""
|
|
24721
|
+
return self._inner_dict.get('template') # type: ignore
|
|
24722
|
+
|
|
24723
|
+
@template.setter
|
|
24724
|
+
def template(self, value: str) -> None:
|
|
24725
|
+
self._inner_dict['template'] = value
|
|
24726
|
+
|
|
24727
|
+
|
|
24637
24728
|
class ApplicationsSettingsClass(DictWrapper):
|
|
24638
24729
|
# No docs available.
|
|
24639
24730
|
|
|
@@ -25817,6 +25908,31 @@ class TelemetryClientIdClass(_Aspect):
|
|
|
25817
25908
|
self._inner_dict['clientId'] = value
|
|
25818
25909
|
|
|
25819
25910
|
|
|
25911
|
+
class DataHubPageTemplateAssetSummaryClass(DictWrapper):
|
|
25912
|
+
"""The page template info for asset summaries"""
|
|
25913
|
+
|
|
25914
|
+
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.template.DataHubPageTemplateAssetSummary")
|
|
25915
|
+
def __init__(self,
|
|
25916
|
+
summaryElements: Union[None, List["SummaryElementClass"]]=None,
|
|
25917
|
+
):
|
|
25918
|
+
super().__init__()
|
|
25919
|
+
|
|
25920
|
+
self.summaryElements = summaryElements
|
|
25921
|
+
|
|
25922
|
+
def _restore_defaults(self) -> None:
|
|
25923
|
+
self.summaryElements = self.RECORD_SCHEMA.fields_dict["summaryElements"].default
|
|
25924
|
+
|
|
25925
|
+
|
|
25926
|
+
@property
|
|
25927
|
+
def summaryElements(self) -> Union[None, List["SummaryElementClass"]]:
|
|
25928
|
+
"""The optional list of properties shown on an asset summary page header."""
|
|
25929
|
+
return self._inner_dict.get('summaryElements') # type: ignore
|
|
25930
|
+
|
|
25931
|
+
@summaryElements.setter
|
|
25932
|
+
def summaryElements(self, value: Union[None, List["SummaryElementClass"]]) -> None:
|
|
25933
|
+
self._inner_dict['summaryElements'] = value
|
|
25934
|
+
|
|
25935
|
+
|
|
25820
25936
|
class DataHubPageTemplatePropertiesClass(_Aspect):
|
|
25821
25937
|
"""The main properties of a DataHub page template"""
|
|
25822
25938
|
|
|
@@ -25831,10 +25947,12 @@ class DataHubPageTemplatePropertiesClass(_Aspect):
|
|
|
25831
25947
|
visibility: "DataHubPageTemplateVisibilityClass",
|
|
25832
25948
|
created: "AuditStampClass",
|
|
25833
25949
|
lastModified: "AuditStampClass",
|
|
25950
|
+
assetSummary: Union[None, "DataHubPageTemplateAssetSummaryClass"]=None,
|
|
25834
25951
|
):
|
|
25835
25952
|
super().__init__()
|
|
25836
25953
|
|
|
25837
25954
|
self.rows = rows
|
|
25955
|
+
self.assetSummary = assetSummary
|
|
25838
25956
|
self.surface = surface
|
|
25839
25957
|
self.visibility = visibility
|
|
25840
25958
|
self.created = created
|
|
@@ -25842,6 +25960,7 @@ class DataHubPageTemplatePropertiesClass(_Aspect):
|
|
|
25842
25960
|
|
|
25843
25961
|
def _restore_defaults(self) -> None:
|
|
25844
25962
|
self.rows = list()
|
|
25963
|
+
self.assetSummary = self.RECORD_SCHEMA.fields_dict["assetSummary"].default
|
|
25845
25964
|
self.surface = DataHubPageTemplateSurfaceClass._construct_with_defaults()
|
|
25846
25965
|
self.visibility = DataHubPageTemplateVisibilityClass._construct_with_defaults()
|
|
25847
25966
|
self.created = AuditStampClass._construct_with_defaults()
|
|
@@ -25858,6 +25977,16 @@ class DataHubPageTemplatePropertiesClass(_Aspect):
|
|
|
25858
25977
|
self._inner_dict['rows'] = value
|
|
25859
25978
|
|
|
25860
25979
|
|
|
25980
|
+
@property
|
|
25981
|
+
def assetSummary(self) -> Union[None, "DataHubPageTemplateAssetSummaryClass"]:
|
|
25982
|
+
"""The optional info for asset summaries. Should be populated if surfaceType is ASSET_SUMMARY"""
|
|
25983
|
+
return self._inner_dict.get('assetSummary') # type: ignore
|
|
25984
|
+
|
|
25985
|
+
@assetSummary.setter
|
|
25986
|
+
def assetSummary(self, value: Union[None, "DataHubPageTemplateAssetSummaryClass"]) -> None:
|
|
25987
|
+
self._inner_dict['assetSummary'] = value
|
|
25988
|
+
|
|
25989
|
+
|
|
25861
25990
|
@property
|
|
25862
25991
|
def surface(self) -> "DataHubPageTemplateSurfaceClass":
|
|
25863
25992
|
"""Info about the surface area of the product that this template is deployed in"""
|
|
@@ -25990,6 +26119,58 @@ class PageTemplateSurfaceTypeClass(object):
|
|
|
25990
26119
|
HOME_PAGE = "HOME_PAGE"
|
|
25991
26120
|
"""This template applies to what to display on the home page for users."""
|
|
25992
26121
|
|
|
26122
|
+
ASSET_SUMMARY = "ASSET_SUMMARY"
|
|
26123
|
+
"""This template applies to what to display on asset summary pages"""
|
|
26124
|
+
|
|
26125
|
+
|
|
26126
|
+
|
|
26127
|
+
class SummaryElementClass(DictWrapper):
|
|
26128
|
+
"""Info for a given asset summary element"""
|
|
26129
|
+
|
|
26130
|
+
RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.template.SummaryElement")
|
|
26131
|
+
def __init__(self,
|
|
26132
|
+
elementType: Union[str, "SummaryElementTypeClass"],
|
|
26133
|
+
structuredPropertyUrn: Union[None, str]=None,
|
|
26134
|
+
):
|
|
26135
|
+
super().__init__()
|
|
26136
|
+
|
|
26137
|
+
self.elementType = elementType
|
|
26138
|
+
self.structuredPropertyUrn = structuredPropertyUrn
|
|
26139
|
+
|
|
26140
|
+
def _restore_defaults(self) -> None:
|
|
26141
|
+
self.elementType = SummaryElementTypeClass.CREATED
|
|
26142
|
+
self.structuredPropertyUrn = self.RECORD_SCHEMA.fields_dict["structuredPropertyUrn"].default
|
|
26143
|
+
|
|
26144
|
+
|
|
26145
|
+
@property
|
|
26146
|
+
def elementType(self) -> Union[str, "SummaryElementTypeClass"]:
|
|
26147
|
+
"""The type of element/property"""
|
|
26148
|
+
return self._inner_dict.get('elementType') # type: ignore
|
|
26149
|
+
|
|
26150
|
+
@elementType.setter
|
|
26151
|
+
def elementType(self, value: Union[str, "SummaryElementTypeClass"]) -> None:
|
|
26152
|
+
self._inner_dict['elementType'] = value
|
|
26153
|
+
|
|
26154
|
+
|
|
26155
|
+
@property
|
|
26156
|
+
def structuredPropertyUrn(self) -> Union[None, str]:
|
|
26157
|
+
"""The urn of the structured property shown. Required if propertyType is STRUCTURED_PROPERTY"""
|
|
26158
|
+
return self._inner_dict.get('structuredPropertyUrn') # type: ignore
|
|
26159
|
+
|
|
26160
|
+
@structuredPropertyUrn.setter
|
|
26161
|
+
def structuredPropertyUrn(self, value: Union[None, str]) -> None:
|
|
26162
|
+
self._inner_dict['structuredPropertyUrn'] = value
|
|
26163
|
+
|
|
26164
|
+
|
|
26165
|
+
class SummaryElementTypeClass(object):
|
|
26166
|
+
# No docs available.
|
|
26167
|
+
|
|
26168
|
+
CREATED = "CREATED"
|
|
26169
|
+
TAGS = "TAGS"
|
|
26170
|
+
GLOSSARY_TERMS = "GLOSSARY_TERMS"
|
|
26171
|
+
OWNERS = "OWNERS"
|
|
26172
|
+
DOMAIN = "DOMAIN"
|
|
26173
|
+
STRUCTURED_PROPERTY = "STRUCTURED_PROPERTY"
|
|
25993
26174
|
|
|
25994
26175
|
|
|
25995
26176
|
class TestDefinitionClass(DictWrapper):
|
|
@@ -27408,6 +27589,9 @@ __SCHEMA_TYPES = {
|
|
|
27408
27589
|
'com.linkedin.pegasus2avro.schemafield.SchemaFieldAliases': SchemaFieldAliasesClass,
|
|
27409
27590
|
'com.linkedin.pegasus2avro.schemafield.SchemaFieldInfo': SchemaFieldInfoClass,
|
|
27410
27591
|
'com.linkedin.pegasus2avro.secret.DataHubSecretValue': DataHubSecretValueClass,
|
|
27592
|
+
'com.linkedin.pegasus2avro.settings.asset.AssetSettings': AssetSettingsClass,
|
|
27593
|
+
'com.linkedin.pegasus2avro.settings.asset.AssetSummarySettings': AssetSummarySettingsClass,
|
|
27594
|
+
'com.linkedin.pegasus2avro.settings.asset.AssetSummarySettingsTemplate': AssetSummarySettingsTemplateClass,
|
|
27411
27595
|
'com.linkedin.pegasus2avro.settings.global.ApplicationsSettings': ApplicationsSettingsClass,
|
|
27412
27596
|
'com.linkedin.pegasus2avro.settings.global.DocPropagationFeatureSettings': DocPropagationFeatureSettingsClass,
|
|
27413
27597
|
'com.linkedin.pegasus2avro.settings.global.GlobalHomePageSettings': GlobalHomePageSettingsClass,
|
|
@@ -27425,12 +27609,15 @@ __SCHEMA_TYPES = {
|
|
|
27425
27609
|
'com.linkedin.pegasus2avro.structured.StructuredPropertyValueAssignment': StructuredPropertyValueAssignmentClass,
|
|
27426
27610
|
'com.linkedin.pegasus2avro.tag.TagProperties': TagPropertiesClass,
|
|
27427
27611
|
'com.linkedin.pegasus2avro.telemetry.TelemetryClientId': TelemetryClientIdClass,
|
|
27612
|
+
'com.linkedin.pegasus2avro.template.DataHubPageTemplateAssetSummary': DataHubPageTemplateAssetSummaryClass,
|
|
27428
27613
|
'com.linkedin.pegasus2avro.template.DataHubPageTemplateProperties': DataHubPageTemplatePropertiesClass,
|
|
27429
27614
|
'com.linkedin.pegasus2avro.template.DataHubPageTemplateRow': DataHubPageTemplateRowClass,
|
|
27430
27615
|
'com.linkedin.pegasus2avro.template.DataHubPageTemplateSurface': DataHubPageTemplateSurfaceClass,
|
|
27431
27616
|
'com.linkedin.pegasus2avro.template.DataHubPageTemplateVisibility': DataHubPageTemplateVisibilityClass,
|
|
27432
27617
|
'com.linkedin.pegasus2avro.template.PageTemplateScope': PageTemplateScopeClass,
|
|
27433
27618
|
'com.linkedin.pegasus2avro.template.PageTemplateSurfaceType': PageTemplateSurfaceTypeClass,
|
|
27619
|
+
'com.linkedin.pegasus2avro.template.SummaryElement': SummaryElementClass,
|
|
27620
|
+
'com.linkedin.pegasus2avro.template.SummaryElementType': SummaryElementTypeClass,
|
|
27434
27621
|
'com.linkedin.pegasus2avro.test.TestDefinition': TestDefinitionClass,
|
|
27435
27622
|
'com.linkedin.pegasus2avro.test.TestDefinitionType': TestDefinitionTypeClass,
|
|
27436
27623
|
'com.linkedin.pegasus2avro.test.TestInfo': TestInfoClass,
|
|
@@ -27918,6 +28105,9 @@ __SCHEMA_TYPES = {
|
|
|
27918
28105
|
'SchemaFieldAliases': SchemaFieldAliasesClass,
|
|
27919
28106
|
'SchemaFieldInfo': SchemaFieldInfoClass,
|
|
27920
28107
|
'DataHubSecretValue': DataHubSecretValueClass,
|
|
28108
|
+
'AssetSettings': AssetSettingsClass,
|
|
28109
|
+
'AssetSummarySettings': AssetSummarySettingsClass,
|
|
28110
|
+
'AssetSummarySettingsTemplate': AssetSummarySettingsTemplateClass,
|
|
27921
28111
|
'ApplicationsSettings': ApplicationsSettingsClass,
|
|
27922
28112
|
'DocPropagationFeatureSettings': DocPropagationFeatureSettingsClass,
|
|
27923
28113
|
'GlobalHomePageSettings': GlobalHomePageSettingsClass,
|
|
@@ -27935,12 +28125,15 @@ __SCHEMA_TYPES = {
|
|
|
27935
28125
|
'StructuredPropertyValueAssignment': StructuredPropertyValueAssignmentClass,
|
|
27936
28126
|
'TagProperties': TagPropertiesClass,
|
|
27937
28127
|
'TelemetryClientId': TelemetryClientIdClass,
|
|
28128
|
+
'DataHubPageTemplateAssetSummary': DataHubPageTemplateAssetSummaryClass,
|
|
27938
28129
|
'DataHubPageTemplateProperties': DataHubPageTemplatePropertiesClass,
|
|
27939
28130
|
'DataHubPageTemplateRow': DataHubPageTemplateRowClass,
|
|
27940
28131
|
'DataHubPageTemplateSurface': DataHubPageTemplateSurfaceClass,
|
|
27941
28132
|
'DataHubPageTemplateVisibility': DataHubPageTemplateVisibilityClass,
|
|
27942
28133
|
'PageTemplateScope': PageTemplateScopeClass,
|
|
27943
28134
|
'PageTemplateSurfaceType': PageTemplateSurfaceTypeClass,
|
|
28135
|
+
'SummaryElement': SummaryElementClass,
|
|
28136
|
+
'SummaryElementType': SummaryElementTypeClass,
|
|
27944
28137
|
'TestDefinition': TestDefinitionClass,
|
|
27945
28138
|
'TestDefinitionType': TestDefinitionTypeClass,
|
|
27946
28139
|
'TestInfo': TestInfoClass,
|
|
@@ -28110,6 +28303,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
|
|
|
28110
28303
|
NotebookInfoClass,
|
|
28111
28304
|
NotebookContentClass,
|
|
28112
28305
|
GlobalSettingsInfoClass,
|
|
28306
|
+
AssetSettingsClass,
|
|
28113
28307
|
DataHubViewInfoClass,
|
|
28114
28308
|
DataJobInfoClass,
|
|
28115
28309
|
EditableDataFlowPropertiesClass,
|
|
@@ -28343,6 +28537,7 @@ class AspectBag(TypedDict, total=False):
|
|
|
28343
28537
|
notebookInfo: NotebookInfoClass
|
|
28344
28538
|
notebookContent: NotebookContentClass
|
|
28345
28539
|
globalSettingsInfo: GlobalSettingsInfoClass
|
|
28540
|
+
assetSettings: AssetSettingsClass
|
|
28346
28541
|
dataHubViewInfo: DataHubViewInfoClass
|
|
28347
28542
|
dataJobInfo: DataJobInfoClass
|
|
28348
28543
|
editableDataFlowProperties: EditableDataFlowPropertiesClass
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# mypy: ignore-errors
|
|
2
|
+
# flake8: noqa
|
|
3
|
+
|
|
4
|
+
# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
|
|
5
|
+
# Do not modify manually!
|
|
6
|
+
|
|
7
|
+
# pylint: skip-file
|
|
8
|
+
# fmt: off
|
|
9
|
+
# isort: skip_file
|
|
10
|
+
from ......schema_classes import AssetSettingsClass
|
|
11
|
+
from ......schema_classes import AssetSummarySettingsClass
|
|
12
|
+
from ......schema_classes import AssetSummarySettingsTemplateClass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
AssetSettings = AssetSettingsClass
|
|
16
|
+
AssetSummarySettings = AssetSummarySettingsClass
|
|
17
|
+
AssetSummarySettingsTemplate = AssetSummarySettingsTemplateClass
|
|
18
|
+
|
|
19
|
+
# fmt: on
|
|
@@ -7,19 +7,25 @@
|
|
|
7
7
|
# pylint: skip-file
|
|
8
8
|
# fmt: off
|
|
9
9
|
# isort: skip_file
|
|
10
|
+
from .....schema_classes import DataHubPageTemplateAssetSummaryClass
|
|
10
11
|
from .....schema_classes import DataHubPageTemplatePropertiesClass
|
|
11
12
|
from .....schema_classes import DataHubPageTemplateRowClass
|
|
12
13
|
from .....schema_classes import DataHubPageTemplateSurfaceClass
|
|
13
14
|
from .....schema_classes import DataHubPageTemplateVisibilityClass
|
|
14
15
|
from .....schema_classes import PageTemplateScopeClass
|
|
15
16
|
from .....schema_classes import PageTemplateSurfaceTypeClass
|
|
17
|
+
from .....schema_classes import SummaryElementClass
|
|
18
|
+
from .....schema_classes import SummaryElementTypeClass
|
|
16
19
|
|
|
17
20
|
|
|
21
|
+
DataHubPageTemplateAssetSummary = DataHubPageTemplateAssetSummaryClass
|
|
18
22
|
DataHubPageTemplateProperties = DataHubPageTemplatePropertiesClass
|
|
19
23
|
DataHubPageTemplateRow = DataHubPageTemplateRowClass
|
|
20
24
|
DataHubPageTemplateSurface = DataHubPageTemplateSurfaceClass
|
|
21
25
|
DataHubPageTemplateVisibility = DataHubPageTemplateVisibilityClass
|
|
22
26
|
PageTemplateScope = PageTemplateScopeClass
|
|
23
27
|
PageTemplateSurfaceType = PageTemplateSurfaceTypeClass
|
|
28
|
+
SummaryElement = SummaryElementClass
|
|
29
|
+
SummaryElementType = SummaryElementTypeClass
|
|
24
30
|
|
|
25
31
|
# fmt: on
|