acryl-datahub 1.2.0.10rc4__py3-none-any.whl → 1.2.0.10rc6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/METADATA +2451 -2425
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/RECORD +21 -14
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/entry_points.txt +2 -0
- datahub/_version.py +1 -1
- datahub/ingestion/autogenerated/capability_summary.json +33 -1
- datahub/ingestion/source/looker/looker_source.py +52 -3
- datahub/ingestion/source/redshift/query.py +23 -19
- datahub/ingestion/source/snaplogic/__init__.py +0 -0
- datahub/ingestion/source/snaplogic/snaplogic.py +355 -0
- datahub/ingestion/source/snaplogic/snaplogic_config.py +37 -0
- datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py +107 -0
- datahub/ingestion/source/snaplogic/snaplogic_parser.py +168 -0
- datahub/ingestion/source/snaplogic/snaplogic_utils.py +31 -0
- datahub/ingestion/source/tableau/tableau.py +3 -3
- datahub/ingestion/transformer/set_browse_path.py +112 -0
- datahub/sdk/_shared.py +126 -0
- datahub/sdk/chart.py +87 -30
- datahub/sdk/dashboard.py +79 -32
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/top_level.txt +0 -0
|
@@ -594,13 +594,13 @@ class TableauConfig(
|
|
|
594
594
|
)
|
|
595
595
|
|
|
596
596
|
extract_lineage_from_unsupported_custom_sql_queries: bool = Field(
|
|
597
|
-
default=
|
|
598
|
-
description="[Experimental]
|
|
597
|
+
default=True,
|
|
598
|
+
description="[Experimental] Extract lineage from Custom SQL queries using DataHub's SQL parser in cases where the Tableau Catalog API fails to return lineage for the query.",
|
|
599
599
|
)
|
|
600
600
|
|
|
601
601
|
force_extraction_of_lineage_from_custom_sql_queries: bool = Field(
|
|
602
602
|
default=False,
|
|
603
|
-
description="[Experimental] Force extraction of lineage from
|
|
603
|
+
description="[Experimental] Force extraction of lineage from Custom SQL queries using DataHub's SQL parser, even when the Tableau Catalog API returns lineage already.",
|
|
604
604
|
)
|
|
605
605
|
|
|
606
606
|
sql_parsing_disable_schema_awareness: bool = Field(
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
from typing import Dict, List, Optional, cast
|
|
4
|
+
|
|
5
|
+
from datahub.configuration.common import (
|
|
6
|
+
TransformerSemanticsConfigModel,
|
|
7
|
+
)
|
|
8
|
+
from datahub.emitter.mce_builder import Aspect
|
|
9
|
+
from datahub.ingestion.api.common import PipelineContext
|
|
10
|
+
from datahub.ingestion.transformer.base_transformer import (
|
|
11
|
+
BaseTransformer,
|
|
12
|
+
SingleAspectTransformer,
|
|
13
|
+
)
|
|
14
|
+
from datahub.metadata.schema_classes import (
|
|
15
|
+
BrowsePathEntryClass,
|
|
16
|
+
BrowsePathsV2Class,
|
|
17
|
+
)
|
|
18
|
+
from datahub.utilities.urns.urn import guess_entity_type
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SetBrowsePathTransformerConfig(TransformerSemanticsConfigModel):
|
|
22
|
+
path: List[str]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SetBrowsePathTransformer(BaseTransformer, SingleAspectTransformer):
|
|
26
|
+
ctx: PipelineContext
|
|
27
|
+
config: SetBrowsePathTransformerConfig
|
|
28
|
+
|
|
29
|
+
def __init__(self, config: SetBrowsePathTransformerConfig, ctx: PipelineContext):
|
|
30
|
+
super().__init__()
|
|
31
|
+
self.ctx = ctx
|
|
32
|
+
self.config = config
|
|
33
|
+
|
|
34
|
+
def aspect_name(self) -> str:
|
|
35
|
+
return "browsePathsV2"
|
|
36
|
+
|
|
37
|
+
def entity_types(self) -> List[str]:
|
|
38
|
+
# This is an arbitrary list, might be adjusted if it makes sense. It might be reasonable to make it configurable
|
|
39
|
+
return ["dataset", "dataJob", "dataFlow", "chart", "dashboard", "container"]
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def create(
|
|
43
|
+
cls, config_dict: dict, ctx: PipelineContext
|
|
44
|
+
) -> "SetBrowsePathTransformer":
|
|
45
|
+
config = SetBrowsePathTransformerConfig.parse_obj(config_dict)
|
|
46
|
+
return cls(config, ctx)
|
|
47
|
+
|
|
48
|
+
@staticmethod
|
|
49
|
+
def _build_model(existing_browse_paths: BrowsePathsV2Class) -> Dict[str, List[str]]:
|
|
50
|
+
template_vars: Dict[str, List[str]] = {}
|
|
51
|
+
model: Dict[str, List[str]] = defaultdict(list)
|
|
52
|
+
for entry in existing_browse_paths.path or []:
|
|
53
|
+
if entry.urn:
|
|
54
|
+
entity_type = guess_entity_type(entry.urn)
|
|
55
|
+
model[entity_type].append(entry.urn)
|
|
56
|
+
|
|
57
|
+
for entity_type, urns in model.items():
|
|
58
|
+
template_vars[f"{entity_type}[*]"] = urns
|
|
59
|
+
for i, urn in enumerate(urns):
|
|
60
|
+
template_vars[f"{entity_type}[{i}]"] = [urn]
|
|
61
|
+
|
|
62
|
+
return template_vars
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def _expand_nodes(
|
|
66
|
+
cls, templates: List[str], template_vars: Dict[str, List[str]]
|
|
67
|
+
) -> BrowsePathsV2Class:
|
|
68
|
+
expanded_nodes: List[str] = []
|
|
69
|
+
for node in templates:
|
|
70
|
+
resolved_nodes = cls._resolve_template_to_nodes(node, template_vars)
|
|
71
|
+
expanded_nodes.extend(resolved_nodes)
|
|
72
|
+
|
|
73
|
+
processed_entries: List[BrowsePathEntryClass] = []
|
|
74
|
+
for node in expanded_nodes:
|
|
75
|
+
if not node or node.isspace():
|
|
76
|
+
continue
|
|
77
|
+
processed_entries.append(
|
|
78
|
+
BrowsePathEntryClass(
|
|
79
|
+
id=node, urn=node if node.startswith("urn:") else None
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
return BrowsePathsV2Class(path=processed_entries)
|
|
83
|
+
|
|
84
|
+
def transform_aspect(
|
|
85
|
+
self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect]
|
|
86
|
+
) -> Optional[Aspect]:
|
|
87
|
+
template_vars: Dict[str, List[str]] = {}
|
|
88
|
+
if aspect is not None:
|
|
89
|
+
assert isinstance(aspect, BrowsePathsV2Class)
|
|
90
|
+
template_vars = self._build_model(aspect)
|
|
91
|
+
new_browse_paths: BrowsePathsV2Class = self._expand_nodes(
|
|
92
|
+
self.config.path, template_vars
|
|
93
|
+
)
|
|
94
|
+
if aspect is not None and not self.config.replace_existing:
|
|
95
|
+
for node in aspect.path:
|
|
96
|
+
new_browse_paths.path.append(node)
|
|
97
|
+
|
|
98
|
+
return cast(Aspect, new_browse_paths)
|
|
99
|
+
|
|
100
|
+
@staticmethod
|
|
101
|
+
def _resolve_template_to_nodes(
|
|
102
|
+
template_str: str, template_vars: Dict[str, List[str]]
|
|
103
|
+
) -> List[str]:
|
|
104
|
+
# This mechanism can be made simpler (match against known variables only) or more complex (e.g. by using a
|
|
105
|
+
# proper templating engine, like jinja).
|
|
106
|
+
template_str = template_str.strip()
|
|
107
|
+
var_pattern = re.findall(r"^\$([a-zA-Z]+\[[0-9*]+]$)", template_str)
|
|
108
|
+
|
|
109
|
+
if not var_pattern:
|
|
110
|
+
return [template_str]
|
|
111
|
+
|
|
112
|
+
return template_vars.get(var_pattern[0], [])
|
datahub/sdk/_shared.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import warnings
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
4
5
|
from datetime import datetime
|
|
5
6
|
from typing import (
|
|
6
7
|
TYPE_CHECKING,
|
|
@@ -61,6 +62,7 @@ DataPlatformInstanceUrnOrStr: TypeAlias = Union[str, DataPlatformInstanceUrn]
|
|
|
61
62
|
DataPlatformUrnOrStr: TypeAlias = Union[str, DataPlatformUrn]
|
|
62
63
|
|
|
63
64
|
ActorUrn: TypeAlias = Union[CorpUserUrn, CorpGroupUrn]
|
|
65
|
+
ActorUrnOrStr: TypeAlias = Union[str, ActorUrn]
|
|
64
66
|
StructuredPropertyUrnOrStr: TypeAlias = Union[str, StructuredPropertyUrn]
|
|
65
67
|
StructuredPropertyValueType: TypeAlias = Union[str, float, int]
|
|
66
68
|
StructuredPropertyInputType: TypeAlias = Dict[
|
|
@@ -110,6 +112,130 @@ def parse_time_stamp(ts: Optional[models.TimeStampClass]) -> Optional[datetime]:
|
|
|
110
112
|
return parse_ts_millis(ts.time)
|
|
111
113
|
|
|
112
114
|
|
|
115
|
+
class ChangeAuditStampsMixin(ABC):
|
|
116
|
+
"""Mixin class for managing audit stamps on entities."""
|
|
117
|
+
|
|
118
|
+
__slots__ = ()
|
|
119
|
+
|
|
120
|
+
@abstractmethod
|
|
121
|
+
def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
|
|
122
|
+
"""Get the audit stamps from the entity properties."""
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
@abstractmethod
|
|
126
|
+
def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
|
|
127
|
+
"""Set the audit stamps on the entity properties."""
|
|
128
|
+
pass
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def last_modified(self) -> Optional[datetime]:
|
|
132
|
+
"""Get the last modification timestamp from audit stamps."""
|
|
133
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
134
|
+
if audit_stamps.lastModified.time == 0:
|
|
135
|
+
return None
|
|
136
|
+
return datetime.fromtimestamp(
|
|
137
|
+
audit_stamps.lastModified.time / 1000
|
|
138
|
+
) # supports only seconds precision
|
|
139
|
+
|
|
140
|
+
def set_last_modified(self, last_modified: datetime) -> None:
|
|
141
|
+
"""Set the last modification timestamp in audit stamps."""
|
|
142
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
143
|
+
audit_stamps.lastModified.time = make_ts_millis(last_modified)
|
|
144
|
+
self._set_audit_stamps(audit_stamps)
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def last_modified_by(self) -> Optional[str]:
|
|
148
|
+
"""Get the last modification actor from audit stamps."""
|
|
149
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
150
|
+
if audit_stamps.lastModified.actor == builder.UNKNOWN_USER:
|
|
151
|
+
return None
|
|
152
|
+
return audit_stamps.lastModified.actor
|
|
153
|
+
|
|
154
|
+
def set_last_modified_by(self, last_modified_by: ActorUrnOrStr) -> None:
|
|
155
|
+
"""Set the last modification actor in audit stamps."""
|
|
156
|
+
if isinstance(last_modified_by, str):
|
|
157
|
+
last_modified_by = make_user_urn(last_modified_by)
|
|
158
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
159
|
+
audit_stamps.lastModified.actor = str(last_modified_by)
|
|
160
|
+
self._set_audit_stamps(audit_stamps)
|
|
161
|
+
|
|
162
|
+
@property
|
|
163
|
+
def created_at(self) -> Optional[datetime]:
|
|
164
|
+
"""Get the creation timestamp from audit stamps."""
|
|
165
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
166
|
+
if audit_stamps.created.time == 0:
|
|
167
|
+
return None
|
|
168
|
+
return datetime.fromtimestamp(
|
|
169
|
+
audit_stamps.created.time / 1000
|
|
170
|
+
) # supports only seconds precision
|
|
171
|
+
|
|
172
|
+
def set_created_at(self, created_at: datetime) -> None:
|
|
173
|
+
"""Set the creation timestamp in audit stamps."""
|
|
174
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
175
|
+
audit_stamps.created.time = make_ts_millis(created_at)
|
|
176
|
+
self._set_audit_stamps(audit_stamps)
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def created_by(self) -> Optional[ActorUrnOrStr]:
|
|
180
|
+
"""Get the creation actor from audit stamps."""
|
|
181
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
182
|
+
if audit_stamps.created.actor == builder.UNKNOWN_USER:
|
|
183
|
+
return None
|
|
184
|
+
return audit_stamps.created.actor
|
|
185
|
+
|
|
186
|
+
def set_created_by(self, created_by: ActorUrnOrStr) -> None:
|
|
187
|
+
"""Set the creation actor in audit stamps."""
|
|
188
|
+
if isinstance(created_by, str):
|
|
189
|
+
created_by = make_user_urn(created_by)
|
|
190
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
191
|
+
audit_stamps.created.actor = str(created_by)
|
|
192
|
+
self._set_audit_stamps(audit_stamps)
|
|
193
|
+
|
|
194
|
+
@property
|
|
195
|
+
def deleted_on(self) -> Optional[datetime]:
|
|
196
|
+
"""Get the deletion timestamp from audit stamps."""
|
|
197
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
198
|
+
if audit_stamps.deleted is None or audit_stamps.deleted.time == 0:
|
|
199
|
+
return None
|
|
200
|
+
return datetime.fromtimestamp(
|
|
201
|
+
audit_stamps.deleted.time / 1000
|
|
202
|
+
) # supports only seconds precision
|
|
203
|
+
|
|
204
|
+
def set_deleted_on(self, deleted_on: datetime) -> None:
|
|
205
|
+
"""Set the deletion timestamp in audit stamps."""
|
|
206
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
207
|
+
# Default constructor sets deleted to None
|
|
208
|
+
if audit_stamps.deleted is None:
|
|
209
|
+
audit_stamps.deleted = models.AuditStampClass(
|
|
210
|
+
time=0, actor=builder.UNKNOWN_USER
|
|
211
|
+
)
|
|
212
|
+
audit_stamps.deleted.time = make_ts_millis(deleted_on)
|
|
213
|
+
self._set_audit_stamps(audit_stamps)
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def deleted_by(self) -> Optional[ActorUrnOrStr]:
|
|
217
|
+
"""Get the deletion actor from audit stamps."""
|
|
218
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
219
|
+
if (
|
|
220
|
+
audit_stamps.deleted is None
|
|
221
|
+
or audit_stamps.deleted.actor == builder.UNKNOWN_USER
|
|
222
|
+
):
|
|
223
|
+
return None
|
|
224
|
+
return audit_stamps.deleted.actor
|
|
225
|
+
|
|
226
|
+
def set_deleted_by(self, deleted_by: ActorUrnOrStr) -> None:
|
|
227
|
+
"""Set the deletion actor in audit stamps."""
|
|
228
|
+
if isinstance(deleted_by, str):
|
|
229
|
+
deleted_by = make_user_urn(deleted_by)
|
|
230
|
+
audit_stamps: models.ChangeAuditStampsClass = self._get_audit_stamps()
|
|
231
|
+
if audit_stamps.deleted is None:
|
|
232
|
+
audit_stamps.deleted = models.AuditStampClass(
|
|
233
|
+
time=0, actor=builder.UNKNOWN_USER
|
|
234
|
+
)
|
|
235
|
+
audit_stamps.deleted.actor = str(deleted_by)
|
|
236
|
+
self._set_audit_stamps(audit_stamps)
|
|
237
|
+
|
|
238
|
+
|
|
113
239
|
class HasPlatformInstance(Entity):
|
|
114
240
|
__slots__ = ()
|
|
115
241
|
|
datahub/sdk/chart.py
CHANGED
|
@@ -10,6 +10,8 @@ import datahub.metadata.schema_classes as models
|
|
|
10
10
|
from datahub.emitter.enum_helpers import get_enum_options
|
|
11
11
|
from datahub.metadata.urns import ChartUrn, DatasetUrn, Urn
|
|
12
12
|
from datahub.sdk._shared import (
|
|
13
|
+
ActorUrnOrStr,
|
|
14
|
+
ChangeAuditStampsMixin,
|
|
13
15
|
DataPlatformInstanceUrnOrStr,
|
|
14
16
|
DataPlatformUrnOrStr,
|
|
15
17
|
DatasetUrnOrStr,
|
|
@@ -34,6 +36,7 @@ from datahub.utilities.sentinels import Unset, unset
|
|
|
34
36
|
|
|
35
37
|
|
|
36
38
|
class Chart(
|
|
39
|
+
ChangeAuditStampsMixin,
|
|
37
40
|
HasPlatformInstance,
|
|
38
41
|
HasSubtype,
|
|
39
42
|
HasOwnership,
|
|
@@ -70,6 +73,11 @@ class Chart(
|
|
|
70
73
|
chart_url: Optional[str] = None,
|
|
71
74
|
custom_properties: Optional[Dict[str, str]] = None,
|
|
72
75
|
last_modified: Optional[datetime] = None,
|
|
76
|
+
last_modified_by: Optional[ActorUrnOrStr] = None,
|
|
77
|
+
created_at: Optional[datetime] = None,
|
|
78
|
+
created_by: Optional[ActorUrnOrStr] = None,
|
|
79
|
+
deleted_on: Optional[datetime] = None,
|
|
80
|
+
deleted_by: Optional[ActorUrnOrStr] = None,
|
|
73
81
|
last_refreshed: Optional[datetime] = None,
|
|
74
82
|
chart_type: Optional[Union[str, models.ChartTypeClass]] = None,
|
|
75
83
|
access: Optional[str] = None,
|
|
@@ -94,13 +102,60 @@ class Chart(
|
|
|
94
102
|
self._set_extra_aspects(extra_aspects)
|
|
95
103
|
|
|
96
104
|
self._set_platform_instance(platform, platform_instance)
|
|
97
|
-
|
|
98
105
|
self._ensure_chart_props(display_name=display_name)
|
|
106
|
+
self._init_chart_properties(
|
|
107
|
+
description,
|
|
108
|
+
display_name,
|
|
109
|
+
external_url,
|
|
110
|
+
chart_url,
|
|
111
|
+
custom_properties,
|
|
112
|
+
last_modified,
|
|
113
|
+
last_modified_by,
|
|
114
|
+
created_at,
|
|
115
|
+
created_by,
|
|
116
|
+
last_refreshed,
|
|
117
|
+
deleted_on,
|
|
118
|
+
deleted_by,
|
|
119
|
+
chart_type,
|
|
120
|
+
access,
|
|
121
|
+
input_datasets,
|
|
122
|
+
)
|
|
123
|
+
self._init_standard_aspects(
|
|
124
|
+
parent_container, subtype, owners, links, tags, terms, domain
|
|
125
|
+
)
|
|
99
126
|
|
|
100
|
-
|
|
101
|
-
|
|
127
|
+
@classmethod
|
|
128
|
+
def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
|
|
129
|
+
assert isinstance(urn, ChartUrn)
|
|
130
|
+
entity = cls(
|
|
131
|
+
platform=urn.dashboard_tool,
|
|
132
|
+
name=urn.chart_id,
|
|
133
|
+
)
|
|
134
|
+
return entity._init_from_graph(current_aspects)
|
|
135
|
+
|
|
136
|
+
def _init_chart_properties(
|
|
137
|
+
self,
|
|
138
|
+
description: Optional[str],
|
|
139
|
+
display_name: Optional[str],
|
|
140
|
+
external_url: Optional[str],
|
|
141
|
+
chart_url: Optional[str],
|
|
142
|
+
custom_properties: Optional[Dict[str, str]],
|
|
143
|
+
last_modified: Optional[datetime],
|
|
144
|
+
last_modified_by: Optional[ActorUrnOrStr],
|
|
145
|
+
created_at: Optional[datetime],
|
|
146
|
+
created_by: Optional[ActorUrnOrStr],
|
|
147
|
+
last_refreshed: Optional[datetime],
|
|
148
|
+
deleted_on: Optional[datetime],
|
|
149
|
+
deleted_by: Optional[ActorUrnOrStr],
|
|
150
|
+
chart_type: Optional[Union[str, models.ChartTypeClass]],
|
|
151
|
+
access: Optional[str],
|
|
152
|
+
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]],
|
|
153
|
+
) -> None:
|
|
154
|
+
"""Initialize chart-specific properties."""
|
|
102
155
|
if description is not None:
|
|
103
156
|
self.set_description(description)
|
|
157
|
+
if display_name is not None:
|
|
158
|
+
self.set_display_name(display_name)
|
|
104
159
|
if external_url is not None:
|
|
105
160
|
self.set_external_url(external_url)
|
|
106
161
|
if chart_url is not None:
|
|
@@ -109,6 +164,16 @@ class Chart(
|
|
|
109
164
|
self.set_custom_properties(custom_properties)
|
|
110
165
|
if last_modified is not None:
|
|
111
166
|
self.set_last_modified(last_modified)
|
|
167
|
+
if last_modified_by is not None:
|
|
168
|
+
self.set_last_modified_by(last_modified_by)
|
|
169
|
+
if created_at is not None:
|
|
170
|
+
self.set_created_at(created_at)
|
|
171
|
+
if created_by is not None:
|
|
172
|
+
self.set_created_by(created_by)
|
|
173
|
+
if deleted_on is not None:
|
|
174
|
+
self.set_deleted_on(deleted_on)
|
|
175
|
+
if deleted_by is not None:
|
|
176
|
+
self.set_deleted_by(deleted_by)
|
|
112
177
|
if last_refreshed is not None:
|
|
113
178
|
self.set_last_refreshed(last_refreshed)
|
|
114
179
|
if chart_type is not None:
|
|
@@ -118,6 +183,17 @@ class Chart(
|
|
|
118
183
|
if input_datasets is not None:
|
|
119
184
|
self.set_input_datasets(input_datasets)
|
|
120
185
|
|
|
186
|
+
def _init_standard_aspects(
|
|
187
|
+
self,
|
|
188
|
+
parent_container: ParentContainerInputType | Unset,
|
|
189
|
+
subtype: Optional[str],
|
|
190
|
+
owners: Optional[OwnersInputType],
|
|
191
|
+
links: Optional[LinksInputType],
|
|
192
|
+
tags: Optional[TagsInputType],
|
|
193
|
+
terms: Optional[TermsInputType],
|
|
194
|
+
domain: Optional[DomainInputType],
|
|
195
|
+
) -> None:
|
|
196
|
+
"""Initialize standard aspects."""
|
|
121
197
|
if parent_container is not unset:
|
|
122
198
|
self._set_container(parent_container)
|
|
123
199
|
if subtype is not None:
|
|
@@ -133,15 +209,6 @@ class Chart(
|
|
|
133
209
|
if domain is not None:
|
|
134
210
|
self.set_domain(domain)
|
|
135
211
|
|
|
136
|
-
@classmethod
|
|
137
|
-
def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
|
|
138
|
-
assert isinstance(urn, ChartUrn)
|
|
139
|
-
entity = cls(
|
|
140
|
-
platform=urn.dashboard_tool,
|
|
141
|
-
name=urn.chart_id,
|
|
142
|
-
)
|
|
143
|
-
return entity._init_from_graph(current_aspects)
|
|
144
|
-
|
|
145
212
|
@property
|
|
146
213
|
def urn(self) -> ChartUrn:
|
|
147
214
|
assert isinstance(self._urn, ChartUrn)
|
|
@@ -159,6 +226,14 @@ class Chart(
|
|
|
159
226
|
)
|
|
160
227
|
)
|
|
161
228
|
|
|
229
|
+
def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
|
|
230
|
+
"""Get the audit stamps from the chart properties."""
|
|
231
|
+
return self._ensure_chart_props().lastModified
|
|
232
|
+
|
|
233
|
+
def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
|
|
234
|
+
"""Set the audit stamps on the chart properties."""
|
|
235
|
+
self._ensure_chart_props().lastModified = audit_stamps
|
|
236
|
+
|
|
162
237
|
@property
|
|
163
238
|
def name(self) -> str:
|
|
164
239
|
"""Get the name of the chart."""
|
|
@@ -220,24 +295,6 @@ class Chart(
|
|
|
220
295
|
"""Set the custom properties of the chart."""
|
|
221
296
|
self._ensure_chart_props().customProperties = custom_properties
|
|
222
297
|
|
|
223
|
-
@property
|
|
224
|
-
def last_modified(self) -> Optional[datetime]:
|
|
225
|
-
"""Get the last modification timestamp of the chart."""
|
|
226
|
-
last_modified_time = self._ensure_chart_props().lastModified.lastModified.time
|
|
227
|
-
if not last_modified_time:
|
|
228
|
-
return None
|
|
229
|
-
return datetime.fromtimestamp(last_modified_time)
|
|
230
|
-
|
|
231
|
-
def set_last_modified(self, last_modified: datetime) -> None:
|
|
232
|
-
"""Set the last modification timestamp of the chart."""
|
|
233
|
-
chart_props = self._ensure_chart_props()
|
|
234
|
-
chart_props.lastModified = models.ChangeAuditStampsClass(
|
|
235
|
-
lastModified=models.AuditStampClass(
|
|
236
|
-
time=int(last_modified.timestamp()),
|
|
237
|
-
actor="urn:li:corpuser:datahub",
|
|
238
|
-
)
|
|
239
|
-
)
|
|
240
|
-
|
|
241
298
|
@property
|
|
242
299
|
def last_refreshed(self) -> Optional[datetime]:
|
|
243
300
|
"""Get the last refresh timestamp of the chart."""
|
datahub/sdk/dashboard.py
CHANGED
|
@@ -9,6 +9,8 @@ from typing_extensions import Self
|
|
|
9
9
|
import datahub.metadata.schema_classes as models
|
|
10
10
|
from datahub.metadata.urns import ChartUrn, DashboardUrn, DatasetUrn, Urn
|
|
11
11
|
from datahub.sdk._shared import (
|
|
12
|
+
ActorUrnOrStr,
|
|
13
|
+
ChangeAuditStampsMixin,
|
|
12
14
|
ChartUrnOrStr,
|
|
13
15
|
DashboardUrnOrStr,
|
|
14
16
|
DataPlatformInstanceUrnOrStr,
|
|
@@ -36,6 +38,7 @@ from datahub.utilities.sentinels import Unset, unset
|
|
|
36
38
|
|
|
37
39
|
|
|
38
40
|
class Dashboard(
|
|
41
|
+
ChangeAuditStampsMixin,
|
|
39
42
|
HasPlatformInstance,
|
|
40
43
|
HasSubtype,
|
|
41
44
|
HasOwnership,
|
|
@@ -72,6 +75,11 @@ class Dashboard(
|
|
|
72
75
|
dashboard_url: Optional[str] = None,
|
|
73
76
|
custom_properties: Optional[Dict[str, str]] = None,
|
|
74
77
|
last_modified: Optional[datetime] = None,
|
|
78
|
+
last_modified_by: Optional[ActorUrnOrStr] = None,
|
|
79
|
+
created_at: Optional[datetime] = None,
|
|
80
|
+
created_by: Optional[ActorUrnOrStr] = None,
|
|
81
|
+
deleted_on: Optional[datetime] = None,
|
|
82
|
+
deleted_by: Optional[ActorUrnOrStr] = None,
|
|
75
83
|
last_refreshed: Optional[datetime] = None,
|
|
76
84
|
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]] = None,
|
|
77
85
|
charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]] = None,
|
|
@@ -96,17 +104,48 @@ class Dashboard(
|
|
|
96
104
|
self._set_extra_aspects(extra_aspects)
|
|
97
105
|
|
|
98
106
|
self._set_platform_instance(platform, platform_instance)
|
|
107
|
+
self._ensure_dashboard_props(display_name=display_name)
|
|
108
|
+
|
|
109
|
+
self._init_dashboard_properties(
|
|
110
|
+
description,
|
|
111
|
+
display_name,
|
|
112
|
+
external_url,
|
|
113
|
+
dashboard_url,
|
|
114
|
+
custom_properties,
|
|
115
|
+
last_modified,
|
|
116
|
+
last_modified_by,
|
|
117
|
+
created_at,
|
|
118
|
+
created_by,
|
|
119
|
+
last_refreshed,
|
|
120
|
+
deleted_on,
|
|
121
|
+
deleted_by,
|
|
122
|
+
input_datasets,
|
|
123
|
+
charts,
|
|
124
|
+
dashboards,
|
|
125
|
+
)
|
|
126
|
+
self._init_standard_aspects(
|
|
127
|
+
parent_container, subtype, owners, links, tags, terms, domain
|
|
128
|
+
)
|
|
99
129
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
130
|
+
def _init_dashboard_properties(
|
|
131
|
+
self,
|
|
132
|
+
description: Optional[str],
|
|
133
|
+
display_name: Optional[str],
|
|
134
|
+
external_url: Optional[str],
|
|
135
|
+
dashboard_url: Optional[str],
|
|
136
|
+
custom_properties: Optional[Dict[str, str]],
|
|
137
|
+
last_modified: Optional[datetime],
|
|
138
|
+
last_modified_by: Optional[ActorUrnOrStr],
|
|
139
|
+
created_at: Optional[datetime],
|
|
140
|
+
created_by: Optional[ActorUrnOrStr],
|
|
141
|
+
last_refreshed: Optional[datetime],
|
|
142
|
+
deleted_on: Optional[datetime],
|
|
143
|
+
deleted_by: Optional[ActorUrnOrStr],
|
|
144
|
+
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]],
|
|
145
|
+
charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]],
|
|
146
|
+
dashboards: Optional[Sequence[Union[DashboardUrnOrStr, Dashboard]]],
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Initialize dashboard-specific properties."""
|
|
110
149
|
if description is not None:
|
|
111
150
|
self.set_description(description)
|
|
112
151
|
if display_name is not None:
|
|
@@ -119,6 +158,16 @@ class Dashboard(
|
|
|
119
158
|
self.set_custom_properties(custom_properties)
|
|
120
159
|
if last_modified is not None:
|
|
121
160
|
self.set_last_modified(last_modified)
|
|
161
|
+
if last_modified_by is not None:
|
|
162
|
+
self.set_last_modified_by(last_modified_by)
|
|
163
|
+
if created_at is not None:
|
|
164
|
+
self.set_created_at(created_at)
|
|
165
|
+
if created_by is not None:
|
|
166
|
+
self.set_created_by(created_by)
|
|
167
|
+
if deleted_on is not None:
|
|
168
|
+
self.set_deleted_on(deleted_on)
|
|
169
|
+
if deleted_by is not None:
|
|
170
|
+
self.set_deleted_by(deleted_by)
|
|
122
171
|
if last_refreshed is not None:
|
|
123
172
|
self.set_last_refreshed(last_refreshed)
|
|
124
173
|
if input_datasets is not None:
|
|
@@ -128,6 +177,17 @@ class Dashboard(
|
|
|
128
177
|
if dashboards is not None:
|
|
129
178
|
self.set_dashboards(dashboards)
|
|
130
179
|
|
|
180
|
+
def _init_standard_aspects(
|
|
181
|
+
self,
|
|
182
|
+
parent_container: ParentContainerInputType | Unset,
|
|
183
|
+
subtype: Optional[str],
|
|
184
|
+
owners: Optional[OwnersInputType],
|
|
185
|
+
links: Optional[LinksInputType],
|
|
186
|
+
tags: Optional[TagsInputType],
|
|
187
|
+
terms: Optional[TermsInputType],
|
|
188
|
+
domain: Optional[DomainInputType],
|
|
189
|
+
) -> None:
|
|
190
|
+
"""Initialize standard aspects."""
|
|
131
191
|
if parent_container is not unset:
|
|
132
192
|
self._set_container(parent_container)
|
|
133
193
|
if subtype is not None:
|
|
@@ -165,16 +225,20 @@ class Dashboard(
|
|
|
165
225
|
models.DashboardInfoClass(
|
|
166
226
|
title=display_name or self.urn.dashboard_id,
|
|
167
227
|
description="",
|
|
168
|
-
lastModified=models.ChangeAuditStampsClass(
|
|
169
|
-
lastModified=models.AuditStampClass(
|
|
170
|
-
time=0, actor="urn:li:corpuser:unknown"
|
|
171
|
-
)
|
|
172
|
-
),
|
|
228
|
+
lastModified=models.ChangeAuditStampsClass(),
|
|
173
229
|
customProperties={},
|
|
174
230
|
dashboards=[],
|
|
175
231
|
)
|
|
176
232
|
)
|
|
177
233
|
|
|
234
|
+
def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
|
|
235
|
+
"""Get the audit stamps from the dashboard properties."""
|
|
236
|
+
return self._ensure_dashboard_props().lastModified
|
|
237
|
+
|
|
238
|
+
def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
|
|
239
|
+
"""Set the audit stamps on the dashboard properties."""
|
|
240
|
+
self._ensure_dashboard_props().lastModified = audit_stamps
|
|
241
|
+
|
|
178
242
|
@property
|
|
179
243
|
def name(self) -> str:
|
|
180
244
|
"""Get the name of the dashboard."""
|
|
@@ -238,23 +302,6 @@ class Dashboard(
|
|
|
238
302
|
"""Set the custom properties of the dashboard."""
|
|
239
303
|
self._ensure_dashboard_props().customProperties = custom_properties
|
|
240
304
|
|
|
241
|
-
@property
|
|
242
|
-
def last_modified(self) -> Optional[datetime]:
|
|
243
|
-
"""Get the last modification timestamp of the dashboard."""
|
|
244
|
-
props = self._ensure_dashboard_props()
|
|
245
|
-
if props.lastModified.lastModified.time == 0:
|
|
246
|
-
return None
|
|
247
|
-
return datetime.fromtimestamp(props.lastModified.lastModified.time)
|
|
248
|
-
|
|
249
|
-
def set_last_modified(self, last_modified: datetime) -> None:
|
|
250
|
-
"""Set the last modification timestamp of the dashboard."""
|
|
251
|
-
self._ensure_dashboard_props().lastModified = models.ChangeAuditStampsClass(
|
|
252
|
-
lastModified=models.AuditStampClass(
|
|
253
|
-
time=int(last_modified.timestamp()),
|
|
254
|
-
actor="urn:li:corpuser:datahub",
|
|
255
|
-
),
|
|
256
|
-
)
|
|
257
|
-
|
|
258
305
|
@property
|
|
259
306
|
def last_refreshed(self) -> Optional[datetime]:
|
|
260
307
|
"""Get the last refresh timestamp of the dashboard."""
|
|
File without changes
|
{acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|