acryl-datahub 0.15.0.5rc10__py3-none-any.whl → 0.15.0.6rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/METADATA +2482 -2482
- {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/RECORD +35 -24
- datahub/_version.py +1 -1
- datahub/errors.py +35 -0
- datahub/ingestion/source/common/subtypes.py +1 -0
- datahub/ingestion/source/mongodb.py +17 -16
- datahub/ingestion/source/powerbi/config.py +1 -0
- datahub/ingestion/source/powerbi/powerbi.py +28 -3
- datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py +6 -2
- datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +11 -36
- datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +17 -4
- datahub/ingestion/source/s3/source.py +14 -5
- datahub/ingestion/source/snowflake/constants.py +1 -0
- datahub/ingestion/source/snowflake/snowflake_config.py +10 -0
- datahub/ingestion/source/snowflake/snowflake_queries.py +45 -10
- datahub/ingestion/source/snowflake/snowflake_query.py +20 -1
- datahub/ingestion/source/snowflake/snowflake_report.py +6 -0
- datahub/ingestion/source/snowflake/snowflake_schema.py +108 -4
- datahub/ingestion/source/snowflake/snowflake_schema_gen.py +298 -69
- datahub/ingestion/source/snowflake/snowflake_utils.py +17 -8
- datahub/ingestion/source/snowflake/snowflake_v2.py +15 -3
- datahub/sdk/__init__.py +33 -0
- datahub/sdk/_all_entities.py +15 -0
- datahub/sdk/_attribution.py +48 -0
- datahub/sdk/_entity.py +89 -0
- datahub/sdk/_shared.py +338 -0
- datahub/sdk/container.py +193 -0
- datahub/sdk/dataset.py +584 -0
- datahub/sdk/entity_client.py +115 -0
- datahub/sdk/main_client.py +56 -0
- datahub/sdk/resolver_client.py +101 -0
- {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/LICENSE +0 -0
- {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/WHEEL +0 -0
- {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/top_level.txt +0 -0
datahub/sdk/container.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Dict, Optional, Type
|
|
5
|
+
|
|
6
|
+
from typing_extensions import Self
|
|
7
|
+
|
|
8
|
+
import datahub.metadata.schema_classes as models
|
|
9
|
+
from datahub.emitter.mce_builder import ALL_ENV_TYPES
|
|
10
|
+
from datahub.emitter.mcp_builder import (
|
|
11
|
+
_INCLUDE_ENV_IN_CONTAINER_PROPERTIES,
|
|
12
|
+
ContainerKey,
|
|
13
|
+
)
|
|
14
|
+
from datahub.errors import SdkUsageError
|
|
15
|
+
from datahub.metadata.urns import (
|
|
16
|
+
ContainerUrn,
|
|
17
|
+
Urn,
|
|
18
|
+
)
|
|
19
|
+
from datahub.sdk._entity import Entity
|
|
20
|
+
from datahub.sdk._shared import (
|
|
21
|
+
DomainInputType,
|
|
22
|
+
HasContainer,
|
|
23
|
+
HasDomain,
|
|
24
|
+
HasOwnership,
|
|
25
|
+
HasPlatformInstance,
|
|
26
|
+
HasSubtype,
|
|
27
|
+
HasTags,
|
|
28
|
+
HasTerms,
|
|
29
|
+
OwnersInputType,
|
|
30
|
+
TagsInputType,
|
|
31
|
+
TermsInputType,
|
|
32
|
+
make_time_stamp,
|
|
33
|
+
parse_time_stamp,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Container(
|
|
38
|
+
HasPlatformInstance,
|
|
39
|
+
HasSubtype,
|
|
40
|
+
HasContainer,
|
|
41
|
+
HasOwnership,
|
|
42
|
+
HasTags,
|
|
43
|
+
HasTerms,
|
|
44
|
+
HasDomain,
|
|
45
|
+
Entity,
|
|
46
|
+
):
|
|
47
|
+
__slots__ = ()
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def get_urn_type(cls) -> Type[ContainerUrn]:
|
|
51
|
+
return ContainerUrn
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
/,
|
|
56
|
+
# Identity.
|
|
57
|
+
container_key: ContainerKey | ContainerUrn,
|
|
58
|
+
*,
|
|
59
|
+
# Container attributes.
|
|
60
|
+
display_name: str,
|
|
61
|
+
qualified_name: Optional[str] = None,
|
|
62
|
+
description: Optional[str] = None,
|
|
63
|
+
external_url: Optional[str] = None,
|
|
64
|
+
# TODO: call this custom properties?
|
|
65
|
+
extra_properties: Optional[Dict[str, str]] = None,
|
|
66
|
+
created: Optional[datetime] = None,
|
|
67
|
+
last_modified: Optional[datetime] = None,
|
|
68
|
+
# Standard aspects.
|
|
69
|
+
subtype: Optional[str] = None,
|
|
70
|
+
owners: Optional[OwnersInputType] = None,
|
|
71
|
+
tags: Optional[TagsInputType] = None,
|
|
72
|
+
terms: Optional[TermsInputType] = None,
|
|
73
|
+
domain: Optional[DomainInputType] = None,
|
|
74
|
+
):
|
|
75
|
+
if isinstance(container_key, ContainerUrn):
|
|
76
|
+
urn = container_key
|
|
77
|
+
else:
|
|
78
|
+
urn = ContainerUrn.from_string(container_key.as_urn())
|
|
79
|
+
super().__init__(urn)
|
|
80
|
+
|
|
81
|
+
# This needs to come first to ensure that the display name is registered.
|
|
82
|
+
self._ensure_container_props(name=display_name)
|
|
83
|
+
|
|
84
|
+
# TODO: Normal usages should require container key. Only the graph init method can accept an urn.
|
|
85
|
+
if isinstance(container_key, ContainerKey):
|
|
86
|
+
self._set_platform_instance(container_key.platform, container_key.instance)
|
|
87
|
+
|
|
88
|
+
self._set_container(container_key.parent_key())
|
|
89
|
+
|
|
90
|
+
self.set_custom_properties(
|
|
91
|
+
{
|
|
92
|
+
**container_key.property_dict(),
|
|
93
|
+
**(extra_properties or {}),
|
|
94
|
+
}
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Extra validation on the env field.
|
|
98
|
+
# In certain cases (mainly for backwards compatibility), the env field will actually
|
|
99
|
+
# have a platform instance name.
|
|
100
|
+
env = container_key.env if container_key.env in ALL_ENV_TYPES else None
|
|
101
|
+
if _INCLUDE_ENV_IN_CONTAINER_PROPERTIES and env is not None:
|
|
102
|
+
self._ensure_container_props().env = env
|
|
103
|
+
|
|
104
|
+
if description is not None:
|
|
105
|
+
self.set_description(description)
|
|
106
|
+
if external_url is not None:
|
|
107
|
+
self.set_external_url(external_url)
|
|
108
|
+
if qualified_name is not None:
|
|
109
|
+
self.set_qualified_name(qualified_name)
|
|
110
|
+
if created is not None:
|
|
111
|
+
self.set_created(created)
|
|
112
|
+
if last_modified is not None:
|
|
113
|
+
self.set_last_modified(last_modified)
|
|
114
|
+
|
|
115
|
+
if subtype is not None:
|
|
116
|
+
self.set_subtype(subtype)
|
|
117
|
+
if owners is not None:
|
|
118
|
+
self.set_owners(owners)
|
|
119
|
+
if tags is not None:
|
|
120
|
+
self.set_tags(tags)
|
|
121
|
+
if terms is not None:
|
|
122
|
+
self.set_terms(terms)
|
|
123
|
+
if domain is not None:
|
|
124
|
+
self.set_domain(domain)
|
|
125
|
+
|
|
126
|
+
@classmethod
|
|
127
|
+
def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
|
|
128
|
+
assert isinstance(urn, ContainerUrn)
|
|
129
|
+
entity = cls(urn, display_name="__dummy_value__")
|
|
130
|
+
return entity._init_from_graph(current_aspects)
|
|
131
|
+
|
|
132
|
+
def _ensure_container_props(
|
|
133
|
+
self, *, name: Optional[str] = None
|
|
134
|
+
) -> models.ContainerPropertiesClass:
|
|
135
|
+
# TODO: Not super happy with this method's implementation, but it's
|
|
136
|
+
# internal-only and enforces the constraints that we need.
|
|
137
|
+
if name is not None:
|
|
138
|
+
return self._setdefault_aspect(models.ContainerPropertiesClass(name=name))
|
|
139
|
+
|
|
140
|
+
props = self._get_aspect(models.ContainerPropertiesClass)
|
|
141
|
+
if props is None:
|
|
142
|
+
raise SdkUsageError("Containers must have a name.")
|
|
143
|
+
return props
|
|
144
|
+
|
|
145
|
+
@property
|
|
146
|
+
def display_name(self) -> str:
|
|
147
|
+
return self._ensure_container_props().name
|
|
148
|
+
|
|
149
|
+
def set_display_name(self, value: str) -> None:
|
|
150
|
+
self._ensure_container_props().name = value
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def description(self) -> Optional[str]:
|
|
154
|
+
return self._ensure_container_props().description
|
|
155
|
+
|
|
156
|
+
def set_description(self, description: str) -> None:
|
|
157
|
+
self._ensure_container_props().description = description
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def custom_properties(self) -> Optional[Dict[str, str]]:
|
|
161
|
+
return self._ensure_container_props().customProperties
|
|
162
|
+
|
|
163
|
+
def set_custom_properties(self, custom_properties: Dict[str, str]) -> None:
|
|
164
|
+
# TODO: How do we ensure that the container key props are always retained?
|
|
165
|
+
self._ensure_container_props().customProperties = custom_properties
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def external_url(self) -> Optional[str]:
|
|
169
|
+
return self._ensure_container_props().externalUrl
|
|
170
|
+
|
|
171
|
+
def set_external_url(self, external_url: str) -> None:
|
|
172
|
+
self._ensure_container_props().externalUrl = external_url
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def qualified_name(self) -> Optional[str]:
|
|
176
|
+
return self._ensure_container_props().qualifiedName
|
|
177
|
+
|
|
178
|
+
def set_qualified_name(self, qualified_name: str) -> None:
|
|
179
|
+
self._ensure_container_props().qualifiedName = qualified_name
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def created(self) -> Optional[datetime]:
|
|
183
|
+
return parse_time_stamp(self._ensure_container_props().created)
|
|
184
|
+
|
|
185
|
+
def set_created(self, created: datetime) -> None:
|
|
186
|
+
self._ensure_container_props().created = make_time_stamp(created)
|
|
187
|
+
|
|
188
|
+
@property
|
|
189
|
+
def last_modified(self) -> Optional[datetime]:
|
|
190
|
+
return parse_time_stamp(self._ensure_container_props().lastModified)
|
|
191
|
+
|
|
192
|
+
def set_last_modified(self, last_modified: datetime) -> None:
|
|
193
|
+
self._ensure_container_props().lastModified = make_time_stamp(last_modified)
|