acryl-datahub 0.15.0.5rc10__py3-none-any.whl → 0.15.0.6rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (35) hide show
  1. {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/METADATA +2482 -2482
  2. {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/RECORD +35 -24
  3. datahub/_version.py +1 -1
  4. datahub/errors.py +35 -0
  5. datahub/ingestion/source/common/subtypes.py +1 -0
  6. datahub/ingestion/source/mongodb.py +17 -16
  7. datahub/ingestion/source/powerbi/config.py +1 -0
  8. datahub/ingestion/source/powerbi/powerbi.py +28 -3
  9. datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py +6 -2
  10. datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +11 -36
  11. datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +17 -4
  12. datahub/ingestion/source/s3/source.py +14 -5
  13. datahub/ingestion/source/snowflake/constants.py +1 -0
  14. datahub/ingestion/source/snowflake/snowflake_config.py +10 -0
  15. datahub/ingestion/source/snowflake/snowflake_queries.py +45 -10
  16. datahub/ingestion/source/snowflake/snowflake_query.py +20 -1
  17. datahub/ingestion/source/snowflake/snowflake_report.py +6 -0
  18. datahub/ingestion/source/snowflake/snowflake_schema.py +108 -4
  19. datahub/ingestion/source/snowflake/snowflake_schema_gen.py +298 -69
  20. datahub/ingestion/source/snowflake/snowflake_utils.py +17 -8
  21. datahub/ingestion/source/snowflake/snowflake_v2.py +15 -3
  22. datahub/sdk/__init__.py +33 -0
  23. datahub/sdk/_all_entities.py +15 -0
  24. datahub/sdk/_attribution.py +48 -0
  25. datahub/sdk/_entity.py +89 -0
  26. datahub/sdk/_shared.py +338 -0
  27. datahub/sdk/container.py +193 -0
  28. datahub/sdk/dataset.py +584 -0
  29. datahub/sdk/entity_client.py +115 -0
  30. datahub/sdk/main_client.py +56 -0
  31. datahub/sdk/resolver_client.py +101 -0
  32. {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/LICENSE +0 -0
  33. {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/WHEEL +0 -0
  34. {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/entry_points.txt +0 -0
  35. {acryl_datahub-0.15.0.5rc10.dist-info → acryl_datahub-0.15.0.6rc2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,193 @@
1
+ from __future__ import annotations
2
+
3
+ from datetime import datetime
4
+ from typing import Dict, Optional, Type
5
+
6
+ from typing_extensions import Self
7
+
8
+ import datahub.metadata.schema_classes as models
9
+ from datahub.emitter.mce_builder import ALL_ENV_TYPES
10
+ from datahub.emitter.mcp_builder import (
11
+ _INCLUDE_ENV_IN_CONTAINER_PROPERTIES,
12
+ ContainerKey,
13
+ )
14
+ from datahub.errors import SdkUsageError
15
+ from datahub.metadata.urns import (
16
+ ContainerUrn,
17
+ Urn,
18
+ )
19
+ from datahub.sdk._entity import Entity
20
+ from datahub.sdk._shared import (
21
+ DomainInputType,
22
+ HasContainer,
23
+ HasDomain,
24
+ HasOwnership,
25
+ HasPlatformInstance,
26
+ HasSubtype,
27
+ HasTags,
28
+ HasTerms,
29
+ OwnersInputType,
30
+ TagsInputType,
31
+ TermsInputType,
32
+ make_time_stamp,
33
+ parse_time_stamp,
34
+ )
35
+
36
+
37
+ class Container(
38
+ HasPlatformInstance,
39
+ HasSubtype,
40
+ HasContainer,
41
+ HasOwnership,
42
+ HasTags,
43
+ HasTerms,
44
+ HasDomain,
45
+ Entity,
46
+ ):
47
+ __slots__ = ()
48
+
49
+ @classmethod
50
+ def get_urn_type(cls) -> Type[ContainerUrn]:
51
+ return ContainerUrn
52
+
53
+ def __init__(
54
+ self,
55
+ /,
56
+ # Identity.
57
+ container_key: ContainerKey | ContainerUrn,
58
+ *,
59
+ # Container attributes.
60
+ display_name: str,
61
+ qualified_name: Optional[str] = None,
62
+ description: Optional[str] = None,
63
+ external_url: Optional[str] = None,
64
+ # TODO: call this custom properties?
65
+ extra_properties: Optional[Dict[str, str]] = None,
66
+ created: Optional[datetime] = None,
67
+ last_modified: Optional[datetime] = None,
68
+ # Standard aspects.
69
+ subtype: Optional[str] = None,
70
+ owners: Optional[OwnersInputType] = None,
71
+ tags: Optional[TagsInputType] = None,
72
+ terms: Optional[TermsInputType] = None,
73
+ domain: Optional[DomainInputType] = None,
74
+ ):
75
+ if isinstance(container_key, ContainerUrn):
76
+ urn = container_key
77
+ else:
78
+ urn = ContainerUrn.from_string(container_key.as_urn())
79
+ super().__init__(urn)
80
+
81
+ # This needs to come first to ensure that the display name is registered.
82
+ self._ensure_container_props(name=display_name)
83
+
84
+ # TODO: Normal usages should require container key. Only the graph init method can accept an urn.
85
+ if isinstance(container_key, ContainerKey):
86
+ self._set_platform_instance(container_key.platform, container_key.instance)
87
+
88
+ self._set_container(container_key.parent_key())
89
+
90
+ self.set_custom_properties(
91
+ {
92
+ **container_key.property_dict(),
93
+ **(extra_properties or {}),
94
+ }
95
+ )
96
+
97
+ # Extra validation on the env field.
98
+ # In certain cases (mainly for backwards compatibility), the env field will actually
99
+ # have a platform instance name.
100
+ env = container_key.env if container_key.env in ALL_ENV_TYPES else None
101
+ if _INCLUDE_ENV_IN_CONTAINER_PROPERTIES and env is not None:
102
+ self._ensure_container_props().env = env
103
+
104
+ if description is not None:
105
+ self.set_description(description)
106
+ if external_url is not None:
107
+ self.set_external_url(external_url)
108
+ if qualified_name is not None:
109
+ self.set_qualified_name(qualified_name)
110
+ if created is not None:
111
+ self.set_created(created)
112
+ if last_modified is not None:
113
+ self.set_last_modified(last_modified)
114
+
115
+ if subtype is not None:
116
+ self.set_subtype(subtype)
117
+ if owners is not None:
118
+ self.set_owners(owners)
119
+ if tags is not None:
120
+ self.set_tags(tags)
121
+ if terms is not None:
122
+ self.set_terms(terms)
123
+ if domain is not None:
124
+ self.set_domain(domain)
125
+
126
+ @classmethod
127
+ def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
128
+ assert isinstance(urn, ContainerUrn)
129
+ entity = cls(urn, display_name="__dummy_value__")
130
+ return entity._init_from_graph(current_aspects)
131
+
132
+ def _ensure_container_props(
133
+ self, *, name: Optional[str] = None
134
+ ) -> models.ContainerPropertiesClass:
135
+ # TODO: Not super happy with this method's implementation, but it's
136
+ # internal-only and enforces the constraints that we need.
137
+ if name is not None:
138
+ return self._setdefault_aspect(models.ContainerPropertiesClass(name=name))
139
+
140
+ props = self._get_aspect(models.ContainerPropertiesClass)
141
+ if props is None:
142
+ raise SdkUsageError("Containers must have a name.")
143
+ return props
144
+
145
+ @property
146
+ def display_name(self) -> str:
147
+ return self._ensure_container_props().name
148
+
149
+ def set_display_name(self, value: str) -> None:
150
+ self._ensure_container_props().name = value
151
+
152
+ @property
153
+ def description(self) -> Optional[str]:
154
+ return self._ensure_container_props().description
155
+
156
+ def set_description(self, description: str) -> None:
157
+ self._ensure_container_props().description = description
158
+
159
+ @property
160
+ def custom_properties(self) -> Optional[Dict[str, str]]:
161
+ return self._ensure_container_props().customProperties
162
+
163
+ def set_custom_properties(self, custom_properties: Dict[str, str]) -> None:
164
+ # TODO: How do we ensure that the container key props are always retained?
165
+ self._ensure_container_props().customProperties = custom_properties
166
+
167
+ @property
168
+ def external_url(self) -> Optional[str]:
169
+ return self._ensure_container_props().externalUrl
170
+
171
+ def set_external_url(self, external_url: str) -> None:
172
+ self._ensure_container_props().externalUrl = external_url
173
+
174
+ @property
175
+ def qualified_name(self) -> Optional[str]:
176
+ return self._ensure_container_props().qualifiedName
177
+
178
+ def set_qualified_name(self, qualified_name: str) -> None:
179
+ self._ensure_container_props().qualifiedName = qualified_name
180
+
181
+ @property
182
+ def created(self) -> Optional[datetime]:
183
+ return parse_time_stamp(self._ensure_container_props().created)
184
+
185
+ def set_created(self, created: datetime) -> None:
186
+ self._ensure_container_props().created = make_time_stamp(created)
187
+
188
+ @property
189
+ def last_modified(self) -> Optional[datetime]:
190
+ return parse_time_stamp(self._ensure_container_props().lastModified)
191
+
192
+ def set_last_modified(self, last_modified: datetime) -> None:
193
+ self._ensure_container_props().lastModified = make_time_stamp(last_modified)