acryl-datahub 0.15.0.1rc13__py3-none-any.whl → 0.15.0.1rc14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (30) hide show
  1. {acryl_datahub-0.15.0.1rc13.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/METADATA +2520 -2520
  2. {acryl_datahub-0.15.0.1rc13.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/RECORD +27 -24
  3. datahub/__init__.py +1 -1
  4. datahub/emitter/mce_builder.py +3 -3
  5. datahub/emitter/mcp_patch_builder.py +36 -12
  6. datahub/ingestion/source/bigquery_v2/bigquery.py +10 -18
  7. datahub/ingestion/source/bigquery_v2/bigquery_config.py +3 -9
  8. datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +11 -17
  9. datahub/ingestion/source/bigquery_v2/lineage.py +9 -22
  10. datahub/ingestion/source/tableau/tableau.py +3 -0
  11. datahub/ingestion/source/tableau/tableau_common.py +18 -0
  12. datahub/specific/aspect_helpers/__init__.py +0 -0
  13. datahub/specific/aspect_helpers/custom_properties.py +79 -0
  14. datahub/specific/aspect_helpers/ownership.py +67 -0
  15. datahub/specific/aspect_helpers/structured_properties.py +72 -0
  16. datahub/specific/aspect_helpers/tags.py +42 -0
  17. datahub/specific/aspect_helpers/terms.py +43 -0
  18. datahub/specific/chart.py +28 -184
  19. datahub/specific/dashboard.py +31 -196
  20. datahub/specific/datajob.py +34 -189
  21. datahub/specific/dataproduct.py +24 -86
  22. datahub/specific/dataset.py +48 -133
  23. datahub/specific/form.py +12 -32
  24. datahub/specific/structured_property.py +9 -9
  25. datahub/specific/custom_properties.py +0 -37
  26. datahub/specific/ownership.py +0 -48
  27. datahub/specific/structured_properties.py +0 -53
  28. {acryl_datahub-0.15.0.1rc13.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/WHEEL +0 -0
  29. {acryl_datahub-0.15.0.1rc13.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/entry_points.txt +0 -0
  30. {acryl_datahub-0.15.0.1rc13.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,72 @@
1
+ from typing import List, Union
2
+
3
+ from typing_extensions import Self
4
+
5
+ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
6
+ from datahub.metadata.schema_classes import (
7
+ StructuredPropertiesClass,
8
+ StructuredPropertyValueAssignmentClass,
9
+ )
10
+ from datahub.utilities.urns.structured_properties_urn import (
11
+ make_structured_property_urn,
12
+ )
13
+
14
+
15
+ class HasStructuredPropertiesPatch(MetadataPatchProposal):
16
+ def set_structured_property(
17
+ self, key: str, value: Union[str, float, List[Union[str, float]]]
18
+ ) -> Self:
19
+ """Add or update a structured property.
20
+
21
+ Args:
22
+ key: the name of the property (either bare or urn form)
23
+ value: the value of the property (for multi-valued properties, this can be a list)
24
+
25
+ Returns:
26
+ The patch builder instance.
27
+ """
28
+ self.remove_structured_property(key)
29
+ self.add_structured_property(key, value)
30
+ return self
31
+
32
+ def remove_structured_property(self, key: str) -> Self:
33
+ """Remove a structured property.
34
+
35
+ Args:
36
+ key: the name of the property (either bare or urn form)
37
+
38
+ Returns:
39
+ The patch builder instance.
40
+ """
41
+
42
+ self._add_patch(
43
+ StructuredPropertiesClass.ASPECT_NAME,
44
+ "remove",
45
+ path=("properties", make_structured_property_urn(key)),
46
+ value={},
47
+ )
48
+ return self
49
+
50
+ def add_structured_property(
51
+ self, key: str, value: Union[str, float, List[Union[str, float]]]
52
+ ) -> Self:
53
+ """Add a structured property.
54
+
55
+ Args:
56
+ key: the name of the property (either bare or urn form)
57
+ value: the value of the property (for multi-valued properties, this value will be appended to the list)
58
+
59
+ Returns:
60
+ The patch builder instance.
61
+ """
62
+
63
+ self._add_patch(
64
+ StructuredPropertiesClass.ASPECT_NAME,
65
+ "add",
66
+ path=("properties", make_structured_property_urn(key)),
67
+ value=StructuredPropertyValueAssignmentClass(
68
+ propertyUrn=make_structured_property_urn(key),
69
+ values=value if isinstance(value, list) else [value],
70
+ ),
71
+ )
72
+ return self
@@ -0,0 +1,42 @@
1
+ from typing import Union
2
+
3
+ from typing_extensions import Self
4
+
5
+ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
6
+ from datahub.metadata.schema_classes import (
7
+ GlobalTagsClass as GlobalTags,
8
+ TagAssociationClass as Tag,
9
+ )
10
+ from datahub.metadata.urns import TagUrn, Urn
11
+
12
+
13
+ class HasTagsPatch(MetadataPatchProposal):
14
+ def add_tag(self, tag: Tag) -> Self:
15
+ """Adds a tag to the entity.
16
+
17
+ Args:
18
+ tag: The Tag object representing the tag to be added.
19
+
20
+ Returns:
21
+ The patch builder instance.
22
+ """
23
+
24
+ # TODO: Make this support raw strings, in addition to Tag objects.
25
+ self._add_patch(
26
+ GlobalTags.ASPECT_NAME, "add", path=("tags", tag.tag), value=tag
27
+ )
28
+ return self
29
+
30
+ def remove_tag(self, tag: Union[str, Urn]) -> Self:
31
+ """Removes a tag from the entity.
32
+
33
+ Args:
34
+ tag: The tag to remove, specified as a string or Urn object.
35
+
36
+ Returns:
37
+ The patch builder instance.
38
+ """
39
+ if isinstance(tag, str) and not tag.startswith("urn:li:tag:"):
40
+ tag = TagUrn.create_from_id(tag)
41
+ self._add_patch(GlobalTags.ASPECT_NAME, "remove", path=("tags", tag), value={})
42
+ return self
@@ -0,0 +1,43 @@
1
+ from typing import Union
2
+
3
+ from typing_extensions import Self
4
+
5
+ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
6
+ from datahub.metadata.schema_classes import (
7
+ GlossaryTermAssociationClass as Term,
8
+ GlossaryTermsClass,
9
+ )
10
+ from datahub.metadata.urns import GlossaryTermUrn, Urn
11
+
12
+
13
+ class HasTermsPatch(MetadataPatchProposal):
14
+ def add_term(self, term: Term) -> Self:
15
+ """Adds a glossary term to the entity.
16
+
17
+ Args:
18
+ term: The Term object representing the glossary term to be added.
19
+
20
+ Returns:
21
+ The patch builder instance.
22
+ """
23
+ # TODO: Make this support raw strings, in addition to Term objects.
24
+ self._add_patch(
25
+ GlossaryTermsClass.ASPECT_NAME, "add", path=("terms", term.urn), value=term
26
+ )
27
+ return self
28
+
29
+ def remove_term(self, term: Union[str, Urn]) -> Self:
30
+ """Removes a glossary term from the entity.
31
+
32
+ Args:
33
+ term: The term to remove, specified as a string or Urn object.
34
+
35
+ Returns:
36
+ The patch builder instance.
37
+ """
38
+ if isinstance(term, str) and not term.startswith("urn:li:glossaryTerm:"):
39
+ term = GlossaryTermUrn(term)
40
+ self._add_patch(
41
+ GlossaryTermsClass.ASPECT_NAME, "remove", path=("terms", term), value={}
42
+ )
43
+ return self
datahub/specific/chart.py CHANGED
@@ -1,28 +1,29 @@
1
- from typing import Dict, List, Optional, Union
1
+ from typing import List, Optional, Tuple, Union
2
2
 
3
- from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
3
+ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal, PatchPath
4
4
  from datahub.metadata.schema_classes import (
5
5
  AccessLevelClass,
6
6
  ChangeAuditStampsClass,
7
7
  ChartInfoClass as ChartInfo,
8
8
  ChartTypeClass,
9
9
  EdgeClass as Edge,
10
- GlobalTagsClass as GlobalTags,
11
- GlossaryTermAssociationClass as Term,
12
- GlossaryTermsClass as GlossaryTerms,
13
10
  KafkaAuditHeaderClass,
14
- OwnerClass as Owner,
15
- OwnershipTypeClass,
16
11
  SystemMetadataClass,
17
- TagAssociationClass as Tag,
18
12
  )
19
- from datahub.specific.custom_properties import CustomPropertiesPatchHelper
20
- from datahub.specific.ownership import OwnershipPatchHelper
21
- from datahub.utilities.urns.tag_urn import TagUrn
13
+ from datahub.specific.aspect_helpers.custom_properties import HasCustomPropertiesPatch
14
+ from datahub.specific.aspect_helpers.ownership import HasOwnershipPatch
15
+ from datahub.specific.aspect_helpers.tags import HasTagsPatch
16
+ from datahub.specific.aspect_helpers.terms import HasTermsPatch
22
17
  from datahub.utilities.urns.urn import Urn
23
18
 
24
19
 
25
- class ChartPatchBuilder(MetadataPatchProposal):
20
+ class ChartPatchBuilder(
21
+ HasOwnershipPatch,
22
+ HasCustomPropertiesPatch,
23
+ HasTagsPatch,
24
+ HasTermsPatch,
25
+ MetadataPatchProposal,
26
+ ):
26
27
  def __init__(
27
28
  self,
28
29
  urn: str,
@@ -40,55 +41,10 @@ class ChartPatchBuilder(MetadataPatchProposal):
40
41
  super().__init__(
41
42
  urn, system_metadata=system_metadata, audit_header=audit_header
42
43
  )
43
- self.custom_properties_patch_helper = CustomPropertiesPatchHelper(
44
- self, ChartInfo.ASPECT_NAME
45
- )
46
- self.ownership_patch_helper = OwnershipPatchHelper(self)
47
-
48
- def add_owner(self, owner: Owner) -> "ChartPatchBuilder":
49
- """
50
- Adds an owner to the ChartPatchBuilder.
51
-
52
- Args:
53
- owner: The Owner object to add.
54
-
55
- Returns:
56
- The ChartPatchBuilder instance.
57
- """
58
- self.ownership_patch_helper.add_owner(owner)
59
- return self
60
44
 
61
- def remove_owner(
62
- self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
63
- ) -> "ChartPatchBuilder":
64
- """
65
- Removes an owner from the ChartPatchBuilder.
66
-
67
- Args:
68
- owner: The owner to remove.
69
- owner_type: The ownership type of the owner (optional).
70
-
71
- Returns:
72
- The ChartPatchBuilder instance.
73
-
74
- Notes:
75
- `owner_type` is optional.
76
- """
77
- self.ownership_patch_helper.remove_owner(owner, owner_type)
78
- return self
79
-
80
- def set_owners(self, owners: List[Owner]) -> "ChartPatchBuilder":
81
- """
82
- Sets the owners of the ChartPatchBuilder.
83
-
84
- Args:
85
- owners: A list of Owner objects.
86
-
87
- Returns:
88
- The ChartPatchBuilder instance.
89
- """
90
- self.ownership_patch_helper.set_owners(owners)
91
- return self
45
+ @classmethod
46
+ def _custom_properties_location(cls) -> Tuple[str, PatchPath]:
47
+ return ChartInfo.ASPECT_NAME, ("customProperties",)
92
48
 
93
49
  def add_input_edge(self, input: Union[Edge, Urn, str]) -> "ChartPatchBuilder":
94
50
  """
@@ -120,7 +76,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
120
76
  self._add_patch(
121
77
  ChartInfo.ASPECT_NAME,
122
78
  "add",
123
- path=f"/inputEdges/{self.quote(input_urn)}",
79
+ path=("inputEdges", input_urn),
124
80
  value=input_urn,
125
81
  )
126
82
  return self
@@ -138,7 +94,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
138
94
  self._add_patch(
139
95
  ChartInfo.ASPECT_NAME,
140
96
  "remove",
141
- path=f"/inputEdges/{self.quote(str(input))}",
97
+ path=("inputEdges", str(input)),
142
98
  value={},
143
99
  )
144
100
  return self
@@ -159,129 +115,17 @@ class ChartPatchBuilder(MetadataPatchProposal):
159
115
  self._add_patch(
160
116
  ChartInfo.ASPECT_NAME,
161
117
  "add",
162
- path="/inputEdges",
118
+ path=("inputEdges",),
163
119
  value=inputs,
164
120
  )
165
121
  return self
166
122
 
167
- def add_tag(self, tag: Tag) -> "ChartPatchBuilder":
168
- """
169
- Adds a tag to the ChartPatchBuilder.
170
-
171
- Args:
172
- tag: The Tag object representing the tag to be added.
173
-
174
- Returns:
175
- The ChartPatchBuilder instance.
176
- """
177
- self._add_patch(
178
- GlobalTags.ASPECT_NAME, "add", path=f"/tags/{tag.tag}", value=tag
179
- )
180
- return self
181
-
182
- def remove_tag(self, tag: Union[str, Urn]) -> "ChartPatchBuilder":
183
- """
184
- Removes a tag from the ChartPatchBuilder.
185
-
186
- Args:
187
- tag: The tag to remove, specified as a string or Urn object.
188
-
189
- Returns:
190
- The ChartPatchBuilder instance.
191
- """
192
- if isinstance(tag, str) and not tag.startswith("urn:li:tag:"):
193
- tag = TagUrn.create_from_id(tag)
194
- self._add_patch(GlobalTags.ASPECT_NAME, "remove", path=f"/tags/{tag}", value={})
195
- return self
196
-
197
- def add_term(self, term: Term) -> "ChartPatchBuilder":
198
- """
199
- Adds a glossary term to the ChartPatchBuilder.
200
-
201
- Args:
202
- term: The Term object representing the glossary term to be added.
203
-
204
- Returns:
205
- The ChartPatchBuilder instance.
206
- """
207
- self._add_patch(
208
- GlossaryTerms.ASPECT_NAME, "add", path=f"/terms/{term.urn}", value=term
209
- )
210
- return self
211
-
212
- def remove_term(self, term: Union[str, Urn]) -> "ChartPatchBuilder":
213
- """
214
- Removes a glossary term from the ChartPatchBuilder.
215
-
216
- Args:
217
- term: The term to remove, specified as a string or Urn object.
218
-
219
- Returns:
220
- The ChartPatchBuilder instance.
221
- """
222
- if isinstance(term, str) and not term.startswith("urn:li:glossaryTerm:"):
223
- term = "urn:li:glossaryTerm:" + term
224
- self._add_patch(
225
- GlossaryTerms.ASPECT_NAME, "remove", path=f"/terms/{term}", value={}
226
- )
227
- return self
228
-
229
- def set_custom_properties(
230
- self, custom_properties: Dict[str, str]
231
- ) -> "ChartPatchBuilder":
232
- """
233
- Sets the custom properties for the ChartPatchBuilder.
234
-
235
- Args:
236
- custom_properties: A dictionary containing the custom properties to be set.
237
-
238
- Returns:
239
- The ChartPatchBuilder instance.
240
-
241
- Notes:
242
- This method replaces all existing custom properties with the given dictionary.
243
- """
244
- self._add_patch(
245
- ChartInfo.ASPECT_NAME,
246
- "add",
247
- path="/customProperties",
248
- value=custom_properties,
249
- )
250
- return self
251
-
252
- def add_custom_property(self, key: str, value: str) -> "ChartPatchBuilder":
253
- """
254
- Adds a custom property to the ChartPatchBuilder.
255
-
256
- Args:
257
- key: The key of the custom property.
258
- value: The value of the custom property.
259
-
260
- Returns:
261
- The ChartPatchBuilder instance.
262
- """
263
- self.custom_properties_patch_helper.add_property(key, value)
264
- return self
265
-
266
- def remove_custom_property(self, key: str) -> "ChartPatchBuilder":
267
- """
268
- Removes a custom property from the ChartPatchBuilder.
269
-
270
- Args:
271
- key: The key of the custom property to remove.
272
-
273
- Returns:
274
- The ChartPatchBuilder instance.
275
- """
276
- self.custom_properties_patch_helper.remove_property(key)
277
- return self
278
-
279
123
  def set_title(self, title: str) -> "ChartPatchBuilder":
280
124
  assert title, "ChartInfo title should not be None"
281
125
  self._add_patch(
282
126
  ChartInfo.ASPECT_NAME,
283
127
  "add",
284
- path="/title",
128
+ path=("title",),
285
129
  value=title,
286
130
  )
287
131
 
@@ -292,7 +136,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
292
136
  self._add_patch(
293
137
  ChartInfo.ASPECT_NAME,
294
138
  "add",
295
- path="/description",
139
+ path=("description",),
296
140
  value=description,
297
141
  )
298
142
 
@@ -303,7 +147,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
303
147
  self._add_patch(
304
148
  ChartInfo.ASPECT_NAME,
305
149
  "add",
306
- path="/lastRefreshed",
150
+ path=("lastRefreshed",),
307
151
  value=last_refreshed,
308
152
  )
309
153
 
@@ -316,7 +160,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
316
160
  self._add_patch(
317
161
  ChartInfo.ASPECT_NAME,
318
162
  "add",
319
- path="/lastModified",
163
+ path=("lastModified",),
320
164
  value=last_modified,
321
165
  )
322
166
 
@@ -327,7 +171,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
327
171
  self._add_patch(
328
172
  ChartInfo.ASPECT_NAME,
329
173
  "add",
330
- path="/externalUrl",
174
+ path=("externalUrl",),
331
175
  value=external_url,
332
176
  )
333
177
  return self
@@ -337,7 +181,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
337
181
  self._add_patch(
338
182
  ChartInfo.ASPECT_NAME,
339
183
  "add",
340
- path="/chartUrl",
184
+ path=("chartUrl",),
341
185
  value=dashboard_url,
342
186
  )
343
187
 
@@ -350,7 +194,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
350
194
  self._add_patch(
351
195
  ChartInfo.ASPECT_NAME,
352
196
  "add",
353
- path="/type",
197
+ path=("type",),
354
198
  value=type,
355
199
  )
356
200
 
@@ -363,7 +207,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
363
207
  self._add_patch(
364
208
  ChartInfo.ASPECT_NAME,
365
209
  "add",
366
- path="/access",
210
+ path=("access",),
367
211
  value=access,
368
212
  )
369
213
 
@@ -375,7 +219,7 @@ class ChartPatchBuilder(MetadataPatchProposal):
375
219
  self._add_patch(
376
220
  aspect_name=ChartInfo.ASPECT_NAME,
377
221
  op="add",
378
- path=f"/inputs/{urn}",
222
+ path=("inputs", urn),
379
223
  value=urn,
380
224
  )
381
225