acryl-datahub 0.15.0.1rc12__py3-none-any.whl → 0.15.0.1rc14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-0.15.0.1rc12.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/METADATA +2369 -2369
- {acryl_datahub-0.15.0.1rc12.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/RECORD +35 -31
- datahub/__init__.py +1 -1
- datahub/emitter/mce_builder.py +3 -3
- datahub/emitter/mcp_patch_builder.py +36 -12
- datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +44 -1
- datahub/ingestion/source/bigquery_v2/bigquery.py +10 -18
- datahub/ingestion/source/bigquery_v2/bigquery_config.py +3 -9
- datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +11 -17
- datahub/ingestion/source/bigquery_v2/lineage.py +9 -22
- datahub/ingestion/source/gc/dataprocess_cleanup.py +4 -4
- datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +159 -71
- datahub/ingestion/source/tableau/tableau.py +3 -0
- datahub/ingestion/source/tableau/tableau_common.py +18 -0
- datahub/metadata/_schema_classes.py +61 -1
- datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +4 -0
- datahub/metadata/schema.avsc +64 -29
- datahub/metadata/schemas/DataJobKey.avsc +2 -1
- datahub/metadata/schemas/DataTransformLogic.avsc +63 -0
- datahub/specific/aspect_helpers/__init__.py +0 -0
- datahub/specific/aspect_helpers/custom_properties.py +79 -0
- datahub/specific/aspect_helpers/ownership.py +67 -0
- datahub/specific/aspect_helpers/structured_properties.py +72 -0
- datahub/specific/aspect_helpers/tags.py +42 -0
- datahub/specific/aspect_helpers/terms.py +43 -0
- datahub/specific/chart.py +28 -184
- datahub/specific/dashboard.py +31 -196
- datahub/specific/datajob.py +34 -189
- datahub/specific/dataproduct.py +24 -86
- datahub/specific/dataset.py +48 -133
- datahub/specific/form.py +12 -32
- datahub/specific/structured_property.py +9 -9
- datahub/specific/custom_properties.py +0 -37
- datahub/specific/ownership.py +0 -48
- datahub/specific/structured_properties.py +0 -53
- {acryl_datahub-0.15.0.1rc12.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/WHEEL +0 -0
- {acryl_datahub-0.15.0.1rc12.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-0.15.0.1rc12.dist-info → acryl_datahub-0.15.0.1rc14.dist-info}/top_level.txt +0 -0
datahub/specific/dashboard.py
CHANGED
|
@@ -1,27 +1,28 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import List, Optional, Tuple, Union
|
|
2
2
|
|
|
3
|
-
from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
|
|
3
|
+
from datahub.emitter.mcp_patch_builder import MetadataPatchProposal, PatchPath
|
|
4
4
|
from datahub.metadata.schema_classes import (
|
|
5
5
|
AccessLevelClass,
|
|
6
6
|
ChangeAuditStampsClass,
|
|
7
7
|
DashboardInfoClass as DashboardInfo,
|
|
8
8
|
EdgeClass as Edge,
|
|
9
|
-
GlobalTagsClass as GlobalTags,
|
|
10
|
-
GlossaryTermAssociationClass as Term,
|
|
11
|
-
GlossaryTermsClass as GlossaryTerms,
|
|
12
9
|
KafkaAuditHeaderClass,
|
|
13
|
-
OwnerClass as Owner,
|
|
14
|
-
OwnershipTypeClass,
|
|
15
10
|
SystemMetadataClass,
|
|
16
|
-
TagAssociationClass as Tag,
|
|
17
11
|
)
|
|
18
|
-
from datahub.specific.custom_properties import
|
|
19
|
-
from datahub.specific.ownership import
|
|
20
|
-
from datahub.
|
|
12
|
+
from datahub.specific.aspect_helpers.custom_properties import HasCustomPropertiesPatch
|
|
13
|
+
from datahub.specific.aspect_helpers.ownership import HasOwnershipPatch
|
|
14
|
+
from datahub.specific.aspect_helpers.tags import HasTagsPatch
|
|
15
|
+
from datahub.specific.aspect_helpers.terms import HasTermsPatch
|
|
21
16
|
from datahub.utilities.urns.urn import Urn
|
|
22
17
|
|
|
23
18
|
|
|
24
|
-
class DashboardPatchBuilder(
|
|
19
|
+
class DashboardPatchBuilder(
|
|
20
|
+
HasOwnershipPatch,
|
|
21
|
+
HasCustomPropertiesPatch,
|
|
22
|
+
HasTagsPatch,
|
|
23
|
+
HasTermsPatch,
|
|
24
|
+
MetadataPatchProposal,
|
|
25
|
+
):
|
|
25
26
|
def __init__(
|
|
26
27
|
self,
|
|
27
28
|
urn: str,
|
|
@@ -39,55 +40,10 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
39
40
|
super().__init__(
|
|
40
41
|
urn, system_metadata=system_metadata, audit_header=audit_header
|
|
41
42
|
)
|
|
42
|
-
self.custom_properties_patch_helper = CustomPropertiesPatchHelper(
|
|
43
|
-
self, DashboardInfo.ASPECT_NAME
|
|
44
|
-
)
|
|
45
|
-
self.ownership_patch_helper = OwnershipPatchHelper(self)
|
|
46
|
-
|
|
47
|
-
def add_owner(self, owner: Owner) -> "DashboardPatchBuilder":
|
|
48
|
-
"""
|
|
49
|
-
Adds an owner to the DashboardPatchBuilder.
|
|
50
|
-
|
|
51
|
-
Args:
|
|
52
|
-
owner: The Owner object to add.
|
|
53
|
-
|
|
54
|
-
Returns:
|
|
55
|
-
The DashboardPatchBuilder instance.
|
|
56
|
-
"""
|
|
57
|
-
self.ownership_patch_helper.add_owner(owner)
|
|
58
|
-
return self
|
|
59
|
-
|
|
60
|
-
def remove_owner(
|
|
61
|
-
self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
|
|
62
|
-
) -> "DashboardPatchBuilder":
|
|
63
|
-
"""
|
|
64
|
-
Removes an owner from the DashboardPatchBuilder.
|
|
65
|
-
|
|
66
|
-
Args:
|
|
67
|
-
owner: The owner to remove.
|
|
68
|
-
owner_type: The ownership type of the owner (optional).
|
|
69
|
-
|
|
70
|
-
Returns:
|
|
71
|
-
The DashboardPatchBuilder instance.
|
|
72
|
-
|
|
73
|
-
Notes:
|
|
74
|
-
`owner_type` is optional.
|
|
75
|
-
"""
|
|
76
|
-
self.ownership_patch_helper.remove_owner(owner, owner_type)
|
|
77
|
-
return self
|
|
78
|
-
|
|
79
|
-
def set_owners(self, owners: List[Owner]) -> "DashboardPatchBuilder":
|
|
80
|
-
"""
|
|
81
|
-
Sets the owners of the DashboardPatchBuilder.
|
|
82
43
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
Returns:
|
|
87
|
-
The DashboardPatchBuilder instance.
|
|
88
|
-
"""
|
|
89
|
-
self.ownership_patch_helper.set_owners(owners)
|
|
90
|
-
return self
|
|
44
|
+
@classmethod
|
|
45
|
+
def _custom_properties_location(cls) -> Tuple[str, PatchPath]:
|
|
46
|
+
return DashboardInfo.ASPECT_NAME, ("customProperties",)
|
|
91
47
|
|
|
92
48
|
def add_dataset_edge(
|
|
93
49
|
self, dataset: Union[Edge, Urn, str]
|
|
@@ -126,7 +82,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
126
82
|
self._add_patch(
|
|
127
83
|
DashboardInfo.ASPECT_NAME,
|
|
128
84
|
"add",
|
|
129
|
-
path=
|
|
85
|
+
path=("datasetEdges", dataset_urn),
|
|
130
86
|
value=dataset_edge,
|
|
131
87
|
)
|
|
132
88
|
return self
|
|
@@ -144,7 +100,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
144
100
|
self._add_patch(
|
|
145
101
|
DashboardInfo.ASPECT_NAME,
|
|
146
102
|
"remove",
|
|
147
|
-
path=
|
|
103
|
+
path=("datasetEdges", dataset),
|
|
148
104
|
value={},
|
|
149
105
|
)
|
|
150
106
|
return self
|
|
@@ -169,7 +125,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
169
125
|
self._add_patch(
|
|
170
126
|
DashboardInfo.ASPECT_NAME,
|
|
171
127
|
"add",
|
|
172
|
-
path="
|
|
128
|
+
path=("datasetEdges",),
|
|
173
129
|
value=datasets,
|
|
174
130
|
)
|
|
175
131
|
return self
|
|
@@ -209,7 +165,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
209
165
|
self._add_patch(
|
|
210
166
|
DashboardInfo.ASPECT_NAME,
|
|
211
167
|
"add",
|
|
212
|
-
path=
|
|
168
|
+
path=("chartEdges", chart_urn),
|
|
213
169
|
value=chart_edge,
|
|
214
170
|
)
|
|
215
171
|
return self
|
|
@@ -227,7 +183,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
227
183
|
self._add_patch(
|
|
228
184
|
DashboardInfo.ASPECT_NAME,
|
|
229
185
|
"remove",
|
|
230
|
-
path=
|
|
186
|
+
path=("chartEdges", chart),
|
|
231
187
|
value={},
|
|
232
188
|
)
|
|
233
189
|
return self
|
|
@@ -252,129 +208,17 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
252
208
|
self._add_patch(
|
|
253
209
|
DashboardInfo.ASPECT_NAME,
|
|
254
210
|
"add",
|
|
255
|
-
path="
|
|
211
|
+
path=("chartEdges",),
|
|
256
212
|
value=charts,
|
|
257
213
|
)
|
|
258
214
|
return self
|
|
259
215
|
|
|
260
|
-
def add_tag(self, tag: Tag) -> "DashboardPatchBuilder":
|
|
261
|
-
"""
|
|
262
|
-
Adds a tag to the DashboardPatchBuilder.
|
|
263
|
-
|
|
264
|
-
Args:
|
|
265
|
-
tag: The Tag object representing the tag to be added.
|
|
266
|
-
|
|
267
|
-
Returns:
|
|
268
|
-
The DashboardPatchBuilder instance.
|
|
269
|
-
"""
|
|
270
|
-
self._add_patch(
|
|
271
|
-
GlobalTags.ASPECT_NAME, "add", path=f"/tags/{tag.tag}", value=tag
|
|
272
|
-
)
|
|
273
|
-
return self
|
|
274
|
-
|
|
275
|
-
def remove_tag(self, tag: Union[str, Urn]) -> "DashboardPatchBuilder":
|
|
276
|
-
"""
|
|
277
|
-
Removes a tag from the DashboardPatchBuilder.
|
|
278
|
-
|
|
279
|
-
Args:
|
|
280
|
-
tag: The tag to remove, specified as a string or Urn object.
|
|
281
|
-
|
|
282
|
-
Returns:
|
|
283
|
-
The DashboardPatchBuilder instance.
|
|
284
|
-
"""
|
|
285
|
-
if isinstance(tag, str) and not tag.startswith("urn:li:tag:"):
|
|
286
|
-
tag = TagUrn.create_from_id(tag)
|
|
287
|
-
self._add_patch(GlobalTags.ASPECT_NAME, "remove", path=f"/tags/{tag}", value={})
|
|
288
|
-
return self
|
|
289
|
-
|
|
290
|
-
def add_term(self, term: Term) -> "DashboardPatchBuilder":
|
|
291
|
-
"""
|
|
292
|
-
Adds a glossary term to the DashboardPatchBuilder.
|
|
293
|
-
|
|
294
|
-
Args:
|
|
295
|
-
term: The Term object representing the glossary term to be added.
|
|
296
|
-
|
|
297
|
-
Returns:
|
|
298
|
-
The DashboardPatchBuilder instance.
|
|
299
|
-
"""
|
|
300
|
-
self._add_patch(
|
|
301
|
-
GlossaryTerms.ASPECT_NAME, "add", path=f"/terms/{term.urn}", value=term
|
|
302
|
-
)
|
|
303
|
-
return self
|
|
304
|
-
|
|
305
|
-
def remove_term(self, term: Union[str, Urn]) -> "DashboardPatchBuilder":
|
|
306
|
-
"""
|
|
307
|
-
Removes a glossary term from the DashboardPatchBuilder.
|
|
308
|
-
|
|
309
|
-
Args:
|
|
310
|
-
term: The term to remove, specified as a string or Urn object.
|
|
311
|
-
|
|
312
|
-
Returns:
|
|
313
|
-
The DashboardPatchBuilder instance.
|
|
314
|
-
"""
|
|
315
|
-
if isinstance(term, str) and not term.startswith("urn:li:glossaryTerm:"):
|
|
316
|
-
term = "urn:li:glossaryTerm:" + term
|
|
317
|
-
self._add_patch(
|
|
318
|
-
GlossaryTerms.ASPECT_NAME, "remove", path=f"/terms/{term}", value={}
|
|
319
|
-
)
|
|
320
|
-
return self
|
|
321
|
-
|
|
322
|
-
def set_custom_properties(
|
|
323
|
-
self, custom_properties: Dict[str, str]
|
|
324
|
-
) -> "DashboardPatchBuilder":
|
|
325
|
-
"""
|
|
326
|
-
Sets the custom properties for the DashboardPatchBuilder.
|
|
327
|
-
|
|
328
|
-
Args:
|
|
329
|
-
custom_properties: A dictionary containing the custom properties to be set.
|
|
330
|
-
|
|
331
|
-
Returns:
|
|
332
|
-
The DashboardPatchBuilder instance.
|
|
333
|
-
|
|
334
|
-
Notes:
|
|
335
|
-
This method replaces all existing custom properties with the given dictionary.
|
|
336
|
-
"""
|
|
337
|
-
self._add_patch(
|
|
338
|
-
DashboardInfo.ASPECT_NAME,
|
|
339
|
-
"add",
|
|
340
|
-
path="/customProperties",
|
|
341
|
-
value=custom_properties,
|
|
342
|
-
)
|
|
343
|
-
return self
|
|
344
|
-
|
|
345
|
-
def add_custom_property(self, key: str, value: str) -> "DashboardPatchBuilder":
|
|
346
|
-
"""
|
|
347
|
-
Adds a custom property to the DashboardPatchBuilder.
|
|
348
|
-
|
|
349
|
-
Args:
|
|
350
|
-
key: The key of the custom property.
|
|
351
|
-
value: The value of the custom property.
|
|
352
|
-
|
|
353
|
-
Returns:
|
|
354
|
-
The DashboardPatchBuilder instance.
|
|
355
|
-
"""
|
|
356
|
-
self.custom_properties_patch_helper.add_property(key, value)
|
|
357
|
-
return self
|
|
358
|
-
|
|
359
|
-
def remove_custom_property(self, key: str) -> "DashboardPatchBuilder":
|
|
360
|
-
"""
|
|
361
|
-
Removes a custom property from the DashboardPatchBuilder.
|
|
362
|
-
|
|
363
|
-
Args:
|
|
364
|
-
key: The key of the custom property to remove.
|
|
365
|
-
|
|
366
|
-
Returns:
|
|
367
|
-
The DashboardPatchBuilder instance.
|
|
368
|
-
"""
|
|
369
|
-
self.custom_properties_patch_helper.remove_property(key)
|
|
370
|
-
return self
|
|
371
|
-
|
|
372
216
|
def set_title(self, title: str) -> "DashboardPatchBuilder":
|
|
373
217
|
assert title, "DashboardInfo title should not be None"
|
|
374
218
|
self._add_patch(
|
|
375
219
|
DashboardInfo.ASPECT_NAME,
|
|
376
220
|
"add",
|
|
377
|
-
path="
|
|
221
|
+
path=("title",),
|
|
378
222
|
value=title,
|
|
379
223
|
)
|
|
380
224
|
|
|
@@ -385,27 +229,18 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
385
229
|
self._add_patch(
|
|
386
230
|
DashboardInfo.ASPECT_NAME,
|
|
387
231
|
"add",
|
|
388
|
-
path="
|
|
232
|
+
path=("description",),
|
|
389
233
|
value=description,
|
|
390
234
|
)
|
|
391
235
|
|
|
392
236
|
return self
|
|
393
237
|
|
|
394
|
-
def add_custom_properties(
|
|
395
|
-
self, custom_properties: Optional[Dict[str, str]] = None
|
|
396
|
-
) -> "DashboardPatchBuilder":
|
|
397
|
-
if custom_properties:
|
|
398
|
-
for key, value in custom_properties.items():
|
|
399
|
-
self.custom_properties_patch_helper.add_property(key, value)
|
|
400
|
-
|
|
401
|
-
return self
|
|
402
|
-
|
|
403
238
|
def set_external_url(self, external_url: Optional[str]) -> "DashboardPatchBuilder":
|
|
404
239
|
if external_url:
|
|
405
240
|
self._add_patch(
|
|
406
241
|
DashboardInfo.ASPECT_NAME,
|
|
407
242
|
"add",
|
|
408
|
-
path="
|
|
243
|
+
path=("externalUrl",),
|
|
409
244
|
value=external_url,
|
|
410
245
|
)
|
|
411
246
|
return self
|
|
@@ -416,7 +251,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
416
251
|
self._add_patch(
|
|
417
252
|
aspect_name=DashboardInfo.ASPECT_NAME,
|
|
418
253
|
op="add",
|
|
419
|
-
path=
|
|
254
|
+
path=("charts", urn),
|
|
420
255
|
value=urn,
|
|
421
256
|
)
|
|
422
257
|
|
|
@@ -430,7 +265,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
430
265
|
self._add_patch(
|
|
431
266
|
aspect_name=DashboardInfo.ASPECT_NAME,
|
|
432
267
|
op="add",
|
|
433
|
-
path=
|
|
268
|
+
path=("datasets", urn),
|
|
434
269
|
value=urn,
|
|
435
270
|
)
|
|
436
271
|
|
|
@@ -443,7 +278,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
443
278
|
self._add_patch(
|
|
444
279
|
DashboardInfo.ASPECT_NAME,
|
|
445
280
|
"add",
|
|
446
|
-
path="
|
|
281
|
+
path=("dashboardUrl",),
|
|
447
282
|
value=dashboard_url,
|
|
448
283
|
)
|
|
449
284
|
|
|
@@ -456,7 +291,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
456
291
|
self._add_patch(
|
|
457
292
|
DashboardInfo.ASPECT_NAME,
|
|
458
293
|
"add",
|
|
459
|
-
path="
|
|
294
|
+
path=("access",),
|
|
460
295
|
value=access,
|
|
461
296
|
)
|
|
462
297
|
|
|
@@ -469,7 +304,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
469
304
|
self._add_patch(
|
|
470
305
|
DashboardInfo.ASPECT_NAME,
|
|
471
306
|
"add",
|
|
472
|
-
path="
|
|
307
|
+
path=("lastRefreshed",),
|
|
473
308
|
value=last_refreshed,
|
|
474
309
|
)
|
|
475
310
|
|
|
@@ -482,7 +317,7 @@ class DashboardPatchBuilder(MetadataPatchProposal):
|
|
|
482
317
|
self._add_patch(
|
|
483
318
|
DashboardInfo.ASPECT_NAME,
|
|
484
319
|
"add",
|
|
485
|
-
path="
|
|
320
|
+
path=("lastModified",),
|
|
486
321
|
value=last_modified,
|
|
487
322
|
)
|
|
488
323
|
|
datahub/specific/datajob.py
CHANGED
|
@@ -1,25 +1,27 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import List, Optional, Tuple, Union
|
|
2
2
|
|
|
3
|
-
from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
|
|
3
|
+
from datahub.emitter.mcp_patch_builder import MetadataPatchProposal, PatchPath
|
|
4
4
|
from datahub.metadata.schema_classes import (
|
|
5
5
|
DataJobInfoClass as DataJobInfo,
|
|
6
6
|
DataJobInputOutputClass as DataJobInputOutput,
|
|
7
7
|
EdgeClass as Edge,
|
|
8
|
-
GlobalTagsClass as GlobalTags,
|
|
9
|
-
GlossaryTermAssociationClass as Term,
|
|
10
|
-
GlossaryTermsClass as GlossaryTerms,
|
|
11
8
|
KafkaAuditHeaderClass,
|
|
12
|
-
OwnerClass as Owner,
|
|
13
|
-
OwnershipTypeClass,
|
|
14
9
|
SystemMetadataClass,
|
|
15
|
-
TagAssociationClass as Tag,
|
|
16
10
|
)
|
|
17
|
-
from datahub.metadata.urns import SchemaFieldUrn,
|
|
18
|
-
from datahub.specific.custom_properties import
|
|
19
|
-
from datahub.specific.ownership import
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
11
|
+
from datahub.metadata.urns import SchemaFieldUrn, Urn
|
|
12
|
+
from datahub.specific.aspect_helpers.custom_properties import HasCustomPropertiesPatch
|
|
13
|
+
from datahub.specific.aspect_helpers.ownership import HasOwnershipPatch
|
|
14
|
+
from datahub.specific.aspect_helpers.tags import HasTagsPatch
|
|
15
|
+
from datahub.specific.aspect_helpers.terms import HasTermsPatch
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DataJobPatchBuilder(
|
|
19
|
+
HasOwnershipPatch,
|
|
20
|
+
HasCustomPropertiesPatch,
|
|
21
|
+
HasTagsPatch,
|
|
22
|
+
HasTermsPatch,
|
|
23
|
+
MetadataPatchProposal,
|
|
24
|
+
):
|
|
23
25
|
def __init__(
|
|
24
26
|
self,
|
|
25
27
|
urn: str,
|
|
@@ -37,55 +39,10 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
37
39
|
super().__init__(
|
|
38
40
|
urn, system_metadata=system_metadata, audit_header=audit_header
|
|
39
41
|
)
|
|
40
|
-
self.custom_properties_patch_helper = CustomPropertiesPatchHelper(
|
|
41
|
-
self, DataJobInfo.ASPECT_NAME
|
|
42
|
-
)
|
|
43
|
-
self.ownership_patch_helper = OwnershipPatchHelper(self)
|
|
44
|
-
|
|
45
|
-
def add_owner(self, owner: Owner) -> "DataJobPatchBuilder":
|
|
46
|
-
"""
|
|
47
|
-
Adds an owner to the DataJobPatchBuilder.
|
|
48
|
-
|
|
49
|
-
Args:
|
|
50
|
-
owner: The Owner object to add.
|
|
51
|
-
|
|
52
|
-
Returns:
|
|
53
|
-
The DataJobPatchBuilder instance.
|
|
54
|
-
"""
|
|
55
|
-
self.ownership_patch_helper.add_owner(owner)
|
|
56
|
-
return self
|
|
57
|
-
|
|
58
|
-
def remove_owner(
|
|
59
|
-
self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
|
|
60
|
-
) -> "DataJobPatchBuilder":
|
|
61
|
-
"""
|
|
62
|
-
Removes an owner from the DataJobPatchBuilder.
|
|
63
|
-
|
|
64
|
-
Args:
|
|
65
|
-
owner: The owner to remove.
|
|
66
|
-
owner_type: The ownership type of the owner (optional).
|
|
67
|
-
|
|
68
|
-
Returns:
|
|
69
|
-
The DataJobPatchBuilder instance.
|
|
70
|
-
|
|
71
|
-
Notes:
|
|
72
|
-
`owner_type` is optional.
|
|
73
|
-
"""
|
|
74
|
-
self.ownership_patch_helper.remove_owner(owner, owner_type)
|
|
75
|
-
return self
|
|
76
|
-
|
|
77
|
-
def set_owners(self, owners: List[Owner]) -> "DataJobPatchBuilder":
|
|
78
|
-
"""
|
|
79
|
-
Sets the owners of the DataJobPatchBuilder.
|
|
80
|
-
|
|
81
|
-
Args:
|
|
82
|
-
owners: A list of Owner objects.
|
|
83
42
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
""
|
|
87
|
-
self.ownership_patch_helper.set_owners(owners)
|
|
88
|
-
return self
|
|
43
|
+
@classmethod
|
|
44
|
+
def _custom_properties_location(cls) -> Tuple[str, PatchPath]:
|
|
45
|
+
return DataJobInfo.ASPECT_NAME, ("customProperties",)
|
|
89
46
|
|
|
90
47
|
def add_input_datajob(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilder":
|
|
91
48
|
"""
|
|
@@ -120,7 +77,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
120
77
|
self._add_patch(
|
|
121
78
|
DataJobInputOutput.ASPECT_NAME,
|
|
122
79
|
"add",
|
|
123
|
-
path=
|
|
80
|
+
path=("inputDatajobEdges", input_urn),
|
|
124
81
|
value=input_edge,
|
|
125
82
|
)
|
|
126
83
|
return self
|
|
@@ -138,7 +95,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
138
95
|
self._add_patch(
|
|
139
96
|
DataJobInputOutput.ASPECT_NAME,
|
|
140
97
|
"remove",
|
|
141
|
-
path=
|
|
98
|
+
path=("inputDatajobEdges", input),
|
|
142
99
|
value={},
|
|
143
100
|
)
|
|
144
101
|
return self
|
|
@@ -163,7 +120,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
163
120
|
self._add_patch(
|
|
164
121
|
DataJobInputOutput.ASPECT_NAME,
|
|
165
122
|
"add",
|
|
166
|
-
path="
|
|
123
|
+
path=("inputDatajobEdges",),
|
|
167
124
|
value=inputs,
|
|
168
125
|
)
|
|
169
126
|
return self
|
|
@@ -201,7 +158,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
201
158
|
self._add_patch(
|
|
202
159
|
DataJobInputOutput.ASPECT_NAME,
|
|
203
160
|
"add",
|
|
204
|
-
path=
|
|
161
|
+
path=("inputDatasetEdges", input_urn),
|
|
205
162
|
value=input_edge,
|
|
206
163
|
)
|
|
207
164
|
return self
|
|
@@ -219,7 +176,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
219
176
|
self._add_patch(
|
|
220
177
|
DataJobInputOutput.ASPECT_NAME,
|
|
221
178
|
"remove",
|
|
222
|
-
path=
|
|
179
|
+
path=("inputDatasetEdges", input),
|
|
223
180
|
value={},
|
|
224
181
|
)
|
|
225
182
|
return self
|
|
@@ -244,7 +201,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
244
201
|
self._add_patch(
|
|
245
202
|
DataJobInputOutput.ASPECT_NAME,
|
|
246
203
|
"add",
|
|
247
|
-
path="
|
|
204
|
+
path=("inputDatasetEdges",),
|
|
248
205
|
value=inputs,
|
|
249
206
|
)
|
|
250
207
|
return self
|
|
@@ -284,7 +241,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
284
241
|
self._add_patch(
|
|
285
242
|
DataJobInputOutput.ASPECT_NAME,
|
|
286
243
|
"add",
|
|
287
|
-
path=
|
|
244
|
+
path=("outputDatasetEdges", output_urn),
|
|
288
245
|
value=output_edge,
|
|
289
246
|
)
|
|
290
247
|
return self
|
|
@@ -302,7 +259,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
302
259
|
self._add_patch(
|
|
303
260
|
DataJobInputOutput.ASPECT_NAME,
|
|
304
261
|
"remove",
|
|
305
|
-
path=
|
|
262
|
+
path=("outputDatasetEdges", output),
|
|
306
263
|
value={},
|
|
307
264
|
)
|
|
308
265
|
return self
|
|
@@ -327,7 +284,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
327
284
|
self._add_patch(
|
|
328
285
|
DataJobInputOutput.ASPECT_NAME,
|
|
329
286
|
"add",
|
|
330
|
-
path="
|
|
287
|
+
path=("outputDatasetEdges",),
|
|
331
288
|
value=outputs,
|
|
332
289
|
)
|
|
333
290
|
return self
|
|
@@ -351,7 +308,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
351
308
|
self._add_patch(
|
|
352
309
|
DataJobInputOutput.ASPECT_NAME,
|
|
353
310
|
"add",
|
|
354
|
-
path=
|
|
311
|
+
path=("inputDatasetFields", input_urn),
|
|
355
312
|
value={},
|
|
356
313
|
)
|
|
357
314
|
return self
|
|
@@ -372,7 +329,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
372
329
|
self._add_patch(
|
|
373
330
|
DataJobInputOutput.ASPECT_NAME,
|
|
374
331
|
"remove",
|
|
375
|
-
path=
|
|
332
|
+
path=("inputDatasetFields", input_urn),
|
|
376
333
|
value={},
|
|
377
334
|
)
|
|
378
335
|
return self
|
|
@@ -397,7 +354,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
397
354
|
self._add_patch(
|
|
398
355
|
DataJobInputOutput.ASPECT_NAME,
|
|
399
356
|
"add",
|
|
400
|
-
path="
|
|
357
|
+
path=("inputDatasetFields",),
|
|
401
358
|
value=inputs,
|
|
402
359
|
)
|
|
403
360
|
return self
|
|
@@ -423,7 +380,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
423
380
|
self._add_patch(
|
|
424
381
|
DataJobInputOutput.ASPECT_NAME,
|
|
425
382
|
"add",
|
|
426
|
-
path=
|
|
383
|
+
path=("outputDatasetFields", output_urn),
|
|
427
384
|
value={},
|
|
428
385
|
)
|
|
429
386
|
return self
|
|
@@ -444,7 +401,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
444
401
|
self._add_patch(
|
|
445
402
|
DataJobInputOutput.ASPECT_NAME,
|
|
446
403
|
"remove",
|
|
447
|
-
path=
|
|
404
|
+
path=("outputDatasetFields", output_urn),
|
|
448
405
|
value={},
|
|
449
406
|
)
|
|
450
407
|
return self
|
|
@@ -469,119 +426,7 @@ class DataJobPatchBuilder(MetadataPatchProposal):
|
|
|
469
426
|
self._add_patch(
|
|
470
427
|
DataJobInputOutput.ASPECT_NAME,
|
|
471
428
|
"add",
|
|
472
|
-
path="
|
|
429
|
+
path=("outputDatasetFields",),
|
|
473
430
|
value=outputs,
|
|
474
431
|
)
|
|
475
432
|
return self
|
|
476
|
-
|
|
477
|
-
def add_tag(self, tag: Tag) -> "DataJobPatchBuilder":
|
|
478
|
-
"""
|
|
479
|
-
Adds a tag to the DataJobPatchBuilder.
|
|
480
|
-
|
|
481
|
-
Args:
|
|
482
|
-
tag: The Tag object representing the tag to be added.
|
|
483
|
-
|
|
484
|
-
Returns:
|
|
485
|
-
The DataJobPatchBuilder instance.
|
|
486
|
-
"""
|
|
487
|
-
self._add_patch(
|
|
488
|
-
GlobalTags.ASPECT_NAME, "add", path=f"/tags/{tag.tag}", value=tag
|
|
489
|
-
)
|
|
490
|
-
return self
|
|
491
|
-
|
|
492
|
-
def remove_tag(self, tag: Union[str, Urn]) -> "DataJobPatchBuilder":
|
|
493
|
-
"""
|
|
494
|
-
Removes a tag from the DataJobPatchBuilder.
|
|
495
|
-
|
|
496
|
-
Args:
|
|
497
|
-
tag: The tag to remove, specified as a string or Urn object.
|
|
498
|
-
|
|
499
|
-
Returns:
|
|
500
|
-
The DataJobPatchBuilder instance.
|
|
501
|
-
"""
|
|
502
|
-
if isinstance(tag, str) and not tag.startswith("urn:li:tag:"):
|
|
503
|
-
tag = TagUrn.create_from_id(tag)
|
|
504
|
-
self._add_patch(GlobalTags.ASPECT_NAME, "remove", path=f"/tags/{tag}", value={})
|
|
505
|
-
return self
|
|
506
|
-
|
|
507
|
-
def add_term(self, term: Term) -> "DataJobPatchBuilder":
|
|
508
|
-
"""
|
|
509
|
-
Adds a glossary term to the DataJobPatchBuilder.
|
|
510
|
-
|
|
511
|
-
Args:
|
|
512
|
-
term: The Term object representing the glossary term to be added.
|
|
513
|
-
|
|
514
|
-
Returns:
|
|
515
|
-
The DataJobPatchBuilder instance.
|
|
516
|
-
"""
|
|
517
|
-
self._add_patch(
|
|
518
|
-
GlossaryTerms.ASPECT_NAME, "add", path=f"/terms/{term.urn}", value=term
|
|
519
|
-
)
|
|
520
|
-
return self
|
|
521
|
-
|
|
522
|
-
def remove_term(self, term: Union[str, Urn]) -> "DataJobPatchBuilder":
|
|
523
|
-
"""
|
|
524
|
-
Removes a glossary term from the DataJobPatchBuilder.
|
|
525
|
-
|
|
526
|
-
Args:
|
|
527
|
-
term: The term to remove, specified as a string or Urn object.
|
|
528
|
-
|
|
529
|
-
Returns:
|
|
530
|
-
The DataJobPatchBuilder instance.
|
|
531
|
-
"""
|
|
532
|
-
if isinstance(term, str) and not term.startswith("urn:li:glossaryTerm:"):
|
|
533
|
-
term = "urn:li:glossaryTerm:" + term
|
|
534
|
-
self._add_patch(
|
|
535
|
-
GlossaryTerms.ASPECT_NAME, "remove", path=f"/terms/{term}", value={}
|
|
536
|
-
)
|
|
537
|
-
return self
|
|
538
|
-
|
|
539
|
-
def set_custom_properties(
|
|
540
|
-
self, custom_properties: Dict[str, str]
|
|
541
|
-
) -> "DataJobPatchBuilder":
|
|
542
|
-
"""
|
|
543
|
-
Sets the custom properties for the DataJobPatchBuilder.
|
|
544
|
-
|
|
545
|
-
Args:
|
|
546
|
-
custom_properties: A dictionary containing the custom properties to be set.
|
|
547
|
-
|
|
548
|
-
Returns:
|
|
549
|
-
The DataJobPatchBuilder instance.
|
|
550
|
-
|
|
551
|
-
Notes:
|
|
552
|
-
This method replaces all existing custom properties with the given dictionary.
|
|
553
|
-
"""
|
|
554
|
-
self._add_patch(
|
|
555
|
-
DataJobInfo.ASPECT_NAME,
|
|
556
|
-
"add",
|
|
557
|
-
path="/customProperties",
|
|
558
|
-
value=custom_properties,
|
|
559
|
-
)
|
|
560
|
-
return self
|
|
561
|
-
|
|
562
|
-
def add_custom_property(self, key: str, value: str) -> "DataJobPatchBuilder":
|
|
563
|
-
"""
|
|
564
|
-
Adds a custom property to the DataJobPatchBuilder.
|
|
565
|
-
|
|
566
|
-
Args:
|
|
567
|
-
key: The key of the custom property.
|
|
568
|
-
value: The value of the custom property.
|
|
569
|
-
|
|
570
|
-
Returns:
|
|
571
|
-
The DataJobPatchBuilder instance.
|
|
572
|
-
"""
|
|
573
|
-
self.custom_properties_patch_helper.add_property(key, value)
|
|
574
|
-
return self
|
|
575
|
-
|
|
576
|
-
def remove_custom_property(self, key: str) -> "DataJobPatchBuilder":
|
|
577
|
-
"""
|
|
578
|
-
Removes a custom property from the DataJobPatchBuilder.
|
|
579
|
-
|
|
580
|
-
Args:
|
|
581
|
-
key: The key of the custom property to remove.
|
|
582
|
-
|
|
583
|
-
Returns:
|
|
584
|
-
The DataJobPatchBuilder instance.
|
|
585
|
-
"""
|
|
586
|
-
self.custom_properties_patch_helper.remove_property(key)
|
|
587
|
-
return self
|