acryl-datahub 1.2.0.7rc2__py3-none-any.whl → 1.2.0.7rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of acryl-datahub might be problematic. Click here for more details.

Files changed (24) hide show
  1. {acryl_datahub-1.2.0.7rc2.dist-info → acryl_datahub-1.2.0.7rc3.dist-info}/METADATA +2525 -2521
  2. {acryl_datahub-1.2.0.7rc2.dist-info → acryl_datahub-1.2.0.7rc3.dist-info}/RECORD +23 -24
  3. datahub/_version.py +1 -1
  4. datahub/ingestion/source/redshift/config.py +9 -6
  5. datahub/ingestion/source/redshift/lineage.py +386 -687
  6. datahub/ingestion/source/redshift/redshift.py +19 -106
  7. datahub/ingestion/source/snowflake/snowflake_schema_gen.py +4 -1
  8. datahub/ingestion/source/snowflake/snowflake_v2.py +1 -0
  9. datahub/ingestion/source/sql/mssql/job_models.py +3 -1
  10. datahub/ingestion/source/sql/mssql/source.py +62 -3
  11. datahub/ingestion/source/unity/config.py +11 -0
  12. datahub/ingestion/source/unity/proxy.py +77 -0
  13. datahub/ingestion/source/unity/proxy_types.py +24 -0
  14. datahub/ingestion/source/unity/report.py +5 -0
  15. datahub/ingestion/source/unity/source.py +99 -1
  16. datahub/metadata/_internal_schema_classes.py +5 -5
  17. datahub/metadata/schema.avsc +66 -60
  18. datahub/metadata/schemas/LogicalParent.avsc +104 -100
  19. datahub/metadata/schemas/SchemaFieldKey.avsc +3 -1
  20. datahub/ingestion/source/redshift/lineage_v2.py +0 -466
  21. {acryl_datahub-1.2.0.7rc2.dist-info → acryl_datahub-1.2.0.7rc3.dist-info}/WHEEL +0 -0
  22. {acryl_datahub-1.2.0.7rc2.dist-info → acryl_datahub-1.2.0.7rc3.dist-info}/entry_points.txt +0 -0
  23. {acryl_datahub-1.2.0.7rc2.dist-info → acryl_datahub-1.2.0.7rc3.dist-info}/licenses/LICENSE +0 -0
  24. {acryl_datahub-1.2.0.7rc2.dist-info → acryl_datahub-1.2.0.7rc3.dist-info}/top_level.txt +0 -0
datahub/metadata/_internal_schema_classes.py CHANGED
@@ -15283,23 +15283,23 @@ class LogicalParentClass(_Aspect):
15283
15283
  RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.logical.LogicalParent")
15284
15284
 
15285
15285
  def __init__(self,
15286
- parent: "EdgeClass",
15286
+ parent: Union[None, "EdgeClass"]=None,
15287
15287
  ):
15288
15288
  super().__init__()
15289
15289
 
15290
15290
  self.parent = parent
15291
15291
 
15292
15292
  def _restore_defaults(self) -> None:
15293
- self.parent = EdgeClass._construct_with_defaults()
15293
+ self.parent = self.RECORD_SCHEMA.fields_dict["parent"].default
15294
15294
 
15295
15295
 
15296
15296
  @property
15297
- def parent(self) -> "EdgeClass":
15297
+ def parent(self) -> Union[None, "EdgeClass"]:
15298
15298
  # No docs available.
15299
15299
  return self._inner_dict.get('parent') # type: ignore
15300
15300
 
15301
15301
  @parent.setter
15302
- def parent(self, value: "EdgeClass") -> None:
15302
+ def parent(self, value: Union[None, "EdgeClass"]) -> None:
15303
15303
  self._inner_dict['parent'] = value
15304
15304
 
15305
15305
 
@@ -17004,7 +17004,7 @@ class SchemaFieldKeyClass(_Aspect):
17004
17004
 
17005
17005
 
17006
17006
  ASPECT_NAME = 'schemaFieldKey'
17007
- ASPECT_INFO = {'keyForEntity': 'schemaField', 'entityCategory': 'core', 'entityAspects': ['schemafieldInfo', 'structuredProperties', 'forms', 'businessAttributes', 'status', 'schemaFieldAliases', 'documentation', 'testResults', 'deprecation', 'subTypes', 'logicalParent']}
17007
+ ASPECT_INFO = {'keyForEntity': 'schemaField', 'entityCategory': 'core', 'entityAspects': ['schemafieldInfo', 'structuredProperties', 'forms', 'businessAttributes', 'status', 'schemaFieldAliases', 'documentation', 'testResults', 'deprecation', 'subTypes', 'logicalParent', 'globalTags', 'glossaryTerms']}
17008
17008
  RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.SchemaFieldKey")
17009
17009
 
17010
17010
  def __init__(self,
datahub/metadata/schema.avsc CHANGED
@@ -4410,7 +4410,9 @@
4410
4410
  "testResults",
4411
4411
  "deprecation",
4412
4412
  "subTypes",
4413
- "logicalParent"
4413
+ "logicalParent",
4414
+ "globalTags",
4415
+ "glossaryTerms"
4414
4416
  ]
4415
4417
  },
4416
4418
  "name": "SchemaFieldKey",
@@ -10423,67 +10425,71 @@
10423
10425
  "queryByDefault": false
10424
10426
  }
10425
10427
  },
10426
- "type": {
10427
- "type": "record",
10428
- "name": "Edge",
10429
- "namespace": "com.linkedin.pegasus2avro.common",
10430
- "fields": [
10431
- {
10432
- "java": {
10433
- "class": "com.linkedin.pegasus2avro.common.urn.Urn"
10428
+ "type": [
10429
+ "null",
10430
+ {
10431
+ "type": "record",
10432
+ "name": "Edge",
10433
+ "namespace": "com.linkedin.pegasus2avro.common",
10434
+ "fields": [
10435
+ {
10436
+ "java": {
10437
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
10438
+ },
10439
+ "Urn": "Urn",
10440
+ "type": [
10441
+ "null",
10442
+ "string"
10443
+ ],
10444
+ "name": "sourceUrn",
10445
+ "default": null,
10446
+ "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to."
10434
10447
  },
10435
- "Urn": "Urn",
10436
- "type": [
10437
- "null",
10438
- "string"
10439
- ],
10440
- "name": "sourceUrn",
10441
- "default": null,
10442
- "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to."
10443
- },
10444
- {
10445
- "java": {
10446
- "class": "com.linkedin.pegasus2avro.common.urn.Urn"
10448
+ {
10449
+ "java": {
10450
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
10451
+ },
10452
+ "Urn": "Urn",
10453
+ "type": "string",
10454
+ "name": "destinationUrn",
10455
+ "doc": "Urn of the destination of this relationship edge."
10447
10456
  },
10448
- "Urn": "Urn",
10449
- "type": "string",
10450
- "name": "destinationUrn",
10451
- "doc": "Urn of the destination of this relationship edge."
10452
- },
10453
- {
10454
- "type": [
10455
- "null",
10456
- "com.linkedin.pegasus2avro.common.AuditStamp"
10457
- ],
10458
- "name": "created",
10459
- "default": null,
10460
- "doc": "Audit stamp containing who created this relationship edge and when"
10461
- },
10462
- {
10463
- "type": [
10464
- "null",
10465
- "com.linkedin.pegasus2avro.common.AuditStamp"
10466
- ],
10467
- "name": "lastModified",
10468
- "default": null,
10469
- "doc": "Audit stamp containing who last modified this relationship edge and when"
10470
- },
10471
- {
10472
- "type": [
10473
- "null",
10474
- {
10475
- "type": "map",
10476
- "values": "string"
10477
- }
10478
- ],
10479
- "name": "properties",
10480
- "default": null,
10481
- "doc": "A generic properties bag that allows us to store specific information on this graph edge."
10482
- }
10483
- ],
10484
- "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
10485
- },
10486
- "name": "parent"
10457
+ {
10458
+ "type": [
10459
+ "null",
10460
+ "com.linkedin.pegasus2avro.common.AuditStamp"
10461
+ ],
10462
+ "name": "created",
10463
+ "default": null,
10464
+ "doc": "Audit stamp containing who created this relationship edge and when"
10465
+ },
10466
+ {
10467
+ "type": [
10468
+ "null",
10469
+ "com.linkedin.pegasus2avro.common.AuditStamp"
10470
+ ],
10471
+ "name": "lastModified",
10472
+ "default": null,
10473
+ "doc": "Audit stamp containing who last modified this relationship edge and when"
10474
+ },
10475
+ {
10476
+ "type": [
10477
+ "null",
10478
+ {
10479
+ "type": "map",
10480
+ "values": "string"
10481
+ }
10482
+ ],
10483
+ "name": "properties",
10484
+ "default": null,
10485
+ "doc": "A generic properties bag that allows us to store specific information on this graph edge."
10486
+ }
10487
+ ],
10488
+ "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
10489
+ }
10490
+ ],
10491
+ "name": "parent",
10492
+ "default": null
10487
10493
  }
10488
10494
  ]
10489
10495
  },
datahub/metadata/schemas/LogicalParent.avsc CHANGED
@@ -31,110 +31,114 @@
31
31
  "queryByDefault": false
32
32
  }
33
33
  },
34
- "type": {
35
- "type": "record",
36
- "name": "Edge",
37
- "namespace": "com.linkedin.pegasus2avro.common",
38
- "fields": [
39
- {
40
- "java": {
41
- "class": "com.linkedin.pegasus2avro.common.urn.Urn"
34
+ "type": [
35
+ "null",
36
+ {
37
+ "type": "record",
38
+ "name": "Edge",
39
+ "namespace": "com.linkedin.pegasus2avro.common",
40
+ "fields": [
41
+ {
42
+ "java": {
43
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
44
+ },
45
+ "type": [
46
+ "null",
47
+ "string"
48
+ ],
49
+ "name": "sourceUrn",
50
+ "default": null,
51
+ "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
52
+ "Urn": "Urn"
42
53
  },
43
- "type": [
44
- "null",
45
- "string"
46
- ],
47
- "name": "sourceUrn",
48
- "default": null,
49
- "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
50
- "Urn": "Urn"
51
- },
52
- {
53
- "java": {
54
- "class": "com.linkedin.pegasus2avro.common.urn.Urn"
54
+ {
55
+ "java": {
56
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
57
+ },
58
+ "type": "string",
59
+ "name": "destinationUrn",
60
+ "doc": "Urn of the destination of this relationship edge.",
61
+ "Urn": "Urn"
55
62
  },
56
- "type": "string",
57
- "name": "destinationUrn",
58
- "doc": "Urn of the destination of this relationship edge.",
59
- "Urn": "Urn"
60
- },
61
- {
62
- "type": [
63
- "null",
64
- {
65
- "type": "record",
66
- "name": "AuditStamp",
67
- "namespace": "com.linkedin.pegasus2avro.common",
68
- "fields": [
69
- {
70
- "type": "long",
71
- "name": "time",
72
- "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
73
- },
74
- {
75
- "java": {
76
- "class": "com.linkedin.pegasus2avro.common.urn.Urn"
63
+ {
64
+ "type": [
65
+ "null",
66
+ {
67
+ "type": "record",
68
+ "name": "AuditStamp",
69
+ "namespace": "com.linkedin.pegasus2avro.common",
70
+ "fields": [
71
+ {
72
+ "type": "long",
73
+ "name": "time",
74
+ "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
77
75
  },
78
- "type": "string",
79
- "name": "actor",
80
- "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
81
- "Urn": "Urn"
82
- },
83
- {
84
- "java": {
85
- "class": "com.linkedin.pegasus2avro.common.urn.Urn"
76
+ {
77
+ "java": {
78
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
79
+ },
80
+ "type": "string",
81
+ "name": "actor",
82
+ "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
83
+ "Urn": "Urn"
86
84
  },
87
- "type": [
88
- "null",
89
- "string"
90
- ],
91
- "name": "impersonator",
92
- "default": null,
93
- "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
94
- "Urn": "Urn"
95
- },
96
- {
97
- "type": [
98
- "null",
99
- "string"
100
- ],
101
- "name": "message",
102
- "default": null,
103
- "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
104
- }
105
- ],
106
- "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
107
- }
108
- ],
109
- "name": "created",
110
- "default": null,
111
- "doc": "Audit stamp containing who created this relationship edge and when"
112
- },
113
- {
114
- "type": [
115
- "null",
116
- "com.linkedin.pegasus2avro.common.AuditStamp"
117
- ],
118
- "name": "lastModified",
119
- "default": null,
120
- "doc": "Audit stamp containing who last modified this relationship edge and when"
121
- },
122
- {
123
- "type": [
124
- "null",
125
- {
126
- "type": "map",
127
- "values": "string"
128
- }
129
- ],
130
- "name": "properties",
131
- "default": null,
132
- "doc": "A generic properties bag that allows us to store specific information on this graph edge."
133
- }
134
- ],
135
- "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
136
- },
137
- "name": "parent"
85
+ {
86
+ "java": {
87
+ "class": "com.linkedin.pegasus2avro.common.urn.Urn"
88
+ },
89
+ "type": [
90
+ "null",
91
+ "string"
92
+ ],
93
+ "name": "impersonator",
94
+ "default": null,
95
+ "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
96
+ "Urn": "Urn"
97
+ },
98
+ {
99
+ "type": [
100
+ "null",
101
+ "string"
102
+ ],
103
+ "name": "message",
104
+ "default": null,
105
+ "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
106
+ }
107
+ ],
108
+ "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
109
+ }
110
+ ],
111
+ "name": "created",
112
+ "default": null,
113
+ "doc": "Audit stamp containing who created this relationship edge and when"
114
+ },
115
+ {
116
+ "type": [
117
+ "null",
118
+ "com.linkedin.pegasus2avro.common.AuditStamp"
119
+ ],
120
+ "name": "lastModified",
121
+ "default": null,
122
+ "doc": "Audit stamp containing who last modified this relationship edge and when"
123
+ },
124
+ {
125
+ "type": [
126
+ "null",
127
+ {
128
+ "type": "map",
129
+ "values": "string"
130
+ }
131
+ ],
132
+ "name": "properties",
133
+ "default": null,
134
+ "doc": "A generic properties bag that allows us to store specific information on this graph edge."
135
+ }
136
+ ],
137
+ "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
138
+ }
139
+ ],
140
+ "name": "parent",
141
+ "default": null
138
142
  }
139
143
  ]
140
144
  }
datahub/metadata/schemas/SchemaFieldKey.avsc CHANGED
@@ -15,7 +15,9 @@
15
15
  "testResults",
16
16
  "deprecation",
17
17
  "subTypes",
18
- "logicalParent"
18
+ "logicalParent",
19
+ "globalTags",
20
+ "glossaryTerms"
19
21
  ]
20
22
  },
21
23
  "name": "SchemaFieldKey",