acryl-datahub 1.2.0.11rc4__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.2.0.11rc4.dist-info → acryl_datahub-1.3.0.dist-info}/METADATA +2582 -2577
- {acryl_datahub-1.2.0.11rc4.dist-info → acryl_datahub-1.3.0.dist-info}/RECORD +43 -40
- datahub/_version.py +1 -1
- datahub/cli/docker_check.py +1 -1
- datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py +296 -0
- datahub/ingestion/api/source.py +29 -5
- datahub/ingestion/api/source_protocols.py +23 -0
- datahub/ingestion/source/bigquery_v2/bigquery_report.py +0 -2
- datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +2 -2
- datahub/ingestion/source/cassandra/cassandra_profiling.py +2 -2
- datahub/ingestion/source/cassandra/cassandra_utils.py +1 -2
- datahub/ingestion/source/dremio/dremio_reporting.py +0 -2
- datahub/ingestion/source/dremio/dremio_source.py +2 -2
- datahub/ingestion/source/fivetran/config.py +30 -5
- datahub/ingestion/source/fivetran/fivetran.py +0 -1
- datahub/ingestion/source/fivetran/fivetran_log_api.py +13 -0
- datahub/ingestion/source/fivetran/fivetran_query.py +43 -28
- datahub/ingestion/source/gc/datahub_gc.py +0 -2
- datahub/ingestion/source/grafana/models.py +9 -1
- datahub/ingestion/source/grafana/report.py +1 -2
- datahub/ingestion/source/hex/hex.py +0 -2
- datahub/ingestion/source/redshift/redshift.py +2 -2
- datahub/ingestion/source/redshift/report.py +0 -2
- datahub/ingestion/source/snowflake/snowflake_report.py +0 -2
- datahub/ingestion/source/snowflake/snowflake_schema_gen.py +2 -2
- datahub/ingestion/source/sql/oracle.py +1 -1
- datahub/ingestion/source/sql/sql_common.py +25 -17
- datahub/ingestion/source/sql/teradata.py +1 -2
- datahub/ingestion/source/sql_queries.py +1 -2
- datahub/ingestion/source/tableau/tableau.py +0 -2
- datahub/ingestion/source/unity/config.py +11 -42
- datahub/ingestion/source/unity/connection.py +61 -0
- datahub/ingestion/source/unity/report.py +1 -2
- datahub/ingestion/source_report/ingestion_stage.py +54 -12
- datahub/metadata/_internal_schema_classes.py +169 -0
- datahub/metadata/com/linkedin/pegasus2avro/platform/event/v1/__init__.py +4 -0
- datahub/metadata/schema.avsc +101 -0
- datahub/metadata/schemas/RelationshipChangeEvent.avsc +215 -0
- datahub/metadata/schemas/StructuredPropertySettings.avsc +9 -0
- {acryl_datahub-1.2.0.11rc4.dist-info → acryl_datahub-1.3.0.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.2.0.11rc4.dist-info → acryl_datahub-1.3.0.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.2.0.11rc4.dist-info → acryl_datahub-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.2.0.11rc4.dist-info → acryl_datahub-1.3.0.dist-info}/top_level.txt +0 -0
datahub/metadata/schema.avsc
CHANGED
|
@@ -14350,6 +14350,98 @@
|
|
|
14350
14350
|
],
|
|
14351
14351
|
"doc": "Shared fields for all entity change events."
|
|
14352
14352
|
},
|
|
14353
|
+
{
|
|
14354
|
+
"type": "record",
|
|
14355
|
+
"Event": {
|
|
14356
|
+
"name": "relationshipChangeEvent"
|
|
14357
|
+
},
|
|
14358
|
+
"name": "RelationshipChangeEvent",
|
|
14359
|
+
"namespace": "com.linkedin.pegasus2avro.platform.event.v1",
|
|
14360
|
+
"fields": [
|
|
14361
|
+
{
|
|
14362
|
+
"type": [
|
|
14363
|
+
"null",
|
|
14364
|
+
"com.linkedin.events.KafkaAuditHeader"
|
|
14365
|
+
],
|
|
14366
|
+
"name": "auditHeader",
|
|
14367
|
+
"default": null,
|
|
14368
|
+
"doc": "Kafka audit header containing metadata about the message itself.\nIncludes information like message ID, timestamp, and server details."
|
|
14369
|
+
},
|
|
14370
|
+
{
|
|
14371
|
+
"java": {
|
|
14372
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
14373
|
+
},
|
|
14374
|
+
"Urn": "Urn",
|
|
14375
|
+
"type": "string",
|
|
14376
|
+
"name": "sourceUrn",
|
|
14377
|
+
"doc": "The URN (Uniform Resource Name) of the source entity in the relationship.\nIn a downstream relationship example, this would be the URN of the upstream dataset."
|
|
14378
|
+
},
|
|
14379
|
+
{
|
|
14380
|
+
"java": {
|
|
14381
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
14382
|
+
},
|
|
14383
|
+
"Urn": "Urn",
|
|
14384
|
+
"type": "string",
|
|
14385
|
+
"name": "destinationUrn",
|
|
14386
|
+
"doc": "The URN of the destination entity in the relationship.\nIn a downstream relationship example, this would be the URN of the downstream dataset."
|
|
14387
|
+
},
|
|
14388
|
+
{
|
|
14389
|
+
"type": {
|
|
14390
|
+
"type": "enum",
|
|
14391
|
+
"name": "RelationshipChangeOperation",
|
|
14392
|
+
"namespace": "com.linkedin.pegasus2avro.platform.event.v1",
|
|
14393
|
+
"symbols": [
|
|
14394
|
+
"ADD",
|
|
14395
|
+
"REMOVE",
|
|
14396
|
+
"RESTATE"
|
|
14397
|
+
]
|
|
14398
|
+
},
|
|
14399
|
+
"name": "operation",
|
|
14400
|
+
"doc": "The operation being performed on this relationship.\nTypically includes operations like ADD, REMOVE, or RESTATE."
|
|
14401
|
+
},
|
|
14402
|
+
{
|
|
14403
|
+
"type": "string",
|
|
14404
|
+
"name": "relationshipType",
|
|
14405
|
+
"doc": "The type/category of relationship being established or modified.\nExamples: \"DownstreamOf\", \"Contains\", \"OwnedBy\", \"DerivedFrom\", etc."
|
|
14406
|
+
},
|
|
14407
|
+
{
|
|
14408
|
+
"type": [
|
|
14409
|
+
"null",
|
|
14410
|
+
"string"
|
|
14411
|
+
],
|
|
14412
|
+
"name": "lifecycleOwner",
|
|
14413
|
+
"default": null,
|
|
14414
|
+
"doc": "The system or service responsible for managing the lifecycle of this relationship.\nThis helps identify which component has authority over the relationship."
|
|
14415
|
+
},
|
|
14416
|
+
{
|
|
14417
|
+
"type": [
|
|
14418
|
+
"null",
|
|
14419
|
+
"string"
|
|
14420
|
+
],
|
|
14421
|
+
"name": "via",
|
|
14422
|
+
"default": null,
|
|
14423
|
+
"doc": "Information about how or through what means this relationship was established.\nCould indicate a specific pipeline, process, or tool that discovered/created the relationship."
|
|
14424
|
+
},
|
|
14425
|
+
{
|
|
14426
|
+
"type": [
|
|
14427
|
+
"null",
|
|
14428
|
+
{
|
|
14429
|
+
"type": "map",
|
|
14430
|
+
"values": "string"
|
|
14431
|
+
}
|
|
14432
|
+
],
|
|
14433
|
+
"name": "properties",
|
|
14434
|
+
"default": null,
|
|
14435
|
+
"doc": "Additional custom properties associated with this relationship.\nAllows for flexible extension without changing the schema."
|
|
14436
|
+
},
|
|
14437
|
+
{
|
|
14438
|
+
"type": "com.linkedin.pegasus2avro.common.AuditStamp",
|
|
14439
|
+
"name": "auditStamp",
|
|
14440
|
+
"doc": "Stores information about who made this change and when.\nContains the actor (user or system) that performed the action and the timestamp."
|
|
14441
|
+
}
|
|
14442
|
+
],
|
|
14443
|
+
"doc": "Kafka event for proposing a relationship change between two entities.\nFor example, when dataset1 establishes a new downstream relationship with dataset2."
|
|
14444
|
+
},
|
|
14353
14445
|
{
|
|
14354
14446
|
"type": "record",
|
|
14355
14447
|
"name": "Filter",
|
|
@@ -20303,6 +20395,15 @@
|
|
|
20303
20395
|
"default": false,
|
|
20304
20396
|
"doc": "Whether or not this asset should be displayed in the asset sidebar"
|
|
20305
20397
|
},
|
|
20398
|
+
{
|
|
20399
|
+
"Searchable": {
|
|
20400
|
+
"fieldType": "BOOLEAN"
|
|
20401
|
+
},
|
|
20402
|
+
"type": "boolean",
|
|
20403
|
+
"name": "hideInAssetSummaryWhenEmpty",
|
|
20404
|
+
"default": false,
|
|
20405
|
+
"doc": "Whether or not this asset should be hidden in the asset sidebar (showInAssetSummary should be enabled)\nwhen its value is empty"
|
|
20406
|
+
},
|
|
20306
20407
|
{
|
|
20307
20408
|
"Searchable": {
|
|
20308
20409
|
"fieldType": "BOOLEAN"
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "record",
|
|
3
|
+
"Event": {
|
|
4
|
+
"name": "relationshipChangeEvent"
|
|
5
|
+
},
|
|
6
|
+
"name": "RelationshipChangeEvent",
|
|
7
|
+
"namespace": "com.linkedin.pegasus2avro.platform.event.v1",
|
|
8
|
+
"fields": [
|
|
9
|
+
{
|
|
10
|
+
"type": [
|
|
11
|
+
"null",
|
|
12
|
+
{
|
|
13
|
+
"type": "record",
|
|
14
|
+
"name": "KafkaAuditHeader",
|
|
15
|
+
"namespace": "com.linkedin.events",
|
|
16
|
+
"fields": [
|
|
17
|
+
{
|
|
18
|
+
"compliance": [
|
|
19
|
+
{
|
|
20
|
+
"policy": "EVENT_TIME"
|
|
21
|
+
}
|
|
22
|
+
],
|
|
23
|
+
"type": "long",
|
|
24
|
+
"name": "time",
|
|
25
|
+
"doc": "The time at which the event was emitted into kafka."
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"compliance": "NONE",
|
|
29
|
+
"type": "string",
|
|
30
|
+
"name": "server",
|
|
31
|
+
"doc": "The fully qualified name of the host from which the event is being emitted."
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"compliance": "NONE",
|
|
35
|
+
"type": [
|
|
36
|
+
"null",
|
|
37
|
+
"string"
|
|
38
|
+
],
|
|
39
|
+
"name": "instance",
|
|
40
|
+
"default": null,
|
|
41
|
+
"doc": "The instance on the server from which the event is being emitted. e.g. i001"
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"compliance": "NONE",
|
|
45
|
+
"type": "string",
|
|
46
|
+
"name": "appName",
|
|
47
|
+
"doc": "The name of the application from which the event is being emitted. see go/appname"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"compliance": "NONE",
|
|
51
|
+
"type": {
|
|
52
|
+
"type": "fixed",
|
|
53
|
+
"name": "UUID",
|
|
54
|
+
"namespace": "com.linkedin.events",
|
|
55
|
+
"size": 16
|
|
56
|
+
},
|
|
57
|
+
"name": "messageId",
|
|
58
|
+
"doc": "A unique identifier for the message"
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"compliance": "NONE",
|
|
62
|
+
"type": [
|
|
63
|
+
"null",
|
|
64
|
+
"int"
|
|
65
|
+
],
|
|
66
|
+
"name": "auditVersion",
|
|
67
|
+
"default": null,
|
|
68
|
+
"doc": "The version that is being used for auditing. In version 0, the audit trail buckets events into 10 minute audit windows based on the EventHeader timestamp. In version 1, the audit trail buckets events as follows: if the schema has an outer KafkaAuditHeader, use the outer audit header timestamp for bucketing; else if the EventHeader has an inner KafkaAuditHeader use that inner audit header's timestamp for bucketing"
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"compliance": "NONE",
|
|
72
|
+
"type": [
|
|
73
|
+
"null",
|
|
74
|
+
"string"
|
|
75
|
+
],
|
|
76
|
+
"name": "fabricUrn",
|
|
77
|
+
"default": null,
|
|
78
|
+
"doc": "The fabricUrn of the host from which the event is being emitted. Fabric Urn in the format of urn:li:fabric:{fabric_name}. See go/fabric."
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"compliance": "NONE",
|
|
82
|
+
"type": [
|
|
83
|
+
"null",
|
|
84
|
+
"string"
|
|
85
|
+
],
|
|
86
|
+
"name": "clusterConnectionString",
|
|
87
|
+
"default": null,
|
|
88
|
+
"doc": "This is a String that the client uses to establish some kind of connection with the Kafka cluster. The exact format of it depends on specific versions of clients and brokers. This information could potentially identify the fabric and cluster with which the client is producing to or consuming from."
|
|
89
|
+
}
|
|
90
|
+
],
|
|
91
|
+
"doc": "This header records information about the context of an event as it is emitted into kafka and is intended to be used by the kafka audit application. For more information see go/kafkaauditheader"
|
|
92
|
+
}
|
|
93
|
+
],
|
|
94
|
+
"name": "auditHeader",
|
|
95
|
+
"default": null,
|
|
96
|
+
"doc": "Kafka audit header containing metadata about the message itself.\nIncludes information like message ID, timestamp, and server details."
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
"java": {
|
|
100
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
101
|
+
},
|
|
102
|
+
"type": "string",
|
|
103
|
+
"name": "sourceUrn",
|
|
104
|
+
"doc": "The URN (Uniform Resource Name) of the source entity in the relationship.\nIn a downstream relationship example, this would be the URN of the upstream dataset.",
|
|
105
|
+
"Urn": "Urn"
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"java": {
|
|
109
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
110
|
+
},
|
|
111
|
+
"type": "string",
|
|
112
|
+
"name": "destinationUrn",
|
|
113
|
+
"doc": "The URN of the destination entity in the relationship.\nIn a downstream relationship example, this would be the URN of the downstream dataset.",
|
|
114
|
+
"Urn": "Urn"
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"type": {
|
|
118
|
+
"type": "enum",
|
|
119
|
+
"name": "RelationshipChangeOperation",
|
|
120
|
+
"namespace": "com.linkedin.pegasus2avro.platform.event.v1",
|
|
121
|
+
"symbols": [
|
|
122
|
+
"ADD",
|
|
123
|
+
"REMOVE",
|
|
124
|
+
"RESTATE"
|
|
125
|
+
]
|
|
126
|
+
},
|
|
127
|
+
"name": "operation",
|
|
128
|
+
"doc": "The operation being performed on this relationship.\nTypically includes operations like ADD, REMOVE, or RESTATE."
|
|
129
|
+
},
|
|
130
|
+
{
|
|
131
|
+
"type": "string",
|
|
132
|
+
"name": "relationshipType",
|
|
133
|
+
"doc": "The type/category of relationship being established or modified.\nExamples: \"DownstreamOf\", \"Contains\", \"OwnedBy\", \"DerivedFrom\", etc."
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"type": [
|
|
137
|
+
"null",
|
|
138
|
+
"string"
|
|
139
|
+
],
|
|
140
|
+
"name": "lifecycleOwner",
|
|
141
|
+
"default": null,
|
|
142
|
+
"doc": "The system or service responsible for managing the lifecycle of this relationship.\nThis helps identify which component has authority over the relationship."
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
"type": [
|
|
146
|
+
"null",
|
|
147
|
+
"string"
|
|
148
|
+
],
|
|
149
|
+
"name": "via",
|
|
150
|
+
"default": null,
|
|
151
|
+
"doc": "Information about how or through what means this relationship was established.\nCould indicate a specific pipeline, process, or tool that discovered/created the relationship."
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
"type": [
|
|
155
|
+
"null",
|
|
156
|
+
{
|
|
157
|
+
"type": "map",
|
|
158
|
+
"values": "string"
|
|
159
|
+
}
|
|
160
|
+
],
|
|
161
|
+
"name": "properties",
|
|
162
|
+
"default": null,
|
|
163
|
+
"doc": "Additional custom properties associated with this relationship.\nAllows for flexible extension without changing the schema."
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
"type": {
|
|
167
|
+
"type": "record",
|
|
168
|
+
"name": "AuditStamp",
|
|
169
|
+
"namespace": "com.linkedin.pegasus2avro.common",
|
|
170
|
+
"fields": [
|
|
171
|
+
{
|
|
172
|
+
"type": "long",
|
|
173
|
+
"name": "time",
|
|
174
|
+
"doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
"java": {
|
|
178
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
179
|
+
},
|
|
180
|
+
"type": "string",
|
|
181
|
+
"name": "actor",
|
|
182
|
+
"doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
|
|
183
|
+
"Urn": "Urn"
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
"java": {
|
|
187
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
188
|
+
},
|
|
189
|
+
"type": [
|
|
190
|
+
"null",
|
|
191
|
+
"string"
|
|
192
|
+
],
|
|
193
|
+
"name": "impersonator",
|
|
194
|
+
"default": null,
|
|
195
|
+
"doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
|
|
196
|
+
"Urn": "Urn"
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
"type": [
|
|
200
|
+
"null",
|
|
201
|
+
"string"
|
|
202
|
+
],
|
|
203
|
+
"name": "message",
|
|
204
|
+
"default": null,
|
|
205
|
+
"doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
|
|
206
|
+
}
|
|
207
|
+
],
|
|
208
|
+
"doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
|
|
209
|
+
},
|
|
210
|
+
"name": "auditStamp",
|
|
211
|
+
"doc": "Stores information about who made this change and when.\nContains the actor (user or system) that performed the action and the timestamp."
|
|
212
|
+
}
|
|
213
|
+
],
|
|
214
|
+
"doc": "Kafka event for proposing a relationship change between two entities.\nFor example, when dataset1 establishes a new downstream relationship with dataset2."
|
|
215
|
+
}
|
|
@@ -33,6 +33,15 @@
|
|
|
33
33
|
"default": false,
|
|
34
34
|
"doc": "Whether or not this asset should be displayed in the asset sidebar"
|
|
35
35
|
},
|
|
36
|
+
{
|
|
37
|
+
"Searchable": {
|
|
38
|
+
"fieldType": "BOOLEAN"
|
|
39
|
+
},
|
|
40
|
+
"type": "boolean",
|
|
41
|
+
"name": "hideInAssetSummaryWhenEmpty",
|
|
42
|
+
"default": false,
|
|
43
|
+
"doc": "Whether or not this asset should be hidden in the asset sidebar (showInAssetSummary should be enabled)\nwhen its value is empty"
|
|
44
|
+
},
|
|
36
45
|
{
|
|
37
46
|
"Searchable": {
|
|
38
47
|
"fieldType": "BOOLEAN"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|