acryl-datahub-cloud 0.3.7.9rc1__py3-none-any.whl → 0.3.7.9.2rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub-cloud might be problematic. Click here for more details.
- acryl_datahub_cloud/_codegen_config.json +1 -1
- acryl_datahub_cloud/metadata/_urns/urn_defs.py +1465 -1465
- acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/ml/metadata/__init__.py +2 -0
- acryl_datahub_cloud/metadata/schema.avsc +20357 -19714
- acryl_datahub_cloud/metadata/schema_classes.py +710 -504
- acryl_datahub_cloud/metadata/schemas/DataProcessInstanceInput.avsc +4 -2
- acryl_datahub_cloud/metadata/schemas/DataProcessInstanceKey.avsc +4 -0
- acryl_datahub_cloud/metadata/schemas/DataProcessInstanceOutput.avsc +4 -1
- acryl_datahub_cloud/metadata/schemas/MLModelGroupProperties.avsc +104 -0
- acryl_datahub_cloud/metadata/schemas/MLModelProperties.avsc +104 -47
- acryl_datahub_cloud/metadata/schemas/MLTrainingRunProperties.avsc +171 -0
- acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc +158 -47
- {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.7.9.2rc1.dist-info}/METADATA +42 -40
- {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.7.9.2rc1.dist-info}/RECORD +17 -16
- {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.7.9.2rc1.dist-info}/WHEEL +1 -1
- {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.7.9.2rc1.dist-info}/entry_points.txt +0 -0
- {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.7.9.2rc1.dist-info}/top_level.txt +0 -0
|
@@ -10,8 +10,10 @@
|
|
|
10
10
|
"Relationship": {
|
|
11
11
|
"/*": {
|
|
12
12
|
"entityTypes": [
|
|
13
|
-
"dataset"
|
|
13
|
+
"dataset",
|
|
14
|
+
"mlModel"
|
|
14
15
|
],
|
|
16
|
+
"isLineage": true,
|
|
15
17
|
"name": "Consumes"
|
|
16
18
|
}
|
|
17
19
|
},
|
|
@@ -29,7 +31,7 @@
|
|
|
29
31
|
"items": "string"
|
|
30
32
|
},
|
|
31
33
|
"name": "inputs",
|
|
32
|
-
"doc": "Input
|
|
34
|
+
"doc": "Input assets consumed",
|
|
33
35
|
"Urn": "Urn",
|
|
34
36
|
"urn_is_array": true
|
|
35
37
|
}
|
|
@@ -12,6 +12,10 @@
|
|
|
12
12
|
"dataProcessInstanceRunEvent",
|
|
13
13
|
"status",
|
|
14
14
|
"testResults",
|
|
15
|
+
"dataPlatformInstance",
|
|
16
|
+
"subTypes",
|
|
17
|
+
"container",
|
|
18
|
+
"mlTrainingRunProperties",
|
|
15
19
|
"lineageFeatures"
|
|
16
20
|
],
|
|
17
21
|
"entityDoc": "DataProcessInstance represents an instance of a datajob/jobflow run"
|
|
@@ -21,6 +21,66 @@
|
|
|
21
21
|
"default": {},
|
|
22
22
|
"doc": "Custom property bag."
|
|
23
23
|
},
|
|
24
|
+
{
|
|
25
|
+
"Relationship": {
|
|
26
|
+
"/*": {
|
|
27
|
+
"entityTypes": [
|
|
28
|
+
"dataJob",
|
|
29
|
+
"dataProcessInstance"
|
|
30
|
+
],
|
|
31
|
+
"isLineage": true,
|
|
32
|
+
"name": "TrainedBy"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"type": [
|
|
36
|
+
"null",
|
|
37
|
+
{
|
|
38
|
+
"type": "array",
|
|
39
|
+
"items": "string"
|
|
40
|
+
}
|
|
41
|
+
],
|
|
42
|
+
"name": "trainingJobs",
|
|
43
|
+
"default": null,
|
|
44
|
+
"doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"Relationship": {
|
|
48
|
+
"/*": {
|
|
49
|
+
"entityTypes": [
|
|
50
|
+
"dataJob",
|
|
51
|
+
"dataProcessInstance"
|
|
52
|
+
],
|
|
53
|
+
"isLineage": true,
|
|
54
|
+
"isUpstream": false,
|
|
55
|
+
"name": "UsedBy"
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"type": [
|
|
59
|
+
"null",
|
|
60
|
+
{
|
|
61
|
+
"type": "array",
|
|
62
|
+
"items": "string"
|
|
63
|
+
}
|
|
64
|
+
],
|
|
65
|
+
"name": "downstreamJobs",
|
|
66
|
+
"default": null,
|
|
67
|
+
"doc": "List of jobs or process instances (if any) that use the model or group."
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"Searchable": {
|
|
71
|
+
"boostScore": 10.0,
|
|
72
|
+
"enableAutocomplete": true,
|
|
73
|
+
"fieldType": "WORD_GRAM",
|
|
74
|
+
"queryByDefault": true
|
|
75
|
+
},
|
|
76
|
+
"type": [
|
|
77
|
+
"null",
|
|
78
|
+
"string"
|
|
79
|
+
],
|
|
80
|
+
"name": "name",
|
|
81
|
+
"default": null,
|
|
82
|
+
"doc": "Display name of the MLModelGroup"
|
|
83
|
+
},
|
|
24
84
|
{
|
|
25
85
|
"Searchable": {
|
|
26
86
|
"fieldType": "TEXT",
|
|
@@ -35,6 +95,7 @@
|
|
|
35
95
|
"doc": "Documentation of the MLModelGroup"
|
|
36
96
|
},
|
|
37
97
|
{
|
|
98
|
+
"deprecated": true,
|
|
38
99
|
"type": [
|
|
39
100
|
"null",
|
|
40
101
|
"long"
|
|
@@ -43,6 +104,49 @@
|
|
|
43
104
|
"default": null,
|
|
44
105
|
"doc": "Date when the MLModelGroup was developed"
|
|
45
106
|
},
|
|
107
|
+
{
|
|
108
|
+
"type": [
|
|
109
|
+
"null",
|
|
110
|
+
{
|
|
111
|
+
"type": "record",
|
|
112
|
+
"name": "TimeStamp",
|
|
113
|
+
"namespace": "com.linkedin.pegasus2avro.common",
|
|
114
|
+
"fields": [
|
|
115
|
+
{
|
|
116
|
+
"type": "long",
|
|
117
|
+
"name": "time",
|
|
118
|
+
"doc": "When did the event occur"
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"java": {
|
|
122
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
123
|
+
},
|
|
124
|
+
"type": [
|
|
125
|
+
"null",
|
|
126
|
+
"string"
|
|
127
|
+
],
|
|
128
|
+
"name": "actor",
|
|
129
|
+
"default": null,
|
|
130
|
+
"doc": "Optional: The actor urn involved in the event.",
|
|
131
|
+
"Urn": "Urn"
|
|
132
|
+
}
|
|
133
|
+
],
|
|
134
|
+
"doc": "A standard event timestamp"
|
|
135
|
+
}
|
|
136
|
+
],
|
|
137
|
+
"name": "created",
|
|
138
|
+
"default": null,
|
|
139
|
+
"doc": "Time and Actor who created the MLModelGroup"
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
"type": [
|
|
143
|
+
"null",
|
|
144
|
+
"com.linkedin.pegasus2avro.common.TimeStamp"
|
|
145
|
+
],
|
|
146
|
+
"name": "lastModified",
|
|
147
|
+
"default": null,
|
|
148
|
+
"doc": "Date when the MLModelGroup was last modified"
|
|
149
|
+
},
|
|
46
150
|
{
|
|
47
151
|
"type": [
|
|
48
152
|
"null",
|
|
@@ -37,6 +37,66 @@
|
|
|
37
37
|
"default": null,
|
|
38
38
|
"doc": "URL where the reference exist"
|
|
39
39
|
},
|
|
40
|
+
{
|
|
41
|
+
"Relationship": {
|
|
42
|
+
"/*": {
|
|
43
|
+
"entityTypes": [
|
|
44
|
+
"dataJob",
|
|
45
|
+
"dataProcessInstance"
|
|
46
|
+
],
|
|
47
|
+
"isLineage": true,
|
|
48
|
+
"name": "TrainedBy"
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
"type": [
|
|
52
|
+
"null",
|
|
53
|
+
{
|
|
54
|
+
"type": "array",
|
|
55
|
+
"items": "string"
|
|
56
|
+
}
|
|
57
|
+
],
|
|
58
|
+
"name": "trainingJobs",
|
|
59
|
+
"default": null,
|
|
60
|
+
"doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
"Relationship": {
|
|
64
|
+
"/*": {
|
|
65
|
+
"entityTypes": [
|
|
66
|
+
"dataJob",
|
|
67
|
+
"dataProcessInstance"
|
|
68
|
+
],
|
|
69
|
+
"isLineage": true,
|
|
70
|
+
"isUpstream": false,
|
|
71
|
+
"name": "UsedBy"
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
"type": [
|
|
75
|
+
"null",
|
|
76
|
+
{
|
|
77
|
+
"type": "array",
|
|
78
|
+
"items": "string"
|
|
79
|
+
}
|
|
80
|
+
],
|
|
81
|
+
"name": "downstreamJobs",
|
|
82
|
+
"default": null,
|
|
83
|
+
"doc": "List of jobs or process instances (if any) that use the model or group."
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"Searchable": {
|
|
87
|
+
"boostScore": 10.0,
|
|
88
|
+
"enableAutocomplete": true,
|
|
89
|
+
"fieldType": "WORD_GRAM",
|
|
90
|
+
"queryByDefault": true
|
|
91
|
+
},
|
|
92
|
+
"type": [
|
|
93
|
+
"null",
|
|
94
|
+
"string"
|
|
95
|
+
],
|
|
96
|
+
"name": "name",
|
|
97
|
+
"default": null,
|
|
98
|
+
"doc": "Display name of the MLModel"
|
|
99
|
+
},
|
|
40
100
|
{
|
|
41
101
|
"Searchable": {
|
|
42
102
|
"fieldType": "TEXT",
|
|
@@ -51,6 +111,7 @@
|
|
|
51
111
|
"doc": "Documentation of the MLModel"
|
|
52
112
|
},
|
|
53
113
|
{
|
|
114
|
+
"deprecated": true,
|
|
54
115
|
"type": [
|
|
55
116
|
"null",
|
|
56
117
|
"long"
|
|
@@ -59,6 +120,49 @@
|
|
|
59
120
|
"default": null,
|
|
60
121
|
"doc": "Date when the MLModel was developed"
|
|
61
122
|
},
|
|
123
|
+
{
|
|
124
|
+
"type": [
|
|
125
|
+
"null",
|
|
126
|
+
{
|
|
127
|
+
"type": "record",
|
|
128
|
+
"name": "TimeStamp",
|
|
129
|
+
"namespace": "com.linkedin.pegasus2avro.common",
|
|
130
|
+
"fields": [
|
|
131
|
+
{
|
|
132
|
+
"type": "long",
|
|
133
|
+
"name": "time",
|
|
134
|
+
"doc": "When did the event occur"
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"java": {
|
|
138
|
+
"class": "com.linkedin.pegasus2avro.common.urn.Urn"
|
|
139
|
+
},
|
|
140
|
+
"type": [
|
|
141
|
+
"null",
|
|
142
|
+
"string"
|
|
143
|
+
],
|
|
144
|
+
"name": "actor",
|
|
145
|
+
"default": null,
|
|
146
|
+
"doc": "Optional: The actor urn involved in the event.",
|
|
147
|
+
"Urn": "Urn"
|
|
148
|
+
}
|
|
149
|
+
],
|
|
150
|
+
"doc": "A standard event timestamp"
|
|
151
|
+
}
|
|
152
|
+
],
|
|
153
|
+
"name": "created",
|
|
154
|
+
"default": null,
|
|
155
|
+
"doc": "Audit stamp containing who created this and when"
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
"type": [
|
|
159
|
+
"null",
|
|
160
|
+
"com.linkedin.pegasus2avro.common.TimeStamp"
|
|
161
|
+
],
|
|
162
|
+
"name": "lastModified",
|
|
163
|
+
"default": null,
|
|
164
|
+
"doc": "Date when the MLModel was last modified"
|
|
165
|
+
},
|
|
62
166
|
{
|
|
63
167
|
"type": [
|
|
64
168
|
"null",
|
|
@@ -287,53 +391,6 @@
|
|
|
287
391
|
"Urn": "Urn",
|
|
288
392
|
"urn_is_array": true
|
|
289
393
|
},
|
|
290
|
-
{
|
|
291
|
-
"Relationship": {
|
|
292
|
-
"/*": {
|
|
293
|
-
"entityTypes": [
|
|
294
|
-
"dataJob"
|
|
295
|
-
],
|
|
296
|
-
"isLineage": true,
|
|
297
|
-
"name": "TrainedBy"
|
|
298
|
-
}
|
|
299
|
-
},
|
|
300
|
-
"type": [
|
|
301
|
-
"null",
|
|
302
|
-
{
|
|
303
|
-
"type": "array",
|
|
304
|
-
"items": "string"
|
|
305
|
-
}
|
|
306
|
-
],
|
|
307
|
-
"name": "trainingJobs",
|
|
308
|
-
"default": null,
|
|
309
|
-
"doc": "List of jobs (if any) used to train the model",
|
|
310
|
-
"Urn": "Urn",
|
|
311
|
-
"urn_is_array": true
|
|
312
|
-
},
|
|
313
|
-
{
|
|
314
|
-
"Relationship": {
|
|
315
|
-
"/*": {
|
|
316
|
-
"entityTypes": [
|
|
317
|
-
"dataJob"
|
|
318
|
-
],
|
|
319
|
-
"isLineage": true,
|
|
320
|
-
"isUpstream": false,
|
|
321
|
-
"name": "UsedBy"
|
|
322
|
-
}
|
|
323
|
-
},
|
|
324
|
-
"type": [
|
|
325
|
-
"null",
|
|
326
|
-
{
|
|
327
|
-
"type": "array",
|
|
328
|
-
"items": "string"
|
|
329
|
-
}
|
|
330
|
-
],
|
|
331
|
-
"name": "downstreamJobs",
|
|
332
|
-
"default": null,
|
|
333
|
-
"doc": "List of jobs (if any) that use the model",
|
|
334
|
-
"Urn": "Urn",
|
|
335
|
-
"urn_is_array": true
|
|
336
|
-
},
|
|
337
394
|
{
|
|
338
395
|
"Relationship": {
|
|
339
396
|
"/*": {
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "record",
|
|
3
|
+
"Aspect": {
|
|
4
|
+
"name": "mlTrainingRunProperties"
|
|
5
|
+
},
|
|
6
|
+
"name": "MLTrainingRunProperties",
|
|
7
|
+
"namespace": "com.linkedin.pegasus2avro.ml.metadata",
|
|
8
|
+
"fields": [
|
|
9
|
+
{
|
|
10
|
+
"Searchable": {
|
|
11
|
+
"/*": {
|
|
12
|
+
"fieldType": "TEXT",
|
|
13
|
+
"queryByDefault": true
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"type": {
|
|
17
|
+
"type": "map",
|
|
18
|
+
"values": "string"
|
|
19
|
+
},
|
|
20
|
+
"name": "customProperties",
|
|
21
|
+
"default": {},
|
|
22
|
+
"doc": "Custom property bag."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"Searchable": {
|
|
26
|
+
"fieldType": "KEYWORD"
|
|
27
|
+
},
|
|
28
|
+
"java": {
|
|
29
|
+
"class": "com.linkedin.pegasus2avro.common.url.Url",
|
|
30
|
+
"coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
|
|
31
|
+
},
|
|
32
|
+
"type": [
|
|
33
|
+
"null",
|
|
34
|
+
"string"
|
|
35
|
+
],
|
|
36
|
+
"name": "externalUrl",
|
|
37
|
+
"default": null,
|
|
38
|
+
"doc": "URL where the reference exist"
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"type": [
|
|
42
|
+
"null",
|
|
43
|
+
"string"
|
|
44
|
+
],
|
|
45
|
+
"name": "id",
|
|
46
|
+
"default": null,
|
|
47
|
+
"doc": "Run Id of the ML Training Run"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"type": [
|
|
51
|
+
"null",
|
|
52
|
+
{
|
|
53
|
+
"type": "array",
|
|
54
|
+
"items": "string"
|
|
55
|
+
}
|
|
56
|
+
],
|
|
57
|
+
"name": "outputUrls",
|
|
58
|
+
"default": null,
|
|
59
|
+
"doc": "List of URLs for the Outputs of the ML Training Run"
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"type": [
|
|
63
|
+
"null",
|
|
64
|
+
{
|
|
65
|
+
"type": "array",
|
|
66
|
+
"items": {
|
|
67
|
+
"type": "record",
|
|
68
|
+
"Aspect": {
|
|
69
|
+
"name": "mlHyperParam"
|
|
70
|
+
},
|
|
71
|
+
"name": "MLHyperParam",
|
|
72
|
+
"namespace": "com.linkedin.pegasus2avro.ml.metadata",
|
|
73
|
+
"fields": [
|
|
74
|
+
{
|
|
75
|
+
"type": "string",
|
|
76
|
+
"name": "name",
|
|
77
|
+
"doc": "Name of the MLHyperParam"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"type": [
|
|
81
|
+
"null",
|
|
82
|
+
"string"
|
|
83
|
+
],
|
|
84
|
+
"name": "description",
|
|
85
|
+
"default": null,
|
|
86
|
+
"doc": "Documentation of the MLHyperParam"
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"type": [
|
|
90
|
+
"null",
|
|
91
|
+
"string"
|
|
92
|
+
],
|
|
93
|
+
"name": "value",
|
|
94
|
+
"default": null,
|
|
95
|
+
"doc": "The value of the MLHyperParam"
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"type": [
|
|
99
|
+
"null",
|
|
100
|
+
"long"
|
|
101
|
+
],
|
|
102
|
+
"name": "createdAt",
|
|
103
|
+
"default": null,
|
|
104
|
+
"doc": "Date when the MLHyperParam was developed"
|
|
105
|
+
}
|
|
106
|
+
],
|
|
107
|
+
"doc": "Properties associated with an ML Hyper Param"
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
],
|
|
111
|
+
"name": "hyperParams",
|
|
112
|
+
"default": null,
|
|
113
|
+
"doc": "Hyperparameters of the ML Training Run"
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"type": [
|
|
117
|
+
"null",
|
|
118
|
+
{
|
|
119
|
+
"type": "array",
|
|
120
|
+
"items": {
|
|
121
|
+
"type": "record",
|
|
122
|
+
"Aspect": {
|
|
123
|
+
"name": "mlMetric"
|
|
124
|
+
},
|
|
125
|
+
"name": "MLMetric",
|
|
126
|
+
"namespace": "com.linkedin.pegasus2avro.ml.metadata",
|
|
127
|
+
"fields": [
|
|
128
|
+
{
|
|
129
|
+
"type": "string",
|
|
130
|
+
"name": "name",
|
|
131
|
+
"doc": "Name of the mlMetric"
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
"type": [
|
|
135
|
+
"null",
|
|
136
|
+
"string"
|
|
137
|
+
],
|
|
138
|
+
"name": "description",
|
|
139
|
+
"default": null,
|
|
140
|
+
"doc": "Documentation of the mlMetric"
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"type": [
|
|
144
|
+
"null",
|
|
145
|
+
"string"
|
|
146
|
+
],
|
|
147
|
+
"name": "value",
|
|
148
|
+
"default": null,
|
|
149
|
+
"doc": "The value of the mlMetric"
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
"type": [
|
|
153
|
+
"null",
|
|
154
|
+
"long"
|
|
155
|
+
],
|
|
156
|
+
"name": "createdAt",
|
|
157
|
+
"default": null,
|
|
158
|
+
"doc": "Date when the mlMetric was developed"
|
|
159
|
+
}
|
|
160
|
+
],
|
|
161
|
+
"doc": "Properties associated with an ML Metric"
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
],
|
|
165
|
+
"name": "trainingMetrics",
|
|
166
|
+
"default": null,
|
|
167
|
+
"doc": "Metrics of the ML Training Run"
|
|
168
|
+
}
|
|
169
|
+
],
|
|
170
|
+
"doc": "The inputs and outputs of this training run"
|
|
171
|
+
}
|