deriva-ml 1.8.11__py3-none-any.whl → 1.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deriva_ml/database_model.py +29 -7
- deriva_ml/dataset.py +16 -13
- deriva_ml/dataset_bag.py +1 -1
- deriva_ml/demo_catalog.py +9 -8
- deriva_ml/deriva_definitions.py +8 -3
- deriva_ml/deriva_ml_base.py +62 -23
- deriva_ml/deriva_model.py +2 -2
- deriva_ml/execution.py +5 -4
- deriva_ml/execution_configuration.py +20 -23
- deriva_ml/schema_setup/annotations.py +1 -1
- deriva_ml/schema_setup/create_schema.py +3 -2
- deriva_ml/upload.py +1 -1
- {deriva_ml-1.8.11.dist-info → deriva_ml-1.9.1.dist-info}/METADATA +11 -2
- deriva_ml-1.9.1.dist-info/RECORD +27 -0
- deriva_ml/build/lib/schema_setup/__init__.py +0 -0
- deriva_ml/build/lib/schema_setup/alter_annotation.py +0 -36
- deriva_ml/build/lib/schema_setup/annotation_temp.py +0 -255
- deriva_ml/build/lib/schema_setup/create_schema.py +0 -165
- deriva_ml/build/lib/schema_setup/table_comments_utils.py +0 -56
- deriva_ml/schema_setup/alter_annotation.py +0 -55
- deriva_ml-1.8.11.dist-info/RECORD +0 -33
- {deriva_ml-1.8.11.dist-info → deriva_ml-1.9.1.dist-info}/WHEEL +0 -0
- {deriva_ml-1.8.11.dist-info → deriva_ml-1.9.1.dist-info}/entry_points.txt +0 -0
- {deriva_ml-1.8.11.dist-info → deriva_ml-1.9.1.dist-info}/licenses/LICENSE +0 -0
- {deriva_ml-1.8.11.dist-info → deriva_ml-1.9.1.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deriva-ml
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.9.1
|
|
4
4
|
Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
|
|
5
5
|
Author-email: ISRD <isrd-dev@isi.edu>
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -16,5 +16,14 @@ Requires-Dist: setuptools-scm<=6.0
|
|
|
16
16
|
Requires-Dist: nbstripout
|
|
17
17
|
Dynamic: license-file
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
# DerivaML
|
|
20
|
+
Deriva-ML is a python library to simplify the process of creating and executing reproducible machine learning workflows
|
|
20
21
|
using a deriva catalog.
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
## Installing the GitHub CLI
|
|
25
|
+
|
|
26
|
+
The script release.sh will create a new release tag in GitHub. This script requires the
|
|
27
|
+
GitHUB CLI be installed.
|
|
28
|
+
|
|
29
|
+
See [https://cli.github.com](https://cli.github.com) for instructions on how to install and configure the CLI.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
deriva_ml/__init__.py,sha256=r1Z9N5vtZkAET7emqhpAx2bf_xJUp5wHOc4_DIplsG8,1082
|
|
2
|
+
deriva_ml/database_model.py,sha256=58iweWRteLeKKjjeNA9_e7TbUb4Av92lxH2zKvZzwA8,14823
|
|
3
|
+
deriva_ml/dataset.py,sha256=h7Zkhnhy66GhPg6O1ud-YCx-jFKAabWF-nwuIDsR8SU,60785
|
|
4
|
+
deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
|
|
5
|
+
deriva_ml/dataset_bag.py,sha256=aOJxFA9t5apjE5BNBrk8Pi9R1Cp8AWnnaL-10P8ELrQ,11515
|
|
6
|
+
deriva_ml/demo_catalog.py,sha256=zQAHWSvrVPxMg-vyRUqoC0Jj5RhfGjkBwXW3mBksLhA,10986
|
|
7
|
+
deriva_ml/deriva_definitions.py,sha256=jNiInYA2Cb1GE4OOT1CofxBygdLDSOmNsw5Wl6NbZQE,8943
|
|
8
|
+
deriva_ml/deriva_ml_base.py,sha256=nzPzn_iLQIUJDCxTdRgAVEWqS7LbRTZriofWYmdEYe8,46975
|
|
9
|
+
deriva_ml/deriva_model.py,sha256=B4gwr3-92IQU-mEZlusgNEnRyulD96esWGS67q9MzHk,12024
|
|
10
|
+
deriva_ml/execution.py,sha256=on8hAtuZr9qFiyxuk_vDCmnRJ9Cv4kFOgHK4HY4CmV8,29585
|
|
11
|
+
deriva_ml/execution_configuration.py,sha256=vsdL31J09dz7CQDd2rYXIjyBPwNlgAWvrTqsXNWi82g,3357
|
|
12
|
+
deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
|
|
13
|
+
deriva_ml/feature.py,sha256=7e8WYPCfJSrGxJh9oUTduYSnB5ekybRhXa_0HIigS_w,5459
|
|
14
|
+
deriva_ml/history.py,sha256=qTDLDs8Ow_6r7mDO0gZm0Fg81SWKOAgtCU5pzZoDRgM,2828
|
|
15
|
+
deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
|
|
16
|
+
deriva_ml/upload.py,sha256=P35ViZzlNNbsXVbnTyq-G781nGHbX4md1tiCp3c_KKI,22264
|
|
17
|
+
deriva_ml/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
+
deriva_ml/schema_setup/annotations.py,sha256=v0gTpmWYxRqsQ-bcnQzsr8WowGv2pi9pZUsO3WWnu1U,9528
|
|
19
|
+
deriva_ml/schema_setup/create_schema.py,sha256=BRdYeWW5I8HxuATkB1hkKuIw4n-JQu620xod7EQoVSE,10674
|
|
20
|
+
deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
|
|
21
|
+
deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
|
|
22
|
+
deriva_ml-1.9.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
23
|
+
deriva_ml-1.9.1.dist-info/METADATA,sha256=zdDl9mmw2-DwvfYyWtq3vKnsB175gRFUsfHvRNLOGLg,941
|
|
24
|
+
deriva_ml-1.9.1.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
|
|
25
|
+
deriva_ml-1.9.1.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
|
|
26
|
+
deriva_ml-1.9.1.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
|
|
27
|
+
deriva_ml-1.9.1.dist-info/RECORD,,
|
|
File without changes
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
import argparse
|
|
3
|
-
from deriva.core import DerivaServer, get_credential, ErmrestCatalog
|
|
4
|
-
from deriva.core.ermrest_model import builtin_types, Schema, Table, Column, ForeignKey
|
|
5
|
-
from annotation_temp import generate_annotation
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def alter_table_annotation(catalog, schema_name: str, table_name: str, annotation: dict):
|
|
9
|
-
model_root = catalog.getCatalogModel()
|
|
10
|
-
table = model_root.dataset_table(schema_name, table_name)
|
|
11
|
-
table.alter(annotations=annotation)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def alter_schema_annotation(catalog, schema_name: str, annotation: dict):
|
|
15
|
-
model_root = catalog.getCatalogModel()
|
|
16
|
-
schema = model_root.schemas[schema_name]
|
|
17
|
-
schema.alter(annotations=annotation)
|
|
18
|
-
|
|
19
|
-
def main():
|
|
20
|
-
parser = argparse.ArgumentParser()
|
|
21
|
-
parser.add_argument('--hostname', type=str, required=True)
|
|
22
|
-
parser.add_argument('--schema_name', type=str, required=True)
|
|
23
|
-
parser.add_argument('--catalog_id', type=str, required=True)
|
|
24
|
-
args = parser.parse_args()
|
|
25
|
-
credentials = get_credential(args.hostname)
|
|
26
|
-
catalog = ErmrestCatalog('https', args.hostname, args.catalog_id, credentials)
|
|
27
|
-
|
|
28
|
-
annotations = generate_annotation(args.schema_name, args.catalog_id)
|
|
29
|
-
alter_table_annotation(catalog, args.schema_name, 'Workflow', annotations["workflow_annotation"])
|
|
30
|
-
alter_table_annotation(catalog, args.schema_name, 'Execution', annotations["execution_annotation"])
|
|
31
|
-
alter_table_annotation(catalog, args.schema_name, 'Execution_Metadata', annotations["execution_metadata_annotation"])
|
|
32
|
-
alter_table_annotation(catalog, args.schema_name, 'Execution_Assets', annotations["execution_assets_annotation"])
|
|
33
|
-
alter_schema_annotation(catalog, args.schema_name, annotations["schema_annotation"])
|
|
34
|
-
|
|
35
|
-
if __name__ == "__main__":
|
|
36
|
-
sys.exit(main())
|
|
@@ -1,255 +0,0 @@
|
|
|
1
|
-
import argparse
|
|
2
|
-
import sys
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
def generate_annotation(catalog_id: str, schema: str) -> dict:
|
|
6
|
-
workflow_annotation = {
|
|
7
|
-
"tag:isrd.isi.edu,2016:visible-columns": {
|
|
8
|
-
"*": [
|
|
9
|
-
"RID",
|
|
10
|
-
"Name",
|
|
11
|
-
"Description",
|
|
12
|
-
{
|
|
13
|
-
"display": {"markdown_pattern": "[{{{URL}}}]({{{URL}}})"},
|
|
14
|
-
"markdown_name": "URL"
|
|
15
|
-
},
|
|
16
|
-
"Checksum",
|
|
17
|
-
"Version",
|
|
18
|
-
{
|
|
19
|
-
"source": [
|
|
20
|
-
{
|
|
21
|
-
"outbound": [
|
|
22
|
-
schema,
|
|
23
|
-
"Workflow_Workflow_Type_fkey"
|
|
24
|
-
]
|
|
25
|
-
},
|
|
26
|
-
"RID"
|
|
27
|
-
]
|
|
28
|
-
}
|
|
29
|
-
]
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
execution_annotation = {
|
|
34
|
-
"tag:isrd.isi.edu,2016:visible-columns": {
|
|
35
|
-
"*": [
|
|
36
|
-
"RID",
|
|
37
|
-
[
|
|
38
|
-
schema,
|
|
39
|
-
"Execution_RCB_fkey"
|
|
40
|
-
],
|
|
41
|
-
"RCT",
|
|
42
|
-
"Description",
|
|
43
|
-
{"source": [
|
|
44
|
-
{"outbound": [
|
|
45
|
-
"eye-ai",
|
|
46
|
-
"Execution_Workflow_fkey"
|
|
47
|
-
]
|
|
48
|
-
},
|
|
49
|
-
"RID"
|
|
50
|
-
]
|
|
51
|
-
},
|
|
52
|
-
"Duration",
|
|
53
|
-
"Status",
|
|
54
|
-
"Status_Detail"
|
|
55
|
-
]
|
|
56
|
-
},
|
|
57
|
-
"tag:isrd.isi.edu,2016:visible-foreign-keys": {
|
|
58
|
-
"detailed": [
|
|
59
|
-
{
|
|
60
|
-
"source": [
|
|
61
|
-
{
|
|
62
|
-
"inbound": [
|
|
63
|
-
schema,
|
|
64
|
-
"Dataset_Execution_Execution_fkey"
|
|
65
|
-
]
|
|
66
|
-
},
|
|
67
|
-
{
|
|
68
|
-
"outbound": [
|
|
69
|
-
schema,
|
|
70
|
-
"Dataset_Execution_Dataset_fkey"
|
|
71
|
-
]
|
|
72
|
-
},
|
|
73
|
-
"RID"
|
|
74
|
-
],
|
|
75
|
-
"markdown_name": "Dataset"
|
|
76
|
-
},
|
|
77
|
-
{
|
|
78
|
-
"source": [
|
|
79
|
-
{
|
|
80
|
-
"inbound": [
|
|
81
|
-
schema,
|
|
82
|
-
"Execution_Assets_Execution_Execution_fkey"
|
|
83
|
-
]
|
|
84
|
-
},
|
|
85
|
-
{
|
|
86
|
-
"outbound": [
|
|
87
|
-
schema,
|
|
88
|
-
"Execution_Assets_Execution_Execution_Assets_fkey"
|
|
89
|
-
]
|
|
90
|
-
},
|
|
91
|
-
"RID"
|
|
92
|
-
],
|
|
93
|
-
"markdown_name": "Execution Assets"
|
|
94
|
-
},
|
|
95
|
-
{
|
|
96
|
-
"source": [
|
|
97
|
-
{
|
|
98
|
-
"inbound": [
|
|
99
|
-
schema,
|
|
100
|
-
"Execution_Metadata_Execution_fkey"
|
|
101
|
-
]
|
|
102
|
-
},
|
|
103
|
-
"RID"
|
|
104
|
-
],
|
|
105
|
-
"markdown_name": "Execution Metadata"
|
|
106
|
-
}
|
|
107
|
-
]
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
execution_assets_annotation = {
|
|
112
|
-
"tag:isrd.isi.edu,2016:table-display": {
|
|
113
|
-
"row_name": {
|
|
114
|
-
"row_markdown_pattern": "{{{Filename}}}"
|
|
115
|
-
}
|
|
116
|
-
},
|
|
117
|
-
"tag:isrd.isi.edu,2016:visible-columns": {
|
|
118
|
-
"compact": [
|
|
119
|
-
"RID",
|
|
120
|
-
"URL",
|
|
121
|
-
"Description",
|
|
122
|
-
"Length",
|
|
123
|
-
[
|
|
124
|
-
schema,
|
|
125
|
-
"Execution_Assets_Execution_Asset_Type_fkey"
|
|
126
|
-
],
|
|
127
|
-
# {
|
|
128
|
-
# "display": {
|
|
129
|
-
# "template_engine": "handlebars",
|
|
130
|
-
# "markdown_pattern": "{{#if (eq _Execution_Asset_Type \"2-5QME\")}}\n ::: iframe []("
|
|
131
|
-
# "https://dev.eye-ai.org/~vivi/deriva-webapps/plot/?config=test-line"
|
|
132
|
-
# "-plot&Execution_Assets_RID={{{RID}}}){class=chaise-autofill "
|
|
133
|
-
# "style=\"min-width: 500px; min-height: 300px;\"} \\n:::\n {{/if}}"
|
|
134
|
-
# },
|
|
135
|
-
# "markdown_name": "ROC Plot"
|
|
136
|
-
# }
|
|
137
|
-
],
|
|
138
|
-
"detailed": [
|
|
139
|
-
"RID",
|
|
140
|
-
"RCT",
|
|
141
|
-
"RMT",
|
|
142
|
-
"RCB",
|
|
143
|
-
"RMB",
|
|
144
|
-
# {
|
|
145
|
-
# "display": {
|
|
146
|
-
# "template_engine": "handlebars",
|
|
147
|
-
# "markdown_pattern": "{{#if (eq _Execution_Asset_Type \"2-5QME\")}} ::: iframe []("
|
|
148
|
-
# "https://dev.eye-ai.org/~vivi/deriva-webapps/plot/?config=test-line"
|
|
149
|
-
# "-plot&Execution_Assets_RID={{{RID}}}){style=\"min-width:1000px; "
|
|
150
|
-
# "min-height:700px; height:70vh;\" class=\"chaise-autofill\"} \\n::: {"
|
|
151
|
-
# "{/if}}"
|
|
152
|
-
# },
|
|
153
|
-
# "markdown_name": "ROC Plot"
|
|
154
|
-
# },
|
|
155
|
-
"URL",
|
|
156
|
-
"Filename",
|
|
157
|
-
"Description",
|
|
158
|
-
"Length",
|
|
159
|
-
"MD5",
|
|
160
|
-
[
|
|
161
|
-
schema,
|
|
162
|
-
"Execution_Assets_Execution_Asset_Type_fkey"
|
|
163
|
-
]
|
|
164
|
-
]
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
execution_metadata_annotation = {
|
|
169
|
-
"tag:isrd.isi.edu,2016:table-display": {
|
|
170
|
-
"row_name": {
|
|
171
|
-
"row_markdown_pattern": "{{{Filename}}}"
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
schema_annotation = {
|
|
177
|
-
"headTitle": "Catalog ML",
|
|
178
|
-
"navbarMenu": {
|
|
179
|
-
"newTab": False,
|
|
180
|
-
"children": [
|
|
181
|
-
{
|
|
182
|
-
"name": "User Info",
|
|
183
|
-
"children": [
|
|
184
|
-
{
|
|
185
|
-
"url": f"/chaise/recordset/#{catalog_id}/public:ERMrest_Client",
|
|
186
|
-
"name": "Users"
|
|
187
|
-
},
|
|
188
|
-
{
|
|
189
|
-
"url": f"/chaise/recordset/#{catalog_id}/public:ERMrest_Group",
|
|
190
|
-
"name": "Groups"
|
|
191
|
-
},
|
|
192
|
-
{
|
|
193
|
-
"url": f"/chaise/recordset/#{catalog_id}/public:ERMrest_RID_Lease",
|
|
194
|
-
"name": "ERMrest RID Lease"
|
|
195
|
-
}
|
|
196
|
-
]
|
|
197
|
-
},
|
|
198
|
-
{
|
|
199
|
-
"name": "FaceBase-ML",
|
|
200
|
-
"children": [
|
|
201
|
-
{
|
|
202
|
-
"url": f"/chaise/recordset/#{catalog_id}/{schema}:Workflow",
|
|
203
|
-
"name": "Workflow"
|
|
204
|
-
},
|
|
205
|
-
{
|
|
206
|
-
"url": f"/chaise/recordset/#{catalog_id}/{schema}:Workflow_Type",
|
|
207
|
-
"name": "Workflow Type"
|
|
208
|
-
},
|
|
209
|
-
{
|
|
210
|
-
"url": f"/chaise/recordset/#{catalog_id}/{schema}:Execution",
|
|
211
|
-
"name": "Execution"
|
|
212
|
-
},
|
|
213
|
-
{
|
|
214
|
-
"url": f"/chaise/recordset/#{catalog_id}/{schema}:Execution_Metadata",
|
|
215
|
-
"name": "Execution Metadata"
|
|
216
|
-
},
|
|
217
|
-
{
|
|
218
|
-
"url": f"/chaise/recordset/#{catalog_id}/{schema}:Execution_Metadata_Type",
|
|
219
|
-
"name": "Execution Metadata Type"
|
|
220
|
-
},
|
|
221
|
-
{
|
|
222
|
-
"url": f"/chaise/recordset/#{catalog_id}/{schema}:Execution_Assets",
|
|
223
|
-
"name": "Execution Assets"
|
|
224
|
-
},
|
|
225
|
-
{
|
|
226
|
-
"url": f"/chaise/recordset/#{catalog_id}/{schema}:Execution_Asset_Type",
|
|
227
|
-
"name": "Execution Asset Type"
|
|
228
|
-
}
|
|
229
|
-
]
|
|
230
|
-
}
|
|
231
|
-
]
|
|
232
|
-
},
|
|
233
|
-
"navbarBrandText": "ML Data Browser",
|
|
234
|
-
"systemColumnsDisplayEntry": ["RID"],
|
|
235
|
-
"systemColumnsDisplayCompact": ["RID"]
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
return {"workflow_annotation": workflow_annotation,
|
|
239
|
-
"execution_annotation": execution_annotation,
|
|
240
|
-
"execution_assets_annotation": execution_assets_annotation,
|
|
241
|
-
"execution_metadata_annotation": execution_metadata_annotation,
|
|
242
|
-
"schema_annotation": schema_annotation
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
def main():
|
|
247
|
-
parser = argparse.ArgumentParser()
|
|
248
|
-
parser.add_argument('--catalog_id', type=str, required=True)
|
|
249
|
-
parser.add_argument('--schema_name', type=str, required=True)
|
|
250
|
-
args = parser.parse_args()
|
|
251
|
-
return generate_annotation(args.catalog_id, args.schema_name)
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
if __name__ == "__main__":
|
|
255
|
-
sys.exit(main())
|
|
@@ -1,165 +0,0 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
|
|
3
|
-
from deriva.core import DerivaServer, get_credential
|
|
4
|
-
from deriva.core.ermrest_model import builtin_types, Schema, Table, Column, ForeignKey
|
|
5
|
-
from deriva.chisel import Model, Schema, Table, Column, ForeignKey
|
|
6
|
-
from deriva_ml.schema_setup.annotation_temp import generate_annotation
|
|
7
|
-
import argparse
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def create_schema_if_not_exist(model, schema_name, schema_comment=None):
|
|
11
|
-
if schema_name not in model.schemas:
|
|
12
|
-
schema = model.create_schema(Schema.define(schema_name, schema_comment))
|
|
13
|
-
return schema
|
|
14
|
-
else:
|
|
15
|
-
schema = model.schemas[schema_name]
|
|
16
|
-
return schema
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def create_table_if_not_exist(schema, table_name, create_spec):
|
|
20
|
-
if table_name not in schema.tables:
|
|
21
|
-
table = schema.create_table(create_spec)
|
|
22
|
-
return table
|
|
23
|
-
else:
|
|
24
|
-
table = schema.tables[table_name]
|
|
25
|
-
return table
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def define_table_workflow(workflow_annotation: dict):
|
|
29
|
-
table_def = Table.define(
|
|
30
|
-
'Workflow',
|
|
31
|
-
column_defs=[
|
|
32
|
-
Column.define('Name', builtin_types.text),
|
|
33
|
-
Column.define('Description', builtin_types.markdown),
|
|
34
|
-
Column.define('URL', builtin_types.ermrest_uri),
|
|
35
|
-
Column.define('Checksum', builtin_types.text),
|
|
36
|
-
Column.define('Version', builtin_types.text)
|
|
37
|
-
],
|
|
38
|
-
fkey_defs=[
|
|
39
|
-
ForeignKey.define(
|
|
40
|
-
['RCB'],
|
|
41
|
-
'public', 'ERMrest_Client',
|
|
42
|
-
['ID']
|
|
43
|
-
)
|
|
44
|
-
],
|
|
45
|
-
annotations=workflow_annotation
|
|
46
|
-
)
|
|
47
|
-
return table_def
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def define_table_execution(execution_annotation: dict):
|
|
51
|
-
table_def = Table.define(
|
|
52
|
-
'Execution',
|
|
53
|
-
column_defs=[
|
|
54
|
-
Column.define('Description', builtin_types.markdown),
|
|
55
|
-
Column.define('Duration', builtin_types.text),
|
|
56
|
-
Column.define('Status', builtin_types.text),
|
|
57
|
-
Column.define('Status_Detail', builtin_types.text)
|
|
58
|
-
],
|
|
59
|
-
fkey_defs=[
|
|
60
|
-
ForeignKey.define(
|
|
61
|
-
['RCB'],
|
|
62
|
-
'public', 'ERMrest_Client',
|
|
63
|
-
['ID']
|
|
64
|
-
)
|
|
65
|
-
],
|
|
66
|
-
annotations=execution_annotation
|
|
67
|
-
)
|
|
68
|
-
return table_def
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def define_asset_execution_metadata(schema: str, execution_metadata_annotation: dict):
|
|
72
|
-
table_def = Table.define_asset(
|
|
73
|
-
sname=schema,
|
|
74
|
-
tname='Execution_Metadata',
|
|
75
|
-
hatrac_template='/hatrac/metadata/{{MD5}}.{{Filename}}',
|
|
76
|
-
fkey_defs=[
|
|
77
|
-
ForeignKey.define(
|
|
78
|
-
['RCB'],
|
|
79
|
-
'public', 'ERMrest_Client',
|
|
80
|
-
['ID']
|
|
81
|
-
)
|
|
82
|
-
],
|
|
83
|
-
annotations=execution_metadata_annotation
|
|
84
|
-
)
|
|
85
|
-
return table_def
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
def define_asset_execution_assets(schema: str, execution_assets_annotation: dict):
|
|
89
|
-
table_def = Table.define_asset(
|
|
90
|
-
sname=schema,
|
|
91
|
-
tname='Execution_Assets',
|
|
92
|
-
hatrac_template='/hatrac/execution_assets/{{MD5}}.{{Filename}}',
|
|
93
|
-
fkey_defs=[
|
|
94
|
-
ForeignKey.define(
|
|
95
|
-
['RCB'],
|
|
96
|
-
'public', 'ERMrest_Client',
|
|
97
|
-
['ID']
|
|
98
|
-
)
|
|
99
|
-
],
|
|
100
|
-
annotations=execution_assets_annotation
|
|
101
|
-
)
|
|
102
|
-
return table_def
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def setup_ml_workflow(model, schema_name, catalog_id):
|
|
106
|
-
curie_template = catalog_id+':{RID}'
|
|
107
|
-
schema = create_schema_if_not_exist(model, schema_name)
|
|
108
|
-
# get annotations
|
|
109
|
-
annotations = generate_annotation(catalog_id, schema_name)
|
|
110
|
-
# Workflow
|
|
111
|
-
workflow_table = create_table_if_not_exist(schema, 'Workflow',
|
|
112
|
-
define_table_workflow(annotations["workflow_annotation"]))
|
|
113
|
-
table_def_workflow_type_vocab = Table.define_vocabulary(
|
|
114
|
-
tname='Workflow_Type', curie_template=curie_template
|
|
115
|
-
)
|
|
116
|
-
workflow_type_table = schema.create_table(table_def_workflow_type_vocab)
|
|
117
|
-
workflow_table.add_reference(workflow_type_table)
|
|
118
|
-
|
|
119
|
-
# Execution
|
|
120
|
-
execution_table = create_table_if_not_exist(schema, 'Execution',
|
|
121
|
-
define_table_execution(annotations["execution_annotation"]))
|
|
122
|
-
execution_table.add_reference(workflow_table)
|
|
123
|
-
# dataset_table = create_table_if_not_exist(schema, 'Dataset', define_table_dataset(schema))
|
|
124
|
-
# association_dataset_execution = schema.create_association(dataset_table, execution_table)
|
|
125
|
-
|
|
126
|
-
# Execution Metadata
|
|
127
|
-
execution_metadata_table = create_table_if_not_exist(schema, 'Execution_Metadata',
|
|
128
|
-
define_asset_execution_metadata(schema,
|
|
129
|
-
annotations["execution_metadata_annotation"]))
|
|
130
|
-
execution_metadata_table.add_reference(execution_table)
|
|
131
|
-
table_def_metadata_type_vocab = Table.define_vocabulary(tname='Execution_Metadata_Type',
|
|
132
|
-
curie_template=curie_template)
|
|
133
|
-
metadata_type_table = schema.create_table(table_def_metadata_type_vocab)
|
|
134
|
-
execution_metadata_table.add_reference(metadata_type_table)
|
|
135
|
-
|
|
136
|
-
# Execution Asset
|
|
137
|
-
execution_assets_table = create_table_if_not_exist(schema, 'Execution_Assets',
|
|
138
|
-
define_asset_execution_assets(schema,
|
|
139
|
-
annotations["execution_assets_annotation"]))
|
|
140
|
-
association_execution_execution_asset = schema.create_association(execution_assets_table, execution_table)
|
|
141
|
-
|
|
142
|
-
table_def_execution_product_type_vocab = Table.define_vocabulary(
|
|
143
|
-
tname='Execution_Asset_Type', curie_template=curie_template
|
|
144
|
-
)
|
|
145
|
-
execution_asset_type_table = schema.create_table(table_def_execution_product_type_vocab)
|
|
146
|
-
execution_assets_table.add_reference(execution_asset_type_table)
|
|
147
|
-
# image_table = create_table_if_not_exist(schema, 'Image', define_asset_image(schema))
|
|
148
|
-
# association_image_execution_asset = schema.create_association(execution_assets_table, image_table)
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def main():
|
|
152
|
-
scheme = 'https'
|
|
153
|
-
parser = argparse.ArgumentParser()
|
|
154
|
-
parser.add_argument('--hostname', type=str, required=True)
|
|
155
|
-
parser.add_argument('--schema_name', type=str, required=True)
|
|
156
|
-
parser.add_argument('--catalog_id', type=str, required=True)
|
|
157
|
-
args = parser.parse_args()
|
|
158
|
-
credentials = get_credential(args.hostname)
|
|
159
|
-
server = DerivaServer(scheme, args.hostname, credentials)
|
|
160
|
-
model = Model.from_catalog(server.connect_ermrest(args.catalog_id))
|
|
161
|
-
setup_ml_workflow(model, args.schema_name, args.catalog_id)
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
if __name__ == "__main__":
|
|
165
|
-
sys.exit(main())
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
from deriva.core import ErmrestCatalog, get_credential
|
|
3
|
-
import argparse
|
|
4
|
-
import os
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def update_table_comments(model, schema_name: str, table_name: str, comments_dir: str) -> None:
|
|
9
|
-
table = model.schemas[schema_name].tables[table_name]
|
|
10
|
-
table_comments_dir = Path(comments_dir)/Path(f"{schema_name}/{table_name}")
|
|
11
|
-
for file in os.listdir(table_comments_dir):
|
|
12
|
-
file_path = os.path.join(table_comments_dir, file)
|
|
13
|
-
with open(file_path, "r") as f:
|
|
14
|
-
comment_str = f.read()
|
|
15
|
-
if file.split(".")[0] == table_name:
|
|
16
|
-
table.comment = comment_str
|
|
17
|
-
else:
|
|
18
|
-
table.columns[file.split(".")[0]].comment = comment_str
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def update_schema_comments(model, schema_name: str, comments_dir: str) -> None:
|
|
22
|
-
schema_comments_dir = Path(comments_dir)/Path(schema_name)
|
|
23
|
-
for table in os.listdir(schema_comments_dir):
|
|
24
|
-
if not table.endswith(".DS_Store"):
|
|
25
|
-
update_table_comments(model, schema_name, table, comments_dir)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def main():
|
|
29
|
-
scheme = 'https'
|
|
30
|
-
parser = argparse.ArgumentParser()
|
|
31
|
-
parser.add_argument('--hostname', type=str, required=True)
|
|
32
|
-
parser.add_argument('--schema_name', type=str, required=True)
|
|
33
|
-
parser.add_argument('--catalog_id', type=str, required=True)
|
|
34
|
-
parser.add_argument('--comments_dir', type=str, required=True,
|
|
35
|
-
help="The directory containing the comments files for the whole catalog")
|
|
36
|
-
parser.add_argument('--table_name', type=str,
|
|
37
|
-
help="Only update the comments for one table")
|
|
38
|
-
args = parser.parse_args()
|
|
39
|
-
|
|
40
|
-
credentials = get_credential(args.hostname)
|
|
41
|
-
catalog = ErmrestCatalog(scheme, args.hostname, args.catalog_id, credentials)
|
|
42
|
-
model = catalog.getCatalogModel()
|
|
43
|
-
if args.table_name:
|
|
44
|
-
update_table_comments(model, args.schema_name, args.table_name, args.comments_dir)
|
|
45
|
-
model.apply()
|
|
46
|
-
else:
|
|
47
|
-
update_schema_comments(model, args.schema_name, args.comments_dir)
|
|
48
|
-
model.apply()
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if __name__ == '__main__':
|
|
52
|
-
sys.exit(main())
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
# docs/<schema-name>/<table-name>/[table|<column-name>.Md
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
import argparse
|
|
3
|
-
from deriva.core import get_credential, ErmrestCatalog
|
|
4
|
-
from deriva.core.ermrest_model import Model
|
|
5
|
-
|
|
6
|
-
from deriva_ml.schema_setup.annotations import generate_annotation
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def alter_table_annotation(
|
|
10
|
-
model_root: Model, schema_name: str, table_name: str, annotation: dict
|
|
11
|
-
):
|
|
12
|
-
table = model_root.schemas[schema_name].tables[table_name]
|
|
13
|
-
table.alter(annotations=annotation)
|
|
14
|
-
model_root.apply()
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def alter_schema_annotation(model_root: Model, schema_name: str, annotation: dict):
|
|
18
|
-
schema = model_root.schemas[schema_name]
|
|
19
|
-
schema.alter(annotations=annotation)
|
|
20
|
-
model_root.apply()
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def main():
|
|
24
|
-
parser = argparse.ArgumentParser()
|
|
25
|
-
parser.add_argument("--hostname", type=str, required=True)
|
|
26
|
-
parser.add_argument("--schema_name", type=str, required=True)
|
|
27
|
-
parser.add_argument("--catalog_id", type=str, required=True)
|
|
28
|
-
args = parser.parse_args()
|
|
29
|
-
credentials = get_credential(args.hostname)
|
|
30
|
-
catalog = ErmrestCatalog("https", args.hostname, args.catalog_id, credentials)
|
|
31
|
-
model = catalog.getCatalogModel()
|
|
32
|
-
annotations = generate_annotation(model)
|
|
33
|
-
alter_table_annotation(
|
|
34
|
-
model, args.schema_name, "Workflow", annotations["workflow_annotation"]
|
|
35
|
-
)
|
|
36
|
-
alter_table_annotation(
|
|
37
|
-
model, args.schema_name, "Execution", annotations["execution_annotation"]
|
|
38
|
-
)
|
|
39
|
-
alter_table_annotation(
|
|
40
|
-
model,
|
|
41
|
-
args.schema_name,
|
|
42
|
-
"Execution_Metadata",
|
|
43
|
-
annotations["execution_metadata_annotation"],
|
|
44
|
-
)
|
|
45
|
-
alter_table_annotation(
|
|
46
|
-
model,
|
|
47
|
-
args.schema_name,
|
|
48
|
-
"Execution_Asset",
|
|
49
|
-
annotations["execution_asset_annotation"],
|
|
50
|
-
)
|
|
51
|
-
alter_schema_annotation(model, args.schema_name, annotations["schema_annotation"])
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
if __name__ == "__main__":
|
|
55
|
-
sys.exit(main())
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
deriva_ml/__init__.py,sha256=r1Z9N5vtZkAET7emqhpAx2bf_xJUp5wHOc4_DIplsG8,1082
|
|
2
|
-
deriva_ml/database_model.py,sha256=uhoyVyd8MQmY8J9ovCH8fjxhZDxxXNkdJyYdeyEGPXA,13898
|
|
3
|
-
deriva_ml/dataset.py,sha256=xC6QPUp4MZcJiEnOEU3NnzoLBL9RcJWtPTyzIQP0Ivw,60666
|
|
4
|
-
deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
|
|
5
|
-
deriva_ml/dataset_bag.py,sha256=e6IHv3saZUnZRfl0EjfnlV2NnmPeOagYYv3PuZqS1l0,11501
|
|
6
|
-
deriva_ml/demo_catalog.py,sha256=xQPhFlflqwJskNQrQ-jdBSnGzBm2-aONBgcRxfsdNKM,11045
|
|
7
|
-
deriva_ml/deriva_definitions.py,sha256=pZLPoUxiuJ-uGglmQ6sF9oVXsSUuOnPEqywoec78XNM,8893
|
|
8
|
-
deriva_ml/deriva_ml_base.py,sha256=KbmJ0-mGuZn7-wuzQemzof8o8mA_3-UVlQCrnDsKod0,45741
|
|
9
|
-
deriva_ml/deriva_model.py,sha256=LV3FjIhIlz13ckZSmu0aOJhT9EVE0-M9oVMudfkxb0g,12004
|
|
10
|
-
deriva_ml/execution.py,sha256=uDblqngcldgR7X4W1PfMV4iPWkxwQYSr9CBmXNlIv1E,29572
|
|
11
|
-
deriva_ml/execution_configuration.py,sha256=bjnZwXN6M7YPy5dFQwoGEBU8YjhQRSe1FW0rL0V9TaM,3422
|
|
12
|
-
deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
|
|
13
|
-
deriva_ml/feature.py,sha256=7e8WYPCfJSrGxJh9oUTduYSnB5ekybRhXa_0HIigS_w,5459
|
|
14
|
-
deriva_ml/history.py,sha256=qTDLDs8Ow_6r7mDO0gZm0Fg81SWKOAgtCU5pzZoDRgM,2828
|
|
15
|
-
deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
|
|
16
|
-
deriva_ml/upload.py,sha256=CKtT-gBln3pnAll9TFaiPhFSHC-bzg9oE4ruh_OSOqY,22270
|
|
17
|
-
deriva_ml/build/lib/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
-
deriva_ml/build/lib/schema_setup/alter_annotation.py,sha256=pkwk0WystN69JfAFK4iBJZAZVQKbRs-gN9IFYuS9rfg,1739
|
|
19
|
-
deriva_ml/build/lib/schema_setup/annotation_temp.py,sha256=Euygu8wNklZFUbR6mz-pDWJemlzdsIn9d6j0f6fCfgE,9102
|
|
20
|
-
deriva_ml/build/lib/schema_setup/create_schema.py,sha256=9LI2ivR0xS4kVBxkz3F1Ac4hoYGGOVkXYmuhllqWvKE,6528
|
|
21
|
-
deriva_ml/build/lib/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
|
|
22
|
-
deriva_ml/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
-
deriva_ml/schema_setup/alter_annotation.py,sha256=HyGWe4fBLQY0wbcKnfrhEoVDs4SfnoOF0l_Rz5-tDlk,1794
|
|
24
|
-
deriva_ml/schema_setup/annotations.py,sha256=Uogm9YkRtoKSdgfQlICqRywbCATppwBO-XryVSVyso8,9546
|
|
25
|
-
deriva_ml/schema_setup/create_schema.py,sha256=jwziMWJPbjRgjiRBT-KtidnXI8YNEFO74A9fwfptjHY,10626
|
|
26
|
-
deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
|
|
27
|
-
deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
|
|
28
|
-
deriva_ml-1.8.11.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
29
|
-
deriva_ml-1.8.11.dist-info/METADATA,sha256=RyttxTkz_MCnWX1hZK2g7ffPyd54txE6AAZ2GMSpQ54,670
|
|
30
|
-
deriva_ml-1.8.11.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
|
|
31
|
-
deriva_ml-1.8.11.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
|
|
32
|
-
deriva_ml-1.8.11.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
|
|
33
|
-
deriva_ml-1.8.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|