deriva-ml 1.13.0__py3-none-any.whl → 1.13.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deriva_ml/__init__.py +2 -2
- deriva_ml/deriva_definitions.py +11 -1
- deriva_ml/execution.py +13 -8
- deriva_ml/execution_configuration.py +3 -3
- deriva_ml/run_notebook.py +40 -16
- deriva_ml/test-files/execution-parameters.json +1 -0
- deriva_ml/test-files/notebook-parameters.json +5 -0
- deriva_ml/test_notebook.ipynb +99 -26
- {deriva_ml-1.13.0.dist-info → deriva_ml-1.13.1.dist-info}/METADATA +1 -1
- {deriva_ml-1.13.0.dist-info → deriva_ml-1.13.1.dist-info}/RECORD +14 -12
- {deriva_ml-1.13.0.dist-info → deriva_ml-1.13.1.dist-info}/WHEEL +0 -0
- {deriva_ml-1.13.0.dist-info → deriva_ml-1.13.1.dist-info}/entry_points.txt +0 -0
- {deriva_ml-1.13.0.dist-info → deriva_ml-1.13.1.dist-info}/licenses/LICENSE +0 -0
- {deriva_ml-1.13.0.dist-info → deriva_ml-1.13.1.dist-info}/top_level.txt +0 -0
deriva_ml/__init__.py
CHANGED
|
@@ -15,7 +15,7 @@ __all__ = [
|
|
|
15
15
|
"UploadState",
|
|
16
16
|
"MLVocab",
|
|
17
17
|
"MLAsset",
|
|
18
|
-
"
|
|
18
|
+
"ExecAssetType",
|
|
19
19
|
"RID",
|
|
20
20
|
"DerivaSystemColumns",
|
|
21
21
|
"VersionPart",
|
|
@@ -34,7 +34,7 @@ from .deriva_definitions import (
|
|
|
34
34
|
DerivaMLException,
|
|
35
35
|
MLVocab,
|
|
36
36
|
MLAsset,
|
|
37
|
-
|
|
37
|
+
ExecAssetType,
|
|
38
38
|
DerivaSystemColumns,
|
|
39
39
|
)
|
|
40
40
|
from .deriva_ml_base import DerivaML
|
deriva_ml/deriva_definitions.py
CHANGED
|
@@ -197,7 +197,7 @@ class MLAsset(StrEnum):
|
|
|
197
197
|
execution_asset = "Execution_Asset"
|
|
198
198
|
|
|
199
199
|
|
|
200
|
-
class
|
|
200
|
+
class ExecMetadataType(StrEnum):
|
|
201
201
|
"""
|
|
202
202
|
Predefined execution metadata types.
|
|
203
203
|
"""
|
|
@@ -206,6 +206,16 @@ class ExecMetadataVocab(StrEnum):
|
|
|
206
206
|
runtime_env = "Runtime_Env"
|
|
207
207
|
|
|
208
208
|
|
|
209
|
+
class ExecAssetType(StrEnum):
|
|
210
|
+
"""
|
|
211
|
+
Predefined execution metadata types.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
input_file = "Input_File"
|
|
215
|
+
output_file = "Output_File"
|
|
216
|
+
notebook_output = "Notebook_Output"
|
|
217
|
+
|
|
218
|
+
|
|
209
219
|
class ColumnDefinition(BaseModel):
|
|
210
220
|
"""Pydantic model for deriva_py Column.define"""
|
|
211
221
|
|
deriva_ml/execution.py
CHANGED
|
@@ -20,7 +20,6 @@ from typing import Iterable, Any, Optional
|
|
|
20
20
|
from deriva.core import format_exception
|
|
21
21
|
from deriva.core.datapath import DataPathException
|
|
22
22
|
from deriva.core.hatrac_store import HatracStore
|
|
23
|
-
from .deriva_definitions import ExecMetadataVocab
|
|
24
23
|
from .deriva_definitions import (
|
|
25
24
|
RID,
|
|
26
25
|
Status,
|
|
@@ -28,6 +27,8 @@ from .deriva_definitions import (
|
|
|
28
27
|
DerivaMLException,
|
|
29
28
|
MLVocab,
|
|
30
29
|
MLAsset,
|
|
30
|
+
ExecMetadataType,
|
|
31
|
+
ExecAssetType,
|
|
31
32
|
DRY_RUN_RID,
|
|
32
33
|
)
|
|
33
34
|
from .deriva_ml_base import DerivaML, FeatureRecord
|
|
@@ -237,9 +238,9 @@ class Execution:
|
|
|
237
238
|
|
|
238
239
|
def _save_runtime_environment(self):
|
|
239
240
|
runtime_env_path = self.asset_file_path(
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
241
|
+
"Execution_Metadata",
|
|
242
|
+
f"environment_snapshot_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt",
|
|
243
|
+
ExecMetadataType.runtime_env.value,
|
|
243
244
|
)
|
|
244
245
|
with open(runtime_env_path, "w") as fp:
|
|
245
246
|
json.dump(get_execution_environment(), fp)
|
|
@@ -296,15 +297,19 @@ class Execution:
|
|
|
296
297
|
# Save configuration details for later upload
|
|
297
298
|
if not reload:
|
|
298
299
|
cfile = self.asset_file_path(
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
300
|
+
MLAsset.execution_metadata,
|
|
301
|
+
"configuration.json",
|
|
302
|
+
ExecMetadataType.execution_config.value,
|
|
302
303
|
)
|
|
303
304
|
with open(cfile.as_posix(), "w", encoding="utf-8") as config_file:
|
|
304
305
|
json.dump(self.configuration.model_dump(), config_file)
|
|
305
306
|
|
|
306
307
|
for parameter_file in self.configuration.parameters:
|
|
307
|
-
self.asset_file_path(
|
|
308
|
+
self.asset_file_path(
|
|
309
|
+
MLAsset.execution_asset,
|
|
310
|
+
parameter_file,
|
|
311
|
+
ExecAssetType.input_file.value,
|
|
312
|
+
)
|
|
308
313
|
|
|
309
314
|
# save runtime env
|
|
310
315
|
self._save_runtime_environment()
|
|
@@ -264,7 +264,7 @@ class Workflow(BaseModel):
|
|
|
264
264
|
checksum = os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"]
|
|
265
265
|
is_notebook = True
|
|
266
266
|
else:
|
|
267
|
-
path, is_notebook = Workflow.
|
|
267
|
+
path, is_notebook = Workflow._get_python_script()
|
|
268
268
|
github_url, checksum = Workflow.get_url_and_checksum(path)
|
|
269
269
|
|
|
270
270
|
return Workflow(
|
|
@@ -332,7 +332,7 @@ class ExecutionConfiguration(BaseModel):
|
|
|
332
332
|
datasets: conlist(DatasetSpec) = []
|
|
333
333
|
assets: list[RID | str] = [] # List of RIDs to model files.
|
|
334
334
|
workflow: RID | Workflow
|
|
335
|
-
parameters: dict[str, Any] = {}
|
|
335
|
+
parameters: dict[str, Any] | Path = {}
|
|
336
336
|
description: str = ""
|
|
337
337
|
argv: conlist(str) = Field(default_factory=lambda: sys.argv)
|
|
338
338
|
|
|
@@ -341,7 +341,7 @@ class ExecutionConfiguration(BaseModel):
|
|
|
341
341
|
@field_validator("parameters", mode="before")
|
|
342
342
|
@classmethod
|
|
343
343
|
def validate_parameters(cls, value: Any) -> Any:
|
|
344
|
-
"""If parameter is a file, assume that it has JSON contents for configuration parameters"""
|
|
344
|
+
"""If a parameter is a file, assume that it has JSON contents for configuration parameters"""
|
|
345
345
|
if isinstance(value, str) or isinstance(value, Path):
|
|
346
346
|
with open(value, "r") as f:
|
|
347
347
|
return json.load(f)
|
deriva_ml/run_notebook.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Module to run a notebook using papermill"""
|
|
2
2
|
|
|
3
|
+
from datetime import datetime
|
|
3
4
|
import json
|
|
4
5
|
import os
|
|
5
6
|
import papermill as pm
|
|
@@ -7,9 +8,9 @@ from pathlib import Path
|
|
|
7
8
|
import regex as re
|
|
8
9
|
import tempfile
|
|
9
10
|
|
|
10
|
-
from deriva_ml import Workflow, DerivaML
|
|
11
|
+
from deriva_ml import Workflow, DerivaML
|
|
11
12
|
from deriva.core import BaseCLI
|
|
12
|
-
from deriva_ml import MLAsset
|
|
13
|
+
from deriva_ml import MLAsset, ExecAssetType
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class DerivaMLRunNotebookCLI(BaseCLI):
|
|
@@ -36,6 +37,19 @@ class DerivaMLRunNotebookCLI(BaseCLI):
|
|
|
36
37
|
help="Display parameters information for the given notebook path.",
|
|
37
38
|
)
|
|
38
39
|
|
|
40
|
+
self.parser.add_argument(
|
|
41
|
+
"--log-output",
|
|
42
|
+
action="store_false",
|
|
43
|
+
help="Display logging output from notebook.",
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
self.parser.add_argument(
|
|
47
|
+
"--catalog",
|
|
48
|
+
metavar="<1>",
|
|
49
|
+
default=1,
|
|
50
|
+
help="Catalog number. Default 1",
|
|
51
|
+
)
|
|
52
|
+
|
|
39
53
|
self.parser.add_argument(
|
|
40
54
|
"--parameter",
|
|
41
55
|
"-p",
|
|
@@ -43,7 +57,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
|
|
|
43
57
|
action="append",
|
|
44
58
|
metavar=("KEY", "VALUE"),
|
|
45
59
|
default=[],
|
|
46
|
-
help="Provide a parameter name
|
|
60
|
+
help="Provide a parameter name and value to inject into the notebook.",
|
|
47
61
|
)
|
|
48
62
|
|
|
49
63
|
self.parser.add_argument(
|
|
@@ -84,19 +98,24 @@ class DerivaMLRunNotebookCLI(BaseCLI):
|
|
|
84
98
|
print("Notebook file must be an ipynb file.")
|
|
85
99
|
exit(1)
|
|
86
100
|
|
|
101
|
+
os.environ["DERIVA_HOST"] = args.host
|
|
102
|
+
os.environ["DERIVA_CATALOG_ID"] = args.catalog
|
|
103
|
+
|
|
87
104
|
# Create a workflow instance for this specific version of the script. Return an existing workflow if one is found.
|
|
88
105
|
notebook_parameters = pm.inspect_notebook(notebook_file)
|
|
89
106
|
if args.inspect:
|
|
90
|
-
for param, value in notebook_parameters:
|
|
107
|
+
for param, value in notebook_parameters.items():
|
|
91
108
|
print(
|
|
92
109
|
f"{param}:{value['inferred_type_name']} (default {value['default']})"
|
|
93
110
|
)
|
|
94
111
|
return
|
|
95
112
|
else:
|
|
96
|
-
notebook_parameters =
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
113
|
+
notebook_parameters = (
|
|
114
|
+
{"host": args.host, "catalog": args.catalog}
|
|
115
|
+
| {k: v["default"] for k, v in notebook_parameters.items()}
|
|
116
|
+
| parameters
|
|
117
|
+
)
|
|
118
|
+
print(f"Running notebook {notebook_file.name} with parameters:")
|
|
100
119
|
for param, value in notebook_parameters.items():
|
|
101
120
|
print(f" {param}:{value}")
|
|
102
121
|
self.run_notebook(notebook_file.resolve(), parameters, args.kernel)
|
|
@@ -121,7 +140,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
|
|
|
121
140
|
r"Execution RID: https://(?P<host>.*)/id/(?P<catalog_id>.*)/(?P<execution_rid>[\w-]+)",
|
|
122
141
|
line,
|
|
123
142
|
):
|
|
124
|
-
|
|
143
|
+
hostname = m["host"]
|
|
125
144
|
catalog_id = m["catalog_id"]
|
|
126
145
|
execution_rid = m["execution_rid"]
|
|
127
146
|
if not execution_rid:
|
|
@@ -129,19 +148,24 @@ class DerivaMLRunNotebookCLI(BaseCLI):
|
|
|
129
148
|
exit(1)
|
|
130
149
|
print("Uploaded notebook output for Execution RID:", execution_rid)
|
|
131
150
|
|
|
132
|
-
ml_instance = DerivaML(hostname=
|
|
133
|
-
|
|
134
|
-
MLVocab.asset_type,
|
|
135
|
-
"Notebook_Output",
|
|
136
|
-
description="Jupyter Notebook Output",
|
|
137
|
-
)
|
|
151
|
+
ml_instance = DerivaML(hostname=hostname, catalog_id=catalog_id)
|
|
152
|
+
|
|
138
153
|
execution = ml_instance.restore_execution(execution_rid)
|
|
139
154
|
execution.asset_file_path(
|
|
140
155
|
asset_name=MLAsset.execution_asset,
|
|
141
156
|
file_name=notebook_output,
|
|
142
|
-
asset_types=
|
|
157
|
+
asset_types=ExecAssetType.notebook_output,
|
|
143
158
|
)
|
|
159
|
+
parameter_file = execution.asset_file_path(
|
|
160
|
+
asset_name=MLAsset.execution_asset,
|
|
161
|
+
file_name=f"notebook-parameters-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json",
|
|
162
|
+
asset_types=ExecAssetType.input_file.value,
|
|
163
|
+
)
|
|
164
|
+
with open(parameter_file, "w") as f:
|
|
165
|
+
json.dump(parameters, f)
|
|
166
|
+
|
|
144
167
|
execution.upload_execution_outputs()
|
|
168
|
+
print(ml_instance.cite(execution_rid))
|
|
145
169
|
|
|
146
170
|
|
|
147
171
|
def main():
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"local-file": "My local file.txt"}
|
deriva_ml/test_notebook.ipynb
CHANGED
|
@@ -3,15 +3,21 @@
|
|
|
3
3
|
{
|
|
4
4
|
"cell_type": "code",
|
|
5
5
|
"id": "0",
|
|
6
|
-
"metadata": {
|
|
6
|
+
"metadata": {
|
|
7
|
+
"ExecuteTime": {
|
|
8
|
+
"end_time": "2025-04-18T22:52:49.930351Z",
|
|
9
|
+
"start_time": "2025-04-18T22:52:48.926842Z"
|
|
10
|
+
}
|
|
11
|
+
},
|
|
7
12
|
"source": [
|
|
8
13
|
"import builtins\n",
|
|
14
|
+
"import os\n",
|
|
15
|
+
"\n",
|
|
9
16
|
"from deriva.core.utils.globus_auth_utils import GlobusNativeLogin\n",
|
|
10
|
-
"from deriva_ml import ExecutionConfiguration, MLVocab,
|
|
11
|
-
"from deriva_ml.demo_catalog import create_demo_catalog, DemoML"
|
|
17
|
+
"from deriva_ml import ExecutionConfiguration, MLVocab, DerivaML, DatasetSpec"
|
|
12
18
|
],
|
|
13
19
|
"outputs": [],
|
|
14
|
-
"execution_count":
|
|
20
|
+
"execution_count": 1
|
|
15
21
|
},
|
|
16
22
|
{
|
|
17
23
|
"cell_type": "code",
|
|
@@ -19,36 +25,61 @@
|
|
|
19
25
|
"metadata": {
|
|
20
26
|
"tags": [
|
|
21
27
|
"parameters"
|
|
22
|
-
]
|
|
28
|
+
],
|
|
29
|
+
"ExecuteTime": {
|
|
30
|
+
"end_time": "2025-04-18T22:52:49.988873Z",
|
|
31
|
+
"start_time": "2025-04-18T22:52:49.986713Z"
|
|
32
|
+
}
|
|
23
33
|
},
|
|
24
34
|
"source": [
|
|
25
35
|
"foo: int = 1\n",
|
|
26
|
-
"
|
|
27
|
-
"
|
|
36
|
+
"assets = []\n",
|
|
37
|
+
"datasets = []\n",
|
|
38
|
+
"parameters = None"
|
|
28
39
|
],
|
|
29
40
|
"outputs": [],
|
|
30
|
-
"execution_count":
|
|
41
|
+
"execution_count": 2
|
|
31
42
|
},
|
|
32
43
|
{
|
|
33
|
-
"metadata": {
|
|
44
|
+
"metadata": {
|
|
45
|
+
"ExecuteTime": {
|
|
46
|
+
"end_time": "2025-04-18T22:52:50.002808Z",
|
|
47
|
+
"start_time": "2025-04-18T22:52:49.999450Z"
|
|
48
|
+
}
|
|
49
|
+
},
|
|
34
50
|
"cell_type": "code",
|
|
35
|
-
"outputs": [],
|
|
36
|
-
"execution_count": null,
|
|
37
51
|
"source": [
|
|
38
|
-
"print(
|
|
39
|
-
"print(
|
|
40
|
-
"print(
|
|
52
|
+
"print(\"foo\", foo)\n",
|
|
53
|
+
"print(\"assets\", assets)\n",
|
|
54
|
+
"print(\"datasets\", datasets)\n",
|
|
55
|
+
"print(\"parameters\", parameters)"
|
|
41
56
|
],
|
|
42
|
-
"id": "70b23cdd933ce669"
|
|
57
|
+
"id": "70b23cdd933ce669",
|
|
58
|
+
"outputs": [
|
|
59
|
+
{
|
|
60
|
+
"name": "stdout",
|
|
61
|
+
"output_type": "stream",
|
|
62
|
+
"text": [
|
|
63
|
+
"foo 1\n",
|
|
64
|
+
"assets []\n",
|
|
65
|
+
"datasets []\n",
|
|
66
|
+
"parameters None\n"
|
|
67
|
+
]
|
|
68
|
+
}
|
|
69
|
+
],
|
|
70
|
+
"execution_count": 3
|
|
43
71
|
},
|
|
44
72
|
{
|
|
45
|
-
"metadata": {
|
|
73
|
+
"metadata": {
|
|
74
|
+
"ExecuteTime": {
|
|
75
|
+
"end_time": "2025-04-18T22:52:50.344660Z",
|
|
76
|
+
"start_time": "2025-04-18T22:52:50.013816Z"
|
|
77
|
+
}
|
|
78
|
+
},
|
|
46
79
|
"cell_type": "code",
|
|
47
|
-
"outputs": [],
|
|
48
|
-
"execution_count": null,
|
|
49
80
|
"source": [
|
|
50
|
-
"hostname =
|
|
51
|
-
"
|
|
81
|
+
"hostname = os.environ.get(\"DERIVA_HOST\") #or \"dev.eye-ai.org\"\n",
|
|
82
|
+
"catalog_id = os.environ.get(\"DERIVA_CATALOG_ID\") #or 'eye-ai'\n",
|
|
52
83
|
"\n",
|
|
53
84
|
"gnl = GlobusNativeLogin(host=hostname)\n",
|
|
54
85
|
"if gnl.is_logged_in([hostname]):\n",
|
|
@@ -57,15 +88,31 @@
|
|
|
57
88
|
" gnl.login([hostname], no_local_server=True, no_browser=True, refresh_tokens=True, update_bdbag_keychain=True)\n",
|
|
58
89
|
" print(\"Login Successful\")\n"
|
|
59
90
|
],
|
|
60
|
-
"id": "2"
|
|
91
|
+
"id": "2",
|
|
92
|
+
"outputs": [
|
|
93
|
+
{
|
|
94
|
+
"ename": "AttributeError",
|
|
95
|
+
"evalue": "'NoneType' object has no attribute 'lower'",
|
|
96
|
+
"output_type": "error",
|
|
97
|
+
"traceback": [
|
|
98
|
+
"\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
|
|
99
|
+
"\u001B[0;31mAttributeError\u001B[0m Traceback (most recent call last)",
|
|
100
|
+
"Cell \u001B[0;32mIn[4], line 5\u001B[0m\n\u001B[1;32m 2\u001B[0m catalog_id \u001B[38;5;241m=\u001B[39m os\u001B[38;5;241m.\u001B[39menviron\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mDERIVA_CATALOG_ID\u001B[39m\u001B[38;5;124m\"\u001B[39m) \u001B[38;5;66;03m#or 'eye-ai'\u001B[39;00m\n\u001B[1;32m 4\u001B[0m gnl \u001B[38;5;241m=\u001B[39m GlobusNativeLogin(host\u001B[38;5;241m=\u001B[39mhostname)\n\u001B[0;32m----> 5\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[43mgnl\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mis_logged_in\u001B[49m\u001B[43m(\u001B[49m\u001B[43m[\u001B[49m\u001B[43mhostname\u001B[49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m:\n\u001B[1;32m 6\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mYou are already logged in.\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 7\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n",
|
|
101
|
+
"File \u001B[0;32m~/opt/anaconda3/envs/deriva-test/lib/python3.10/site-packages/deriva/core/utils/globus_auth_utils.py:582\u001B[0m, in \u001B[0;36mGlobusNativeLogin.is_logged_in\u001B[0;34m(self, hosts, requested_scopes, hosts_to_scope_map, exclude_defaults)\u001B[0m\n\u001B[1;32m 576\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21mis_logged_in\u001B[39m(\u001B[38;5;28mself\u001B[39m,\n\u001B[1;32m 577\u001B[0m hosts\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m 578\u001B[0m requested_scopes\u001B[38;5;241m=\u001B[39m(),\n\u001B[1;32m 579\u001B[0m hosts_to_scope_map\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m 580\u001B[0m exclude_defaults\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m):\n\u001B[1;32m 581\u001B[0m scopes \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mset\u001B[39m(requested_scopes)\n\u001B[0;32m--> 582\u001B[0m scope_map \u001B[38;5;241m=\u001B[39m hosts_to_scope_map \u001B[38;5;28;01mif\u001B[39;00m hosts_to_scope_map \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhosts_to_scope_map\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhosts\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01mor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhosts\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 583\u001B[0m scopes\u001B[38;5;241m.\u001B[39mupdate(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mscope_set_from_scope_map(scope_map))\n\u001B[1;32m 584\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m exclude_defaults:\n",
|
|
102
|
+
"File \u001B[0;32m~/opt/anaconda3/envs/deriva-test/lib/python3.10/site-packages/deriva/core/utils/globus_auth_utils.py:607\u001B[0m, in \u001B[0;36mGlobusNativeLogin.hosts_to_scope_map\u001B[0;34m(self, hosts, match_scope_tag, all_tagged_scopes, force_refresh, warn_on_discovery_failure)\u001B[0m\n\u001B[1;32m 605\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m host \u001B[38;5;129;01min\u001B[39;00m hosts:\n\u001B[1;32m 606\u001B[0m scope_map\u001B[38;5;241m.\u001B[39mupdate({host: []})\n\u001B[0;32m--> 607\u001B[0m scopes \u001B[38;5;241m=\u001B[39m \u001B[43mget_oauth_scopes_for_host\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhost\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 608\u001B[0m \u001B[43m \u001B[49m\u001B[43mconfig_file\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mconfig_file\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 609\u001B[0m \u001B[43m \u001B[49m\u001B[43mforce_refresh\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mforce_refresh\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 610\u001B[0m \u001B[43m \u001B[49m\u001B[43mwarn_on_discovery_failure\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mwarn_on_discovery_failure\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 611\u001B[0m scope_list \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m()\n\u001B[1;32m 612\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m scopes:\n",
|
|
103
|
+
"File \u001B[0;32m~/opt/anaconda3/envs/deriva-test/lib/python3.10/site-packages/deriva/core/utils/core_utils.py:300\u001B[0m, in \u001B[0;36mget_oauth_scopes_for_host\u001B[0;34m(host, config_file, force_refresh, warn_on_discovery_failure)\u001B[0m\n\u001B[1;32m 298\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m required_scopes:\n\u001B[1;32m 299\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m hostname, scopes \u001B[38;5;129;01min\u001B[39;00m required_scopes\u001B[38;5;241m.\u001B[39mitems():\n\u001B[0;32m--> 300\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[43mhost\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlower\u001B[49m() \u001B[38;5;241m==\u001B[39m hostname\u001B[38;5;241m.\u001B[39mlower():\n\u001B[1;32m 301\u001B[0m result \u001B[38;5;241m=\u001B[39m scopes\n\u001B[1;32m 302\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n",
|
|
104
|
+
"\u001B[0;31mAttributeError\u001B[0m: 'NoneType' object has no attribute 'lower'"
|
|
105
|
+
]
|
|
106
|
+
}
|
|
107
|
+
],
|
|
108
|
+
"execution_count": 4
|
|
61
109
|
},
|
|
62
110
|
{
|
|
63
111
|
"cell_type": "code",
|
|
64
112
|
"id": "3",
|
|
65
113
|
"metadata": {},
|
|
66
114
|
"source": [
|
|
67
|
-
"ml_instance =
|
|
68
|
-
"print(f'Creating catalog at {ml_instance.catalog_id}')\n",
|
|
115
|
+
"ml_instance = DerivaML(hostname, catalog_id)\n",
|
|
69
116
|
"\n",
|
|
70
117
|
"ml_instance.add_term(MLVocab.workflow_type, \"Manual Workflow\", description=\"Initial setup of Model File\")\n",
|
|
71
118
|
"ml_instance.add_term(MLVocab.asset_type, \"API_Model\", description=\"Model for our API workflow\")"
|
|
@@ -92,12 +139,38 @@
|
|
|
92
139
|
"id": "6",
|
|
93
140
|
"metadata": {},
|
|
94
141
|
"source": [
|
|
95
|
-
"manual_execution = ml_instance.create_execution(
|
|
96
|
-
"
|
|
97
|
-
"
|
|
142
|
+
"manual_execution = ml_instance.create_execution(\n",
|
|
143
|
+
" ExecutionConfiguration(\n",
|
|
144
|
+
" description=\"Sample Execution\",\n",
|
|
145
|
+
" workflow=api_workflow,\n",
|
|
146
|
+
" datasets=[DatasetSpec(rid=ds, version=ml_instance.dataset_version(ds)) for ds in datasets],\n",
|
|
147
|
+
" assets=assets,\n",
|
|
148
|
+
" parameters=parameters\n",
|
|
149
|
+
" )\n",
|
|
150
|
+
")"
|
|
98
151
|
],
|
|
99
152
|
"outputs": [],
|
|
100
153
|
"execution_count": null
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
"metadata": {},
|
|
157
|
+
"cell_type": "code",
|
|
158
|
+
"source": [
|
|
159
|
+
"print(f'parameters: {manual_execution.parameters}')\n",
|
|
160
|
+
"print(f'datasets: {manual_execution.datasets}')\n",
|
|
161
|
+
"print(f'assets: {manual_execution.asset_paths}')"
|
|
162
|
+
],
|
|
163
|
+
"id": "4b2a3b8c16333645",
|
|
164
|
+
"outputs": [],
|
|
165
|
+
"execution_count": null
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
"metadata": {},
|
|
169
|
+
"cell_type": "code",
|
|
170
|
+
"source": "manual_execution.upload_execution_outputs()",
|
|
171
|
+
"id": "efa8cb1b0ed438bb",
|
|
172
|
+
"outputs": [],
|
|
173
|
+
"execution_count": null
|
|
101
174
|
}
|
|
102
175
|
],
|
|
103
176
|
"metadata": {
|
|
@@ -1,29 +1,31 @@
|
|
|
1
|
-
deriva_ml/__init__.py,sha256=
|
|
1
|
+
deriva_ml/__init__.py,sha256=GfneBq7xDphMqUQY96sW9ixRj74M3UTUCmD4KMIRSaM,1101
|
|
2
2
|
deriva_ml/database_model.py,sha256=lMbAEqn4n0m7h_JstMX_LX9gbvBIEydG3sRilPn3eLU,14885
|
|
3
3
|
deriva_ml/dataset.py,sha256=OyWUKWnYeP0ctimSBQ4em-uJrzCNOohx4GPT2uIl6R4,60649
|
|
4
4
|
deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
|
|
5
5
|
deriva_ml/dataset_bag.py,sha256=yS8oYVshfFtRDyhGPRqtbvxjyd3ZFF29lrB783OP4vM,11849
|
|
6
6
|
deriva_ml/demo_catalog.py,sha256=9Qo3JD4bUIwnL3ngPctc2QBeWApvMR_5UyaK9ockTrY,11536
|
|
7
|
-
deriva_ml/deriva_definitions.py,sha256=
|
|
7
|
+
deriva_ml/deriva_definitions.py,sha256=avdOgxtB60yb8XsWm-AYtCdvg2QkQbyfkZuA9xx9t2U,9221
|
|
8
8
|
deriva_ml/deriva_ml_base.py,sha256=JYTG_a8SURhrPQBTz6OaGMk0D0sSPWpXqCnoVnSNViI,38501
|
|
9
9
|
deriva_ml/deriva_model.py,sha256=wytGCAHutiUaRfnRKr80Ks_P6ci0_wXRU3vq3lthfYU,13260
|
|
10
|
-
deriva_ml/execution.py,sha256=
|
|
11
|
-
deriva_ml/execution_configuration.py,sha256=
|
|
10
|
+
deriva_ml/execution.py,sha256=Oyja5wonSBUDUIVSC01w3AojGEkWyw_8_kBMv3MTZBM,38126
|
|
11
|
+
deriva_ml/execution_configuration.py,sha256=KKg2HhvOiOmYc3jJ9iJeeHYyRu05Bb8JpojmPn1gYW0,14072
|
|
12
12
|
deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
|
|
13
13
|
deriva_ml/feature.py,sha256=07g0uSrhumdopJluWuWSRMrzagaikAOihqB09bzXBP4,5475
|
|
14
14
|
deriva_ml/history.py,sha256=qTDLDs8Ow_6r7mDO0gZm0Fg81SWKOAgtCU5pzZoDRgM,2828
|
|
15
|
-
deriva_ml/run_notebook.py,sha256=
|
|
15
|
+
deriva_ml/run_notebook.py,sha256=vhmij4P1Va52MIj8hOc-WmjLRp3sTmK6p7LXCWrzejc,6308
|
|
16
16
|
deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
|
|
17
|
-
deriva_ml/test_notebook.ipynb,sha256=
|
|
17
|
+
deriva_ml/test_notebook.ipynb,sha256=_5D6rkSGbmENPJZbDgfZ6-yt94BNEwxytVUDmG3RE3w,10166
|
|
18
18
|
deriva_ml/upload.py,sha256=gHTGXAVlf56EwNzmw5zY0gbBf8h08eU2q2GBbb2FdVc,16087
|
|
19
19
|
deriva_ml/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
deriva_ml/schema_setup/annotations.py,sha256=v0gTpmWYxRqsQ-bcnQzsr8WowGv2pi9pZUsO3WWnu1U,9528
|
|
21
21
|
deriva_ml/schema_setup/create_schema.py,sha256=hNMc-v5tferd0UjfdB6nBw7Rc_o-Mg6NkPqQGie9YOw,11700
|
|
22
22
|
deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
|
|
23
23
|
deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
|
|
24
|
-
deriva_ml-
|
|
25
|
-
deriva_ml-
|
|
26
|
-
deriva_ml-1.13.
|
|
27
|
-
deriva_ml-1.13.
|
|
28
|
-
deriva_ml-1.13.
|
|
29
|
-
deriva_ml-1.13.
|
|
24
|
+
deriva_ml/test-files/execution-parameters.json,sha256=1vBqXlaMa0cysonE20TweVDfTGRdSi9CUuAkW1xiYNo,36
|
|
25
|
+
deriva_ml/test-files/notebook-parameters.json,sha256=7uEE2sLQSrSc9cEGQ_RKE7t5dwkEYv0qLo5mRbzo8Og,108
|
|
26
|
+
deriva_ml-1.13.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
27
|
+
deriva_ml-1.13.1.dist-info/METADATA,sha256=OKuCDvSR63ii7fO1W6tw-7-6RtYaKMHR59AbiURo_tI,999
|
|
28
|
+
deriva_ml-1.13.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
|
29
|
+
deriva_ml-1.13.1.dist-info/entry_points.txt,sha256=cJnALMa6pjdk6RQCt4HFbKHqALpVa0k6wPeQDPedLJI,295
|
|
30
|
+
deriva_ml-1.13.1.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
|
|
31
|
+
deriva_ml-1.13.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|