hpcflow-new2 0.2.0a107__py3-none-any.whl → 0.2.0a109__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/sdk/app.py +12 -1
- hpcflow/sdk/cli.py +15 -3
- hpcflow/sdk/cli_common.py +6 -0
- hpcflow/sdk/config/cli.py +26 -1
- hpcflow/sdk/config/config.py +15 -2
- hpcflow/sdk/core/actions.py +27 -15
- hpcflow/sdk/core/command_files.py +6 -12
- hpcflow/sdk/core/task.py +11 -0
- hpcflow/sdk/core/task_schema.py +21 -0
- hpcflow/sdk/core/workflow.py +14 -1
- hpcflow/sdk/demo/cli.py +7 -1
- hpcflow/tests/unit/test_action.py +37 -0
- hpcflow/tests/unit/test_task.py +7 -0
- {hpcflow_new2-0.2.0a107.dist-info → hpcflow_new2-0.2.0a109.dist-info}/METADATA +1 -1
- {hpcflow_new2-0.2.0a107.dist-info → hpcflow_new2-0.2.0a109.dist-info}/RECORD +18 -18
- {hpcflow_new2-0.2.0a107.dist-info → hpcflow_new2-0.2.0a109.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a107.dist-info → hpcflow_new2-0.2.0a109.dist-info}/entry_points.txt +0 -0
hpcflow/_version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.2.
|
1
|
+
__version__ = "0.2.0a109"
|
hpcflow/sdk/app.py
CHANGED
@@ -1108,6 +1108,7 @@ class BaseApp(metaclass=Singleton):
|
|
1108
1108
|
JS_parallelism: Optional[bool] = None,
|
1109
1109
|
wait: Optional[bool] = False,
|
1110
1110
|
add_to_known: Optional[bool] = True,
|
1111
|
+
return_idx: Optional[bool] = False,
|
1111
1112
|
) -> Dict[int, int]:
|
1112
1113
|
"""Generate and submit a new {app_name} workflow from a file or string containing a
|
1113
1114
|
workflow template parametrisation.
|
@@ -1150,6 +1151,9 @@ class BaseApp(metaclass=Singleton):
|
|
1150
1151
|
add_to_known
|
1151
1152
|
If True, add the new submission to the known-submissions file, which is
|
1152
1153
|
used by the `show` command to monitor current and recent submissions.
|
1154
|
+
return_idx
|
1155
|
+
If True, return a dict representing the jobscript indices submitted for each
|
1156
|
+
submission.
|
1153
1157
|
"""
|
1154
1158
|
|
1155
1159
|
self.API_logger.info("make_and_submit_workflow called")
|
@@ -1169,6 +1173,7 @@ class BaseApp(metaclass=Singleton):
|
|
1169
1173
|
JS_parallelism=JS_parallelism,
|
1170
1174
|
wait=wait,
|
1171
1175
|
add_to_known=add_to_known,
|
1176
|
+
return_idx=return_idx,
|
1172
1177
|
)
|
1173
1178
|
|
1174
1179
|
def _make_demo_workflow(
|
@@ -1240,6 +1245,7 @@ class BaseApp(metaclass=Singleton):
|
|
1240
1245
|
JS_parallelism: Optional[bool] = None,
|
1241
1246
|
wait: Optional[bool] = False,
|
1242
1247
|
add_to_known: Optional[bool] = True,
|
1248
|
+
return_idx: Optional[bool] = False,
|
1243
1249
|
) -> Dict[int, int]:
|
1244
1250
|
"""Generate and submit a new {app_name} workflow from a file or string containing a
|
1245
1251
|
workflow template parametrisation.
|
@@ -1279,6 +1285,9 @@ class BaseApp(metaclass=Singleton):
|
|
1279
1285
|
add_to_known
|
1280
1286
|
If True, add the new submission to the known-submissions file, which is
|
1281
1287
|
used by the `show` command to monitor current and recent submissions.
|
1288
|
+
return_idx
|
1289
|
+
If True, return a dict representing the jobscript indices submitted for each
|
1290
|
+
submission.
|
1282
1291
|
"""
|
1283
1292
|
|
1284
1293
|
self.API_logger.info("make_and_submit_demo_workflow called")
|
@@ -1297,6 +1306,7 @@ class BaseApp(metaclass=Singleton):
|
|
1297
1306
|
JS_parallelism=JS_parallelism,
|
1298
1307
|
wait=wait,
|
1299
1308
|
add_to_known=add_to_known,
|
1309
|
+
return_idx=return_idx,
|
1300
1310
|
)
|
1301
1311
|
|
1302
1312
|
def _submit_workflow(
|
@@ -1304,6 +1314,7 @@ class BaseApp(metaclass=Singleton):
|
|
1304
1314
|
workflow_path: PathLike,
|
1305
1315
|
JS_parallelism: Optional[bool] = None,
|
1306
1316
|
wait: Optional[bool] = False,
|
1317
|
+
return_idx: Optional[bool] = False,
|
1307
1318
|
) -> Dict[int, int]:
|
1308
1319
|
"""Submit an existing {app_name} workflow.
|
1309
1320
|
|
@@ -1319,7 +1330,7 @@ class BaseApp(metaclass=Singleton):
|
|
1319
1330
|
|
1320
1331
|
self.API_logger.info("submit_workflow called")
|
1321
1332
|
wk = self.Workflow(workflow_path)
|
1322
|
-
return wk.submit(JS_parallelism=JS_parallelism, wait=wait)
|
1333
|
+
return wk.submit(JS_parallelism=JS_parallelism, wait=wait, return_idx=return_idx)
|
1323
1334
|
|
1324
1335
|
def _run_hpcflow_tests(self, *args):
|
1325
1336
|
"""Run hpcflow test suite. This function is only available from derived apps."""
|
hpcflow/sdk/cli.py
CHANGED
@@ -22,6 +22,7 @@ from hpcflow.sdk.cli_common import (
|
|
22
22
|
js_parallelism_option,
|
23
23
|
wait_option,
|
24
24
|
add_to_known_opt,
|
25
|
+
print_idx_opt,
|
25
26
|
)
|
26
27
|
from hpcflow.sdk.helper.cli import get_helper_CLI
|
27
28
|
from hpcflow.sdk.submission.shells import ALL_SHELLS
|
@@ -104,6 +105,7 @@ def _make_API_CLI(app):
|
|
104
105
|
@js_parallelism_option
|
105
106
|
@wait_option
|
106
107
|
@add_to_known_opt
|
108
|
+
@print_idx_opt
|
107
109
|
def make_and_submit_workflow(
|
108
110
|
template_file_or_str,
|
109
111
|
string,
|
@@ -117,6 +119,7 @@ def _make_API_CLI(app):
|
|
117
119
|
js_parallelism=None,
|
118
120
|
wait=False,
|
119
121
|
add_to_known=True,
|
122
|
+
print_idx=False,
|
120
123
|
):
|
121
124
|
"""Generate and submit a new {app_name} workflow.
|
122
125
|
|
@@ -124,7 +127,7 @@ def _make_API_CLI(app):
|
|
124
127
|
format, or a YAML/JSON string.
|
125
128
|
|
126
129
|
"""
|
127
|
-
app.make_and_submit_workflow(
|
130
|
+
out = app.make_and_submit_workflow(
|
128
131
|
template_file_or_str=template_file_or_str,
|
129
132
|
is_string=string,
|
130
133
|
template_format=format,
|
@@ -137,7 +140,10 @@ def _make_API_CLI(app):
|
|
137
140
|
JS_parallelism=js_parallelism,
|
138
141
|
wait=wait,
|
139
142
|
add_to_known=add_to_known,
|
143
|
+
return_idx=print_idx,
|
140
144
|
)
|
145
|
+
if print_idx:
|
146
|
+
click.echo(out)
|
141
147
|
|
142
148
|
@click.command(context_settings={"ignore_unknown_options": True})
|
143
149
|
@click.argument("py_test_args", nargs=-1, type=click.UNPROCESSED)
|
@@ -288,14 +294,20 @@ def _make_workflow_CLI(app):
|
|
288
294
|
@js_parallelism_option
|
289
295
|
@wait_option
|
290
296
|
@add_to_known_opt
|
297
|
+
@print_idx_opt
|
291
298
|
@click.pass_context
|
292
|
-
def submit_workflow(
|
299
|
+
def submit_workflow(
|
300
|
+
ctx, js_parallelism=None, wait=False, add_to_known=True, print_idx=False
|
301
|
+
):
|
293
302
|
"""Submit the workflow."""
|
294
|
-
ctx.obj["workflow"].submit(
|
303
|
+
out = ctx.obj["workflow"].submit(
|
295
304
|
JS_parallelism=js_parallelism,
|
296
305
|
wait=wait,
|
297
306
|
add_to_known=add_to_known,
|
307
|
+
return_idx=print_idx,
|
298
308
|
)
|
309
|
+
if print_idx:
|
310
|
+
click.echo(out)
|
299
311
|
|
300
312
|
@workflow.command(name="wait")
|
301
313
|
@click.option(
|
hpcflow/sdk/cli_common.py
CHANGED
@@ -79,3 +79,9 @@ add_to_known_opt = click.option(
|
|
79
79
|
default=True,
|
80
80
|
help="If True, add this submission to the known-submissions file.",
|
81
81
|
)
|
82
|
+
print_idx_opt = click.option(
|
83
|
+
"--print-idx",
|
84
|
+
help="If True, print the submitted jobscript indices for each submission index.",
|
85
|
+
is_flag=True,
|
86
|
+
default=False,
|
87
|
+
)
|
hpcflow/sdk/config/cli.py
CHANGED
@@ -289,7 +289,32 @@ def get_config_CLI(app):
|
|
289
289
|
defaults = json.loads(defaults)
|
290
290
|
else:
|
291
291
|
defaults = {}
|
292
|
-
ctx.obj["config"].add_scheduler(name, defaults
|
292
|
+
ctx.obj["config"].add_scheduler(name, **defaults)
|
293
|
+
ctx.obj["config"].save()
|
294
|
+
|
295
|
+
@config.command()
|
296
|
+
@click.argument("name")
|
297
|
+
@click.option("--defaults")
|
298
|
+
@click.pass_context
|
299
|
+
@CLI_exception_wrapper_gen(ConfigError)
|
300
|
+
def add_shell(ctx, name, defaults):
|
301
|
+
if defaults:
|
302
|
+
defaults = json.loads(defaults)
|
303
|
+
else:
|
304
|
+
defaults = {}
|
305
|
+
ctx.obj["config"].add_shell(name, **defaults)
|
306
|
+
ctx.obj["config"].save()
|
307
|
+
|
308
|
+
@config.command()
|
309
|
+
@click.option("--defaults")
|
310
|
+
@click.pass_context
|
311
|
+
@CLI_exception_wrapper_gen(ConfigError)
|
312
|
+
def add_shell_wsl(ctx, defaults):
|
313
|
+
if defaults:
|
314
|
+
defaults = json.loads(defaults)
|
315
|
+
else:
|
316
|
+
defaults = {}
|
317
|
+
ctx.obj["config"].add_shell_WSL(**defaults)
|
293
318
|
ctx.obj["config"].save()
|
294
319
|
|
295
320
|
@config.command()
|
hpcflow/sdk/config/config.py
CHANGED
@@ -737,11 +737,24 @@ class Config:
|
|
737
737
|
self._logger.info(f"Resetting config file to defaults.")
|
738
738
|
self._app.reset_config()
|
739
739
|
|
740
|
-
def add_scheduler(self, scheduler, **
|
740
|
+
def add_scheduler(self, scheduler, **defaults):
|
741
741
|
if scheduler in self.get("schedulers"):
|
742
742
|
print(f"Scheduler {scheduler!r} already exists.")
|
743
743
|
return
|
744
|
-
self.update(f"schedulers.{scheduler}",
|
744
|
+
self.update(f"schedulers.{scheduler}.defaults", defaults)
|
745
|
+
|
746
|
+
def add_shell(self, shell, **defaults):
|
747
|
+
if shell in self.get("shells"):
|
748
|
+
return
|
749
|
+
if shell.lower() == "wsl":
|
750
|
+
# check direct_posix scheduler is added:
|
751
|
+
self.add_scheduler("direct_posix")
|
752
|
+
self.update(f"shells.{shell}.defaults", defaults)
|
753
|
+
|
754
|
+
def add_shell_WSL(self, **defaults):
|
755
|
+
if "WSL_executable" not in defaults:
|
756
|
+
defaults["WSL_executable"] = "wsl.exe"
|
757
|
+
self.add_shell("wsl", **defaults)
|
745
758
|
|
746
759
|
def import_from_file(self, file_path, rename=True, make_new=False):
|
747
760
|
"""Import config items from a (remote or local) YAML file. Existing config items
|
hpcflow/sdk/core/actions.py
CHANGED
@@ -513,16 +513,14 @@ class ElementActionRun:
|
|
513
513
|
outputs[out_typ] = self.get(f"outputs.{out_typ}")
|
514
514
|
return outputs
|
515
515
|
|
516
|
-
def compose_source(self) -> str:
|
516
|
+
def compose_source(self, snip_path: Path) -> str:
|
517
517
|
"""Generate the file contents of this source."""
|
518
518
|
|
519
|
-
script_name =
|
520
|
-
|
521
|
-
script_path = self.app.scripts.get(script_key)
|
522
|
-
with script_path.open("rt") as fp:
|
519
|
+
script_name = snip_path.name
|
520
|
+
with snip_path.open("rt") as fp:
|
523
521
|
script_str = fp.read()
|
524
522
|
|
525
|
-
is_python =
|
523
|
+
is_python = snip_path.suffix == ".py"
|
526
524
|
if is_python:
|
527
525
|
py_imports = dedent(
|
528
526
|
"""\
|
@@ -639,10 +637,12 @@ class ElementActionRun:
|
|
639
637
|
|
640
638
|
# write the script if it is specified as a app data script, otherwise we assume
|
641
639
|
# the script already exists in the working directory:
|
642
|
-
|
643
|
-
|
640
|
+
snip_path = self.action.get_snippet_script_path(self.action.script)
|
641
|
+
if snip_path:
|
642
|
+
script_name = snip_path.name
|
643
|
+
source_str = self.compose_source(snip_path)
|
644
644
|
with Path(script_name).open("wt", newline="\n") as fp:
|
645
|
-
fp.write(
|
645
|
+
fp.write(source_str)
|
646
646
|
|
647
647
|
def _param_dump_JSON(self, dump_path: Path):
|
648
648
|
with dump_path.open("wt") as fp:
|
@@ -1358,15 +1358,16 @@ class Action(JSONLike):
|
|
1358
1358
|
)
|
1359
1359
|
|
1360
1360
|
@staticmethod
|
1361
|
-
def
|
1361
|
+
def is_snippet_script(script: str) -> bool:
|
1362
|
+
"""Returns True if the provided script string represents a script snippets that is
|
1363
|
+
to be modified before execution (e.g. to receive and provide parameter data)."""
|
1362
1364
|
return script.startswith("<<script:")
|
1363
1365
|
|
1364
1366
|
@classmethod
|
1365
1367
|
def get_script_name(cls, script: str) -> str:
|
1366
1368
|
"""Return the script name."""
|
1367
|
-
if cls.
|
1368
|
-
|
1369
|
-
pattern = r"\<\<script:(?:.*\/)*(.*:?)\>\>"
|
1369
|
+
if cls.is_snippet_script(script):
|
1370
|
+
pattern = r"\<\<script:(?:.*(?:\/|\\))*(.*)\>\>"
|
1370
1371
|
match_obj = re.match(pattern, script)
|
1371
1372
|
return match_obj.group(1)
|
1372
1373
|
else:
|
@@ -1374,8 +1375,8 @@ class Action(JSONLike):
|
|
1374
1375
|
return script
|
1375
1376
|
|
1376
1377
|
@classmethod
|
1377
|
-
def
|
1378
|
-
if not cls.
|
1378
|
+
def get_snippet_script_str(cls, script) -> str:
|
1379
|
+
if not cls.is_snippet_script(script):
|
1379
1380
|
raise ValueError(
|
1380
1381
|
f"Must be an app-data script name (e.g. "
|
1381
1382
|
f"<<script:path/to/app/data/script.py>>), but received {script}"
|
@@ -1384,6 +1385,17 @@ class Action(JSONLike):
|
|
1384
1385
|
match_obj = re.match(pattern, script)
|
1385
1386
|
return match_obj.group(1)
|
1386
1387
|
|
1388
|
+
@classmethod
|
1389
|
+
def get_snippet_script_path(cls, script_path) -> Path:
|
1390
|
+
if not cls.is_snippet_script(script_path):
|
1391
|
+
return False
|
1392
|
+
|
1393
|
+
path = cls.get_snippet_script_str(script_path)
|
1394
|
+
if path in cls.app.scripts:
|
1395
|
+
path = cls.app.scripts.get(path)
|
1396
|
+
|
1397
|
+
return Path(path)
|
1398
|
+
|
1387
1399
|
@staticmethod
|
1388
1400
|
def get_param_dump_file_stem(js_idx: int, js_act_idx: int):
|
1389
1401
|
return f"js_{js_idx}_act_{js_act_idx}_inputs"
|
@@ -134,12 +134,9 @@ class InputFileGenerator(JSONLike):
|
|
134
134
|
def compose_source(self, action) -> str:
|
135
135
|
"""Generate the file contents of this input file generator source."""
|
136
136
|
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
script_main_func = Path(script_name).stem
|
141
|
-
|
142
|
-
with script_path.open("rt") as fp:
|
137
|
+
snip_path = action.get_snippet_script_path(self.script)
|
138
|
+
script_main_func = snip_path.stem
|
139
|
+
with snip_path.open("rt") as fp:
|
143
140
|
script_str = fp.read()
|
144
141
|
|
145
142
|
main_block = dedent(
|
@@ -261,12 +258,9 @@ class OutputFileParser(JSONLike):
|
|
261
258
|
def compose_source(self, action) -> str:
|
262
259
|
"""Generate the file contents of this output file parser source."""
|
263
260
|
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
script_main_func = Path(script_name).stem
|
268
|
-
|
269
|
-
with script_path.open("rt") as fp:
|
261
|
+
snip_path = action.get_snippet_script_path(self.script)
|
262
|
+
script_main_func = snip_path.stem
|
263
|
+
with snip_path.open("rt") as fp:
|
270
264
|
script_str = fp.read()
|
271
265
|
|
272
266
|
main_block = dedent(
|
hpcflow/sdk/core/task.py
CHANGED
@@ -222,6 +222,17 @@ class ElementSet(JSONLike):
|
|
222
222
|
return tuple(self._element_local_idx_range)
|
223
223
|
|
224
224
|
def _validate(self):
|
225
|
+
|
226
|
+
# support inputs passed as a dict:
|
227
|
+
_inputs = []
|
228
|
+
try:
|
229
|
+
for k, v in self.inputs.items():
|
230
|
+
_inputs.append(self.app.InputValue(parameter=k, value=v))
|
231
|
+
except AttributeError:
|
232
|
+
pass
|
233
|
+
else:
|
234
|
+
self.inputs = _inputs
|
235
|
+
|
225
236
|
inp_paths = [i.normalised_inputs_path for i in self.inputs]
|
226
237
|
dup_inp_paths = get_duplicate_items(inp_paths)
|
227
238
|
if dup_inp_paths:
|
hpcflow/sdk/core/task_schema.py
CHANGED
@@ -217,6 +217,27 @@ class TaskSchema(JSONLike):
|
|
217
217
|
tab_cmds_i = Table(show_header=False, box=None)
|
218
218
|
tab_cmds_i.add_column(justify="right")
|
219
219
|
tab_cmds_i.add_column()
|
220
|
+
if act.rules:
|
221
|
+
seen_rules = [] # bug: some rules seem to be repeated
|
222
|
+
for act_rule_j in act.rules:
|
223
|
+
if act_rule_j.rule in seen_rules:
|
224
|
+
continue
|
225
|
+
else:
|
226
|
+
seen_rules.append(act_rule_j.rule)
|
227
|
+
r_path = ""
|
228
|
+
if act_rule_j.rule.check_missing:
|
229
|
+
r_cond = f"check missing: {act_rule_j.rule.check_missing}"
|
230
|
+
elif act_rule_j.rule.check_exists:
|
231
|
+
r_cond = f"check exists: {act_rule_j.rule.check_exists}"
|
232
|
+
elif act_rule_j.rule.condition:
|
233
|
+
r_path = f"{act_rule_j.rule.path}: "
|
234
|
+
r_cond = str(act_rule_j.rule.condition.to_json_like())
|
235
|
+
else:
|
236
|
+
continue
|
237
|
+
tab_cmds_i.add_row(
|
238
|
+
"[italic]rule:[/italic]",
|
239
|
+
escape(f"{r_path}{r_cond}"),
|
240
|
+
)
|
220
241
|
tab_cmds_i.add_row(
|
221
242
|
"[italic]scope:[/italic]",
|
222
243
|
escape(act.get_precise_scope().to_string()),
|
hpcflow/sdk/core/workflow.py
CHANGED
@@ -127,6 +127,7 @@ class WorkflowTemplate(JSONLike):
|
|
127
127
|
loops: Optional[List[app.Loop]] = field(default_factory=lambda: [])
|
128
128
|
workflow: Optional[app.Workflow] = None
|
129
129
|
resources: Optional[Dict[str, Dict]] = None
|
130
|
+
source_file: Optional[str] = field(default=None, compare=False)
|
130
131
|
|
131
132
|
def __post_init__(self):
|
132
133
|
self.resources = self.app.ResourceList.normalise(self.resources)
|
@@ -230,6 +231,7 @@ class WorkflowTemplate(JSONLike):
|
|
230
231
|
cls.app.logger.debug("parsing workflow template from a YAML file")
|
231
232
|
data = read_YAML_file(path)
|
232
233
|
cls._check_name(data, path)
|
234
|
+
data["source_file"] = str(path)
|
233
235
|
return cls._from_data(data)
|
234
236
|
|
235
237
|
@classmethod
|
@@ -257,6 +259,7 @@ class WorkflowTemplate(JSONLike):
|
|
257
259
|
cls.app.logger.debug("parsing workflow template from a JSON file")
|
258
260
|
data = read_JSON_file(path)
|
259
261
|
cls._check_name(data, path)
|
262
|
+
data["source_file"] = str(path)
|
260
263
|
return cls._from_data(data)
|
261
264
|
|
262
265
|
@classmethod
|
@@ -1359,6 +1362,11 @@ class Workflow:
|
|
1359
1362
|
# actually make template inputs/resources persistent, now the workflow exists:
|
1360
1363
|
wk_dummy.make_persistent(wk)
|
1361
1364
|
|
1365
|
+
if template.source_file:
|
1366
|
+
wk.artifacts_path.mkdir(exist_ok=False)
|
1367
|
+
src = Path(template.source_file)
|
1368
|
+
wk.artifacts_path.joinpath(src.name).write_text(src.read_text())
|
1369
|
+
|
1362
1370
|
return wk
|
1363
1371
|
|
1364
1372
|
def to_zip(self, log=None) -> str:
|
@@ -1805,6 +1813,7 @@ class Workflow:
|
|
1805
1813
|
print_stdout: Optional[bool] = False,
|
1806
1814
|
wait: Optional[bool] = False,
|
1807
1815
|
add_to_known: Optional[bool] = True,
|
1816
|
+
return_idx: Optional[bool] = False,
|
1808
1817
|
) -> Dict[int, int]:
|
1809
1818
|
"""Submit the workflow for execution.
|
1810
1819
|
|
@@ -1824,6 +1833,9 @@ class Workflow:
|
|
1824
1833
|
add_to_known
|
1825
1834
|
If True, add the submitted submissions to the known-submissions file, which is
|
1826
1835
|
used by the `show` command to monitor current and recent submissions.
|
1836
|
+
return_idx
|
1837
|
+
If True, return a dict representing the jobscript indices submitted for each
|
1838
|
+
submission.
|
1827
1839
|
"""
|
1828
1840
|
|
1829
1841
|
console = rich.console.Console()
|
@@ -1858,7 +1870,8 @@ class Workflow:
|
|
1858
1870
|
if wait:
|
1859
1871
|
self.wait(submitted_js)
|
1860
1872
|
|
1861
|
-
|
1873
|
+
if return_idx:
|
1874
|
+
return submitted_js
|
1862
1875
|
|
1863
1876
|
def wait(self, sub_js: Optional[Dict] = None):
|
1864
1877
|
"""Wait for the completion of specified/all submitted jobscripts."""
|
hpcflow/sdk/demo/cli.py
CHANGED
@@ -14,6 +14,7 @@ from hpcflow.sdk.cli_common import (
|
|
14
14
|
js_parallelism_option,
|
15
15
|
wait_option,
|
16
16
|
add_to_known_opt,
|
17
|
+
print_idx_opt,
|
17
18
|
)
|
18
19
|
|
19
20
|
|
@@ -130,6 +131,7 @@ def get_demo_workflow_CLI(app):
|
|
130
131
|
@js_parallelism_option
|
131
132
|
@wait_option
|
132
133
|
@add_to_known_opt
|
134
|
+
@print_idx_opt
|
133
135
|
def make_and_submit_demo_workflow(
|
134
136
|
workflow_name,
|
135
137
|
format,
|
@@ -142,8 +144,9 @@ def get_demo_workflow_CLI(app):
|
|
142
144
|
js_parallelism=None,
|
143
145
|
wait=False,
|
144
146
|
add_to_known=True,
|
147
|
+
print_idx=False,
|
145
148
|
):
|
146
|
-
app.make_and_submit_demo_workflow(
|
149
|
+
out = app.make_and_submit_demo_workflow(
|
147
150
|
workflow_name=workflow_name,
|
148
151
|
template_format=format,
|
149
152
|
path=path,
|
@@ -155,7 +158,10 @@ def get_demo_workflow_CLI(app):
|
|
155
158
|
JS_parallelism=js_parallelism,
|
156
159
|
wait=wait,
|
157
160
|
add_to_known=add_to_known,
|
161
|
+
return_idx=print_idx,
|
158
162
|
)
|
163
|
+
if print_idx:
|
164
|
+
click.echo(out)
|
159
165
|
|
160
166
|
@demo_workflow.command("copy")
|
161
167
|
@click.argument("workflow_name")
|
@@ -1,3 +1,4 @@
|
|
1
|
+
from pathlib import Path
|
1
2
|
import pytest
|
2
3
|
|
3
4
|
from hpcflow.app import app as hf
|
@@ -215,3 +216,39 @@ def test_get_command_input_types_label_sub_parameters_false_no_sub_param():
|
|
215
216
|
def test_get_command_input_types_label_sub_parameters_false_with_sub_parameter():
|
216
217
|
act = hf.Action(commands=[hf.Command("Write-Output (<<parameter:p1[one].a>> + 100)")])
|
217
218
|
assert act.get_command_input_types(sub_parameters=False) == ("p1[one]",)
|
219
|
+
|
220
|
+
|
221
|
+
def test_get_script_name(null_config):
|
222
|
+
expected = {
|
223
|
+
"<<script:/software/hello.py>>": "hello.py",
|
224
|
+
"<<script:software/hello.py>>": "hello.py",
|
225
|
+
r"<<script:C:\long\path\to\script.py>>": "script.py",
|
226
|
+
"/path/to/script.py": "/path/to/script.py",
|
227
|
+
}
|
228
|
+
for k, v in expected.items():
|
229
|
+
assert hf.Action.get_script_name(k) == v
|
230
|
+
|
231
|
+
|
232
|
+
def test_is_snippet_script(null_config):
|
233
|
+
expected = {
|
234
|
+
"<<script:/software/hello.py>>": True,
|
235
|
+
"<<script:software/hello.py>>": True,
|
236
|
+
r"<<script:C:\long\path\to\script.py>>": True,
|
237
|
+
"/path/to/script.py": False,
|
238
|
+
}
|
239
|
+
for k, v in expected.items():
|
240
|
+
assert hf.Action.is_snippet_script(k) == v
|
241
|
+
|
242
|
+
|
243
|
+
def test_get_snippet_script_path(null_config):
|
244
|
+
expected = {
|
245
|
+
"<<script:/software/hello.py>>": Path("/software/hello.py"),
|
246
|
+
"<<script:software/hello.py>>": Path("software/hello.py"),
|
247
|
+
r"<<script:C:\long\path\to\script.py>>": Path(r"C:\long\path\to\script.py"),
|
248
|
+
}
|
249
|
+
for k, v in expected.items():
|
250
|
+
assert hf.Action.get_snippet_script_path(k) == v
|
251
|
+
|
252
|
+
|
253
|
+
def test_get_snippet_script_path_False(null_config):
|
254
|
+
assert not hf.Action.get_snippet_script_path("/path/to/script.py")
|
hpcflow/tests/unit/test_task.py
CHANGED
@@ -2083,3 +2083,10 @@ def test_path_to_PV_classes_resources_path_ignored(path_to_PV_classes_workflow):
|
|
2083
2083
|
assert path_to_PV_classes_workflow.tasks.t1._paths_to_PV_classes(
|
2084
2084
|
paths_1
|
2085
2085
|
) == path_to_PV_classes_workflow.tasks.t1._paths_to_PV_classes(paths_2)
|
2086
|
+
|
2087
|
+
|
2088
|
+
def test_input_values_specified_by_dict(null_config):
|
2089
|
+
ts = hf.TaskSchema(objective="t1", inputs=[hf.SchemaInput("p1")])
|
2090
|
+
t1 = hf.Task(schema=ts, inputs=[hf.InputValue(parameter="p1", value=101)])
|
2091
|
+
t2 = hf.Task(schema=ts, inputs={"p1": 101})
|
2092
|
+
assert t1 == t2
|
@@ -1,23 +1,23 @@
|
|
1
1
|
hpcflow/__init__.py,sha256=WIETuRHeOp2SqUqHUzpjQ-lk9acbYv-6aWOhZPRdlhs,64
|
2
2
|
hpcflow/__pyinstaller/__init__.py,sha256=YOzBlPSck6slucv6lJM9K80JtsJWxXRL00cv6tRj3oc,98
|
3
3
|
hpcflow/__pyinstaller/hook-hpcflow.py,sha256=xyEXMIKYRpy0cf3N2ksLEbZxNb6Bw9WOYbLuuj4t1qM,924
|
4
|
-
hpcflow/_version.py,sha256=
|
4
|
+
hpcflow/_version.py,sha256=Nkwb1thRPq6eII0fnUVrpgGvVYJ0wpqCGaJ0ybaTlvE,26
|
5
5
|
hpcflow/app.py,sha256=CMr8XlaGbAEQ0uvKrmnhnYw7TSQ7KhVkOUbLioecjL4,1377
|
6
6
|
hpcflow/cli.py,sha256=G2J3D9v6MnMWOWMMWK6UEKLn_6wnV9lT_qygEBBxg-I,66
|
7
7
|
hpcflow/examples.ipynb,sha256=cLKp4QsxwwMXRanDnfWY9kqsV23q6G4raOpu6IZXnMw,28553
|
8
8
|
hpcflow/sdk/__init__.py,sha256=QN1hUM9MdJ5yB3ou8q3ky530TdSd8BJi1V4XI2NmT4k,5643
|
9
|
-
hpcflow/sdk/app.py,sha256=
|
10
|
-
hpcflow/sdk/cli.py,sha256=
|
11
|
-
hpcflow/sdk/cli_common.py,sha256=
|
9
|
+
hpcflow/sdk/app.py,sha256=Ov6XFxcdQqwhntSqEhb0hcaEDmn2dT4XoJApfuyFVsE,72773
|
10
|
+
hpcflow/sdk/cli.py,sha256=ivH02wh1iEWRqwYErajGyUEnBfZPK8UvR-R2HOErwP8,28299
|
11
|
+
hpcflow/sdk/cli_common.py,sha256=pBzclQ9GAHctDG6abX-pxrQDPVRu1GD7BTX5awI8-iY,2661
|
12
12
|
hpcflow/sdk/config/__init__.py,sha256=qJrrxcAN4f1u_RyTtXgz-xlTLwNafE9v0VEMP1x6-bU,70
|
13
13
|
hpcflow/sdk/config/callbacks.py,sha256=7z0rFX7ULUTAo24IJr6kLzPmZCgmkaQ8P5e-AfYTcY8,5339
|
14
|
-
hpcflow/sdk/config/cli.py,sha256=
|
15
|
-
hpcflow/sdk/config/config.py,sha256=
|
14
|
+
hpcflow/sdk/config/cli.py,sha256=jep8rypMvmTqph-BzqUI-nZSW4Q8_gS4-VTs3WwVCr8,10383
|
15
|
+
hpcflow/sdk/config/config.py,sha256=0HLUtoah3kWm985rwuvvOsoIQxEw0yzm838kfcv3O90,29666
|
16
16
|
hpcflow/sdk/config/config_file.py,sha256=JlMcprj0aujFVk8552ahP2f8EXB0tglMaHwzbcGZH6w,12373
|
17
17
|
hpcflow/sdk/config/errors.py,sha256=2D7HJ1dbyeoD3xk4MuaGSsbJsUyQzyw8kaThEBZfP2I,6876
|
18
18
|
hpcflow/sdk/core/__init__.py,sha256=GcIklEsXy3M5PWpmxyhd2KoI0u6HjXRIjD_aR1bgRjo,215
|
19
|
-
hpcflow/sdk/core/actions.py,sha256=
|
20
|
-
hpcflow/sdk/core/command_files.py,sha256=
|
19
|
+
hpcflow/sdk/core/actions.py,sha256=r8ZiPsOYmgdY7DIywjoK-_c6gos4JHoqhzQreQozgDs,63480
|
20
|
+
hpcflow/sdk/core/command_files.py,sha256=vdxpQYDfyg2EoytsTD5NzsOsZNypHwBq0Gu6IkQSnGo,16935
|
21
21
|
hpcflow/sdk/core/commands.py,sha256=1nVU_E7QntuY0FiNluhf_ouxu3ML4KoRqwvbfsqXOz0,11061
|
22
22
|
hpcflow/sdk/core/element.py,sha256=HBX1mGpgr_UNFvvC4zJJ1tI6SH2qCQxA0-_3Onh6SYQ,44524
|
23
23
|
hpcflow/sdk/core/environment.py,sha256=DGUz1NvliKh6opP0IueGHD69rn_8wFLhDsq6kAmEgM4,4849
|
@@ -28,12 +28,12 @@ hpcflow/sdk/core/object_list.py,sha256=S52GCgS8QxdNIX-SmCS8nCevu227IdYpyMUZ4Uw-C
|
|
28
28
|
hpcflow/sdk/core/parallel.py,sha256=LI-g-qOuOR1oaEUWVT0qW0hmiP9hsJyUP8_IfSTKYYo,95
|
29
29
|
hpcflow/sdk/core/parameters.py,sha256=KZc0BLqPkbMIcsCerDgQ1zPQQ33s0LsCUfmR_hFy0GM,60462
|
30
30
|
hpcflow/sdk/core/rule.py,sha256=chMn-Unu_4Mtc3T4z93OpmYcqOapqYAdBWuRSP8CBdg,4510
|
31
|
-
hpcflow/sdk/core/task.py,sha256=
|
32
|
-
hpcflow/sdk/core/task_schema.py,sha256=
|
31
|
+
hpcflow/sdk/core/task.py,sha256=MIleH_ahiLlPe1cDRAJ4rolFJLN0ixum_faJRMSB7Jk,103373
|
32
|
+
hpcflow/sdk/core/task_schema.py,sha256=Z6yxywwVmF8a4ITumhqC8mZNwUYk32UHB7UMY4SGKkU,18471
|
33
33
|
hpcflow/sdk/core/test_utils.py,sha256=Zrun61DdJWngVMneXhpJCp6nXbjS0OLfoSM1BT_55VI,6660
|
34
34
|
hpcflow/sdk/core/utils.py,sha256=QHzVae2b3JheweeNWHbQjKB-kFHRwv0pBYpP2HY1QGo,19836
|
35
35
|
hpcflow/sdk/core/validation.py,sha256=KBKiy5DdfGiGmMaB0HdKTY0V972u5dJzvkYkX0_KtCo,518
|
36
|
-
hpcflow/sdk/core/workflow.py,sha256=
|
36
|
+
hpcflow/sdk/core/workflow.py,sha256=4z5dgthfNfLa2VMzgSEgMMFCXWfeiWIq3OW7oVnuhaw,92589
|
37
37
|
hpcflow/sdk/core/zarr_io.py,sha256=V_Zm6uSiuaCbXyHFJUO74K1pAr4Zqrj3aLCBjohCwvs,5724
|
38
38
|
hpcflow/sdk/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
39
39
|
hpcflow/sdk/data/config_file_schema.yaml,sha256=7i3z_m3GBRtLyB4c7qPngnlQWqcIq1CyCcOysDyq4es,791
|
@@ -49,7 +49,7 @@ hpcflow/sdk/data/template_components/parameters.yaml,sha256=rJ-3f1GRiNGS6oDV9T1-
|
|
49
49
|
hpcflow/sdk/data/template_components/task_schemas.yaml,sha256=VMtzckqDyfL9JpjYkG9FkzlVGYxaFvPo307Sx6ctVFk,3217
|
50
50
|
hpcflow/sdk/data/workflow_spec_schema.yaml,sha256=LLUV6RsdgT2BQrEPNKLCUB5Uijt9wxT9Bac2PLyGha8,1797
|
51
51
|
hpcflow/sdk/demo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
52
|
-
hpcflow/sdk/demo/cli.py,sha256=
|
52
|
+
hpcflow/sdk/demo/cli.py,sha256=iXQgzHfXsktNz7kH1zTzBdH03kmN4z4uOF8PcfvbR2Q,5090
|
53
53
|
hpcflow/sdk/demo/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
54
54
|
hpcflow/sdk/demo/data/workflow_1.json,sha256=xdqa8nIOxNN-A2F7wJX7LTOT3KOEZmulaa-9tsSB-nk,119
|
55
55
|
hpcflow/sdk/demo/data/workflow_1.yaml,sha256=7mGwCCQX3VBWcpSYecAKqz_Gd-IODjhOlDNPh95Rw2U,85
|
@@ -98,7 +98,7 @@ hpcflow/tests/conftest.py,sha256=Xjyq9tjVdFGIf5HD1G0iZ1V_-cJAYC6A240OX8eBexU,261
|
|
98
98
|
hpcflow/tests/schedulers/direct_linux/test_direct_linux_submission.py,sha256=pgHHG4iak0tx-1JTtpo8sCIvcZF2XayzEysjqWa_9LM,456
|
99
99
|
hpcflow/tests/schedulers/slurm/test_slurm_submission.py,sha256=XMv_0YQ6GUeDPpQ4tO2LEm_kT_A40t_JzK8v-gl-ji8,442
|
100
100
|
hpcflow/tests/shells/wsl/test_wsl_submission.py,sha256=cRkl_1H2Y4k91TJOX9wXZthW8AJFuuWuQ8X98624JDw,589
|
101
|
-
hpcflow/tests/unit/test_action.py,sha256=
|
101
|
+
hpcflow/tests/unit/test_action.py,sha256=qh8v9GhFxkK7zmBqHyjWQh6vlJYXADLysPE_qhmvEqo,9078
|
102
102
|
hpcflow/tests/unit/test_action_rule.py,sha256=r3G7-EEdsF1ND6kCOGI0S6N9WLi1vd3OABRIp6EvtMA,547
|
103
103
|
hpcflow/tests/unit/test_app.py,sha256=W89TvXt9AD9AEn0tBkZgmEpiEeg1R_SpuQLV2tjQSAs,2916
|
104
104
|
hpcflow/tests/unit/test_cli.py,sha256=9oQZOlX0z5LC4e2JFLuIgGnUXgmR2RCAtbXR5XRwqJs,288
|
@@ -120,7 +120,7 @@ hpcflow/tests/unit/test_runtime.py,sha256=yhTX0XF_-_qgruMG_wZZNCMCs8dUw5sMkXlsN5
|
|
120
120
|
hpcflow/tests/unit/test_schema_input.py,sha256=_OS5EnVGwL15-3V0vXaPxzigXVltRxLUnpkc0qq3SsQ,6658
|
121
121
|
hpcflow/tests/unit/test_shell.py,sha256=FDtQ9fHRhSKiVtxMJ8BRisoeSvvk8zmJndTB4LlhqGc,3442
|
122
122
|
hpcflow/tests/unit/test_submission.py,sha256=OloEF4vpNLW4KtQCb_f1Xgkh7t8k4MiAgVG375hTdBk,5825
|
123
|
-
hpcflow/tests/unit/test_task.py,sha256=
|
123
|
+
hpcflow/tests/unit/test_task.py,sha256=WaqwlbUJa38dc9rUdGsiWt2gKNM2fG698MxQ_8fDbNc,67360
|
124
124
|
hpcflow/tests/unit/test_task_schema.py,sha256=7a7o42gQhrZPMXfH0a6sGzFCJnuFrbDEl9u3u_bFsgw,3624
|
125
125
|
hpcflow/tests/unit/test_utils.py,sha256=C-0rd7-gEZY_TJItJcSdN1TVPuHsOUfVfl0NscFRg8I,11200
|
126
126
|
hpcflow/tests/unit/test_value_sequence.py,sha256=Kf90_FJVoAvXvDJ0y_z8_vAqib6e43fMTVMtbl75PZ4,14127
|
@@ -128,7 +128,7 @@ hpcflow/tests/unit/test_workflow.py,sha256=zHM9KCkbEupKocrnTOkYagas7amXSpj9QNuEG
|
|
128
128
|
hpcflow/tests/unit/test_workflow_template.py,sha256=4AFA2d4ZYOHdOJEH485D-fg8x1O-psZ0Qr2kOlrCc88,963
|
129
129
|
hpcflow/tests/workflows/test_workflows.py,sha256=xWx2x70_3WdGR0ijgjk2F5Tg_V9YRCgN0E02ZFUi9Hk,13390
|
130
130
|
hpcflow/viz_demo.ipynb,sha256=1QdnVsk72vihv2L6hOGyk318uEa22ZSgGxQCa7hW2oo,6238
|
131
|
-
hpcflow_new2-0.2.
|
132
|
-
hpcflow_new2-0.2.
|
133
|
-
hpcflow_new2-0.2.
|
134
|
-
hpcflow_new2-0.2.
|
131
|
+
hpcflow_new2-0.2.0a109.dist-info/METADATA,sha256=Km5lIPZ33qmI4LpUEyqcMvh58koOEjj0tXrqJF19rhw,1879
|
132
|
+
hpcflow_new2-0.2.0a109.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
|
133
|
+
hpcflow_new2-0.2.0a109.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
|
134
|
+
hpcflow_new2-0.2.0a109.dist-info/RECORD,,
|
File without changes
|
File without changes
|