hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a190__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/sdk/__init__.py +21 -15
- hpcflow/sdk/app.py +2133 -770
- hpcflow/sdk/cli.py +281 -250
- hpcflow/sdk/cli_common.py +6 -2
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +77 -42
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +578 -311
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +112 -85
- hpcflow/sdk/config/types.py +145 -0
- hpcflow/sdk/core/actions.py +1054 -994
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +81 -63
- hpcflow/sdk/core/command_files.py +275 -185
- hpcflow/sdk/core/commands.py +111 -107
- hpcflow/sdk/core/element.py +724 -503
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +398 -51
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +380 -334
- hpcflow/sdk/core/loop_cache.py +160 -43
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +728 -600
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +33 -22
- hpcflow/sdk/core/task.py +1546 -1325
- hpcflow/sdk/core/task_schema.py +240 -196
- hpcflow/sdk/core/test_utils.py +126 -88
- hpcflow/sdk/core/types.py +387 -0
- hpcflow/sdk/core/utils.py +410 -305
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +1192 -1028
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/demo/cli.py +46 -33
- hpcflow/sdk/helper/cli.py +18 -16
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +83 -59
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +988 -586
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +408 -153
- hpcflow/sdk/persistence/pending.py +158 -123
- hpcflow/sdk/persistence/store_resource.py +37 -22
- hpcflow/sdk/persistence/types.py +307 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +477 -420
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +444 -404
- hpcflow/sdk/submission/schedulers/__init__.py +133 -40
- hpcflow/sdk/submission/schedulers/direct.py +97 -71
- hpcflow/sdk/submission/schedulers/sge.py +132 -126
- hpcflow/sdk/submission/schedulers/slurm.py +263 -268
- hpcflow/sdk/submission/schedulers/utils.py +7 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +102 -29
- hpcflow/sdk/submission/shells/bash.py +72 -55
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +37 -29
- hpcflow/sdk/submission/submission.py +203 -257
- hpcflow/sdk/submission/types.py +143 -0
- hpcflow/sdk/typing.py +163 -12
- hpcflow/tests/conftest.py +8 -6
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_main_scripts.py +60 -30
- hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
- hpcflow/tests/unit/test_action.py +86 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +13 -6
- hpcflow/tests/unit/test_cli.py +1 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +20 -15
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +3 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +65 -58
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +16 -7
- hpcflow/tests/unit/test_persistence.py +48 -35
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +8 -3
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +3 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +39 -19
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/workflows/test_jobscript.py +2 -1
- hpcflow/tests/workflows/test_workflows.py +18 -13
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
- hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
hpcflow/tests/unit/test_loop.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
from pathlib import Path
|
1
3
|
import pytest
|
2
4
|
|
3
|
-
from valida.conditions import Value
|
4
|
-
|
5
|
+
from valida.conditions import Value # type: ignore
|
5
6
|
|
6
7
|
from hpcflow.app import app as hf
|
7
8
|
from hpcflow.sdk.core.errors import LoopAlreadyExistsError, LoopTaskSubsetError
|
@@ -9,9 +10,9 @@ from hpcflow.sdk.core.test_utils import P1_parameter_cls, make_workflow
|
|
9
10
|
|
10
11
|
|
11
12
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
12
|
-
def test_loop_tasks_obj_insert_ID_equivalence(tmp_path, store):
|
13
|
+
def test_loop_tasks_obj_insert_ID_equivalence(tmp_path: Path, store: str):
|
13
14
|
wk_1 = make_workflow(
|
14
|
-
schemas_spec=[
|
15
|
+
schemas_spec=[({"p1": None}, ("p1",), "t1")],
|
15
16
|
local_inputs={0: ("p1",)},
|
16
17
|
path=tmp_path,
|
17
18
|
store=store,
|
@@ -21,9 +22,9 @@ def test_loop_tasks_obj_insert_ID_equivalence(tmp_path, store):
|
|
21
22
|
assert lp_0.task_insert_IDs == lp_1.task_insert_IDs
|
22
23
|
|
23
24
|
|
24
|
-
def test_raise_on_add_loop_same_name(tmp_path):
|
25
|
+
def test_raise_on_add_loop_same_name(tmp_path: Path):
|
25
26
|
wk = make_workflow(
|
26
|
-
schemas_spec=[
|
27
|
+
schemas_spec=[({"p1": None}, ("p1",), "t1"), ({"p2": None}, ("p2",), "t2")],
|
27
28
|
local_inputs={0: ("p1",), 1: ("p2",)},
|
28
29
|
path=tmp_path,
|
29
30
|
store="json",
|
@@ -38,10 +39,10 @@ def test_raise_on_add_loop_same_name(tmp_path):
|
|
38
39
|
|
39
40
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
40
41
|
def test_wk_loop_data_idx_single_task_single_element_single_parameter_three_iters(
|
41
|
-
tmp_path, store
|
42
|
+
tmp_path: Path, store: str
|
42
43
|
):
|
43
44
|
wk = make_workflow(
|
44
|
-
schemas_spec=[
|
45
|
+
schemas_spec=[({"p1": None}, ("p1",), "t1")],
|
45
46
|
local_inputs={0: ("p1",)},
|
46
47
|
path=tmp_path,
|
47
48
|
store=store,
|
@@ -59,10 +60,10 @@ def test_wk_loop_data_idx_single_task_single_element_single_parameter_three_iter
|
|
59
60
|
|
60
61
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
61
62
|
def test_wk_loop_EARs_initialised_single_task_single_element_single_parameter_three_iters(
|
62
|
-
tmp_path, store
|
63
|
+
tmp_path: Path, store: str
|
63
64
|
):
|
64
65
|
wk = make_workflow(
|
65
|
-
schemas_spec=[
|
66
|
+
schemas_spec=[({"p1": None}, ("p1",), "t1")],
|
66
67
|
local_inputs={0: ("p1",)},
|
67
68
|
path=tmp_path,
|
68
69
|
store=store,
|
@@ -74,10 +75,10 @@ def test_wk_loop_EARs_initialised_single_task_single_element_single_parameter_th
|
|
74
75
|
|
75
76
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
76
77
|
def test_wk_loop_data_idx_single_task_multi_element_single_parameter_three_iters(
|
77
|
-
tmp_path, store
|
78
|
+
tmp_path: Path, store: str
|
78
79
|
):
|
79
80
|
wk = make_workflow(
|
80
|
-
schemas_spec=[
|
81
|
+
schemas_spec=[({"p1": None}, ("p1",), "t1")],
|
81
82
|
local_sequences={0: [("inputs.p1", 2, 0)]},
|
82
83
|
path=tmp_path,
|
83
84
|
store=store,
|
@@ -106,13 +107,13 @@ def test_wk_loop_data_idx_single_task_multi_element_single_parameter_three_iters
|
|
106
107
|
|
107
108
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
108
109
|
def test_wk_loop_data_idx_multi_task_single_element_single_parameter_two_iters(
|
109
|
-
tmp_path, store
|
110
|
+
tmp_path: Path, store: str
|
110
111
|
):
|
111
112
|
wk = make_workflow(
|
112
113
|
schemas_spec=[
|
113
|
-
|
114
|
-
|
115
|
-
|
114
|
+
({"p1": None}, ("p1",), "t1"),
|
115
|
+
({"p1": None}, ("p1",), "t2"),
|
116
|
+
({"p1": None}, ("p1",), "t3"),
|
116
117
|
],
|
117
118
|
local_inputs={0: ("p1",)},
|
118
119
|
path=tmp_path,
|
@@ -149,10 +150,10 @@ def test_wk_loop_data_idx_multi_task_single_element_single_parameter_two_iters(
|
|
149
150
|
|
150
151
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
151
152
|
def test_wk_loop_data_idx_single_task_single_element_single_parameter_three_iters_non_iterable_param(
|
152
|
-
tmp_path, store
|
153
|
+
tmp_path: Path, store: str
|
153
154
|
):
|
154
155
|
wk = make_workflow(
|
155
|
-
schemas_spec=[
|
156
|
+
schemas_spec=[({"p1": None}, ("p1",), "t1")],
|
156
157
|
local_inputs={0: ("p1",)},
|
157
158
|
path=tmp_path,
|
158
159
|
store=store,
|
@@ -171,12 +172,12 @@ def test_wk_loop_data_idx_single_task_single_element_single_parameter_three_iter
|
|
171
172
|
|
172
173
|
|
173
174
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
174
|
-
def test_wk_loop_iterable_parameters(tmp_path, store):
|
175
|
+
def test_wk_loop_iterable_parameters(tmp_path: Path, store: str):
|
175
176
|
wk = make_workflow(
|
176
177
|
schemas_spec=[
|
177
|
-
|
178
|
-
|
179
|
-
|
178
|
+
({"p1": None, "p2": None}, ("p1", "p2"), "t1"),
|
179
|
+
({"p1": None}, ("p1",), "t2"),
|
180
|
+
({"p1": None, "p2": None}, ("p1", "p2"), "t3"),
|
180
181
|
],
|
181
182
|
local_inputs={0: ("p1", "p2"), 1: ("p1",)},
|
182
183
|
path=tmp_path,
|
@@ -190,12 +191,14 @@ def test_wk_loop_iterable_parameters(tmp_path, store):
|
|
190
191
|
|
191
192
|
|
192
193
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
193
|
-
def test_wk_loop_input_sources_including_local_single_element_two_iters(
|
194
|
+
def test_wk_loop_input_sources_including_local_single_element_two_iters(
|
195
|
+
tmp_path: Path, store: str
|
196
|
+
):
|
194
197
|
wk = make_workflow(
|
195
198
|
schemas_spec=[
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
+
({"p1": None, "p2": None}, ("p1", "p2"), "t1"),
|
200
|
+
({"p1": None}, ("p1",), "t2"),
|
201
|
+
({"p1": None, "p2": None}, ("p1", "p2"), "t3"),
|
199
202
|
],
|
200
203
|
local_inputs={0: ("p1", "p2"), 1: ("p1",)},
|
201
204
|
path=tmp_path,
|
@@ -228,10 +231,10 @@ def test_wk_loop_input_sources_including_local_single_element_two_iters(tmp_path
|
|
228
231
|
|
229
232
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
230
233
|
def test_get_iteration_task_pathway_single_task_single_element_three_iters(
|
231
|
-
tmp_path, store
|
234
|
+
tmp_path: Path, store: str
|
232
235
|
):
|
233
236
|
wk = make_workflow(
|
234
|
-
schemas_spec=[
|
237
|
+
schemas_spec=[({"p1": None}, ("p1",), "t1")],
|
235
238
|
local_inputs={0: ("p1",)},
|
236
239
|
path=tmp_path,
|
237
240
|
store=store,
|
@@ -245,7 +248,7 @@ def test_get_iteration_task_pathway_single_task_single_element_three_iters(
|
|
245
248
|
]
|
246
249
|
|
247
250
|
|
248
|
-
def test_get_iteration_task_pathway_nested_loops_multi_iter(null_config, tmp_path):
|
251
|
+
def test_get_iteration_task_pathway_nested_loops_multi_iter(null_config, tmp_path: Path):
|
249
252
|
ts1 = hf.TaskSchema(
|
250
253
|
objective="t1",
|
251
254
|
inputs=[hf.SchemaInput("p1")],
|
@@ -288,7 +291,9 @@ def test_get_iteration_task_pathway_nested_loops_multi_iter(null_config, tmp_pat
|
|
288
291
|
@pytest.mark.skip(
|
289
292
|
reason="second set of asserts fail; need to re-source inputs on adding iterations."
|
290
293
|
)
|
291
|
-
def test_get_iteration_task_pathway_nested_loops_multi_iter_jagged(
|
294
|
+
def test_get_iteration_task_pathway_nested_loops_multi_iter_jagged(
|
295
|
+
null_config, tmp_path: Path
|
296
|
+
):
|
292
297
|
ts1 = hf.TaskSchema(
|
293
298
|
objective="t1",
|
294
299
|
inputs=[hf.SchemaInput("p1")],
|
@@ -347,7 +352,7 @@ def test_get_iteration_task_pathway_nested_loops_multi_iter_jagged(null_config,
|
|
347
352
|
|
348
353
|
|
349
354
|
def test_get_iteration_task_pathway_nested_loops_multi_iter_add_outer_iter(
|
350
|
-
null_config, tmp_path
|
355
|
+
null_config, tmp_path: Path
|
351
356
|
):
|
352
357
|
ts1 = hf.TaskSchema(
|
353
358
|
objective="t1",
|
@@ -395,7 +400,7 @@ def test_get_iteration_task_pathway_nested_loops_multi_iter_add_outer_iter(
|
|
395
400
|
@pytest.mark.skip(
|
396
401
|
reason="second set of asserts fail; need to re-source inputs on adding iterations."
|
397
402
|
)
|
398
|
-
def test_get_iteration_task_pathway_unconnected_loops(null_config, tmp_path):
|
403
|
+
def test_get_iteration_task_pathway_unconnected_loops(null_config, tmp_path: Path):
|
399
404
|
ts1 = hf.TaskSchema(
|
400
405
|
objective="t1",
|
401
406
|
inputs=[hf.SchemaInput("p1")],
|
@@ -448,7 +453,9 @@ def test_get_iteration_task_pathway_unconnected_loops(null_config, tmp_path):
|
|
448
453
|
assert pathway[4][2][0]["inputs.p1"] == pathway[3][2][0]["outputs.p1"]
|
449
454
|
|
450
455
|
|
451
|
-
def test_wk_loop_input_sources_including_non_iteration_task_source(
|
456
|
+
def test_wk_loop_input_sources_including_non_iteration_task_source(
|
457
|
+
null_config, tmp_path: Path
|
458
|
+
):
|
452
459
|
act_env = hf.ActionEnvironment("null_env")
|
453
460
|
ts1 = hf.TaskSchema(
|
454
461
|
objective="t1",
|
@@ -522,7 +529,7 @@ def test_wk_loop_input_sources_including_non_iteration_task_source(null_config,
|
|
522
529
|
assert t2_iter_1["inputs.p3"] == t3_iter_0["outputs.p3"]
|
523
530
|
|
524
531
|
|
525
|
-
def test_wk_loop_input_sources_default(null_config, tmp_path):
|
532
|
+
def test_wk_loop_input_sources_default(null_config, tmp_path: Path):
|
526
533
|
act_env = hf.ActionEnvironment("null_env")
|
527
534
|
ts1 = hf.TaskSchema(
|
528
535
|
objective="t1",
|
@@ -552,7 +559,7 @@ def test_wk_loop_input_sources_default(null_config, tmp_path):
|
|
552
559
|
assert t1_iter_0["inputs.p2"] == t1_iter_1["inputs.p2"]
|
553
560
|
|
554
561
|
|
555
|
-
def test_wk_loop_input_sources_iterable_param_default(null_config, tmp_path):
|
562
|
+
def test_wk_loop_input_sources_iterable_param_default(null_config, tmp_path: Path):
|
556
563
|
act_env = hf.ActionEnvironment("null_env")
|
557
564
|
ts1 = hf.TaskSchema(
|
558
565
|
objective="t1",
|
@@ -589,7 +596,7 @@ def test_wk_loop_input_sources_iterable_param_default(null_config, tmp_path):
|
|
589
596
|
|
590
597
|
|
591
598
|
def test_wk_loop_input_sources_iterable_param_default_conditional_action(
|
592
|
-
null_config, tmp_path
|
599
|
+
null_config, tmp_path: Path
|
593
600
|
):
|
594
601
|
act_env = hf.ActionEnvironment("null_env")
|
595
602
|
ts1 = hf.TaskSchema(
|
@@ -637,7 +644,7 @@ def test_wk_loop_input_sources_iterable_param_default_conditional_action(
|
|
637
644
|
|
638
645
|
|
639
646
|
def test_wk_loop_input_sources_including_non_iteration_task_source_with_groups(
|
640
|
-
null_config, tmp_path
|
647
|
+
null_config, tmp_path: Path
|
641
648
|
):
|
642
649
|
act_env = hf.ActionEnvironment("null_env")
|
643
650
|
ts1 = hf.TaskSchema(
|
@@ -733,7 +740,7 @@ def test_wk_loop_input_sources_including_non_iteration_task_source_with_groups(
|
|
733
740
|
]
|
734
741
|
|
735
742
|
|
736
|
-
def test_loop_local_sub_parameters(null_config, tmp_path):
|
743
|
+
def test_loop_local_sub_parameters(null_config, tmp_path: Path):
|
737
744
|
act_env = hf.ActionEnvironment("null_env")
|
738
745
|
ts1 = hf.TaskSchema(
|
739
746
|
objective="t1",
|
@@ -796,7 +803,7 @@ def test_loop_local_sub_parameters(null_config, tmp_path):
|
|
796
803
|
assert t1_iter_0["inputs.p1c.d"] == t1_iter_1["inputs.p1c.d"]
|
797
804
|
|
798
805
|
|
799
|
-
def test_nested_loop_iter_loop_idx(null_config, tmp_path):
|
806
|
+
def test_nested_loop_iter_loop_idx(null_config, tmp_path: Path):
|
800
807
|
ts1 = hf.TaskSchema(
|
801
808
|
objective="t1",
|
802
809
|
inputs=[hf.SchemaInput("p1")],
|
@@ -828,7 +835,7 @@ def test_nested_loop_iter_loop_idx(null_config, tmp_path):
|
|
828
835
|
}
|
829
836
|
|
830
837
|
|
831
|
-
def test_schema_input_with_group_sourced_from_prev_iteration(null_config, tmp_path):
|
838
|
+
def test_schema_input_with_group_sourced_from_prev_iteration(null_config, tmp_path: Path):
|
832
839
|
s1 = hf.TaskSchema(
|
833
840
|
objective="t1",
|
834
841
|
inputs=[hf.SchemaInput("p1")],
|
@@ -907,7 +914,7 @@ def test_schema_input_with_group_sourced_from_prev_iteration(null_config, tmp_pa
|
|
907
914
|
] == [wk.tasks.t2.elements[0].iterations[1].get_data_idx()["outputs.p3"]] * 3
|
908
915
|
|
909
916
|
|
910
|
-
def test_loop_downstream_tasks(null_config, tmp_path):
|
917
|
+
def test_loop_downstream_tasks(null_config, tmp_path: Path):
|
911
918
|
ts1 = hf.TaskSchema(
|
912
919
|
objective="t1",
|
913
920
|
inputs=[hf.SchemaInput("p1")],
|
@@ -951,11 +958,11 @@ def test_loop_downstream_tasks(null_config, tmp_path):
|
|
951
958
|
hf.Loop(name="my_loop", tasks=[1, 2], num_iterations=2),
|
952
959
|
],
|
953
960
|
)
|
954
|
-
assert wk.loops.my_loop.downstream_tasks == [wk.tasks[3]]
|
955
|
-
assert wk.loops.my_loop.upstream_tasks == [wk.tasks[0]]
|
961
|
+
assert list(wk.loops.my_loop.downstream_tasks) == [wk.tasks[3]]
|
962
|
+
assert list(wk.loops.my_loop.upstream_tasks) == [wk.tasks[0]]
|
956
963
|
|
957
964
|
|
958
|
-
def test_raise_loop_task_subset_error(null_config, tmp_path):
|
965
|
+
def test_raise_loop_task_subset_error(null_config, tmp_path: Path):
|
959
966
|
ts1 = hf.TaskSchema(
|
960
967
|
objective="t1",
|
961
968
|
inputs=[hf.SchemaInput("p1")],
|
@@ -986,7 +993,7 @@ def test_raise_loop_task_subset_error(null_config, tmp_path):
|
|
986
993
|
)
|
987
994
|
|
988
995
|
|
989
|
-
def test_raise_downstream_task_with_iterable_parameter(null_config, tmp_path):
|
996
|
+
def test_raise_downstream_task_with_iterable_parameter(null_config, tmp_path: Path):
|
990
997
|
ts1 = hf.TaskSchema(
|
991
998
|
objective="t1",
|
992
999
|
inputs=[hf.SchemaInput("p1")],
|
@@ -1017,7 +1024,7 @@ def test_raise_downstream_task_with_iterable_parameter(null_config, tmp_path):
|
|
1017
1024
|
)
|
1018
1025
|
|
1019
1026
|
|
1020
|
-
def test_adjacent_loops_iteration_pathway(null_config, tmp_path):
|
1027
|
+
def test_adjacent_loops_iteration_pathway(null_config, tmp_path: Path):
|
1021
1028
|
ts1 = hf.TaskSchema(
|
1022
1029
|
objective="t1",
|
1023
1030
|
inputs=[hf.SchemaInput("p1")],
|
@@ -1071,7 +1078,7 @@ def test_adjacent_loops_iteration_pathway(null_config, tmp_path):
|
|
1071
1078
|
]
|
1072
1079
|
|
1073
1080
|
|
1074
|
-
def test_get_child_loops_ordered_by_depth(null_config, tmp_path):
|
1081
|
+
def test_get_child_loops_ordered_by_depth(null_config, tmp_path: Path):
|
1075
1082
|
ts1 = hf.TaskSchema(
|
1076
1083
|
objective="t1",
|
1077
1084
|
inputs=[hf.SchemaInput("p1")],
|
@@ -1104,7 +1111,7 @@ def test_get_child_loops_ordered_by_depth(null_config, tmp_path):
|
|
1104
1111
|
assert wk.loops.outer.get_child_loops() == [wk.loops.middle, wk.loops.inner]
|
1105
1112
|
|
1106
1113
|
|
1107
|
-
def test_multi_nested_loops(null_config, tmp_path):
|
1114
|
+
def test_multi_nested_loops(null_config, tmp_path: Path):
|
1108
1115
|
ts1 = hf.TaskSchema(
|
1109
1116
|
objective="t1",
|
1110
1117
|
inputs=[hf.SchemaInput("p1")],
|
@@ -1161,14 +1168,14 @@ def test_multi_nested_loops(null_config, tmp_path):
|
|
1161
1168
|
]
|
1162
1169
|
|
1163
1170
|
|
1164
|
-
def test_nested_loop_input_from_parent_loop_task(null_config, tmp_path):
|
1171
|
+
def test_nested_loop_input_from_parent_loop_task(null_config, tmp_path: Path):
|
1165
1172
|
"""Test that an input in a nested-loop task is correctly sourced from latest
|
1166
1173
|
iteration of the parent loop."""
|
1167
1174
|
wk = make_workflow(
|
1168
1175
|
schemas_spec=[
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1176
|
+
({"p1": None}, ("p2", "p3")),
|
1177
|
+
({"p2": None}, ("p4",)),
|
1178
|
+
({"p4": None, "p3": None}, ("p2", "p1")), # testing p3 source
|
1172
1179
|
],
|
1173
1180
|
path=tmp_path,
|
1174
1181
|
local_inputs={0: {"p1": 101}},
|
@@ -1186,16 +1193,16 @@ def test_nested_loop_input_from_parent_loop_task(null_config, tmp_path):
|
|
1186
1193
|
assert p3_inp_idx == [p3_out_idx[0]] * 3 + [p3_out_idx[1]] * 3
|
1187
1194
|
|
1188
1195
|
|
1189
|
-
def test_doubly_nested_loop_input_from_parent_loop_task(null_config, tmp_path):
|
1196
|
+
def test_doubly_nested_loop_input_from_parent_loop_task(null_config, tmp_path: Path):
|
1190
1197
|
"""Test that an input in a doubly-nested-loop task is correctly sourced from latest
|
1191
1198
|
iteration of the parent loop."""
|
1192
1199
|
# test source of p6 in final task:
|
1193
1200
|
wk = make_workflow(
|
1194
1201
|
schemas_spec=[
|
1195
|
-
|
1196
|
-
|
1197
|
-
|
1198
|
-
|
1202
|
+
({"p5": None}, ("p6", "p1")),
|
1203
|
+
({"p1": None}, ("p2", "p3")),
|
1204
|
+
({"p2": None}, ("p4",)),
|
1205
|
+
({"p4": None, "p3": None, "p6": None}, ("p2", "p1", "p5")),
|
1199
1206
|
],
|
1200
1207
|
path=tmp_path,
|
1201
1208
|
local_inputs={0: {"p5": 101}},
|
@@ -1216,7 +1223,7 @@ def test_doubly_nested_loop_input_from_parent_loop_task(null_config, tmp_path):
|
|
1216
1223
|
assert p6_inp_idx == [p6_out_idx[0]] * 9 + [p6_out_idx[1]] * 9 + [p6_out_idx[2]] * 9
|
1217
1224
|
|
1218
1225
|
|
1219
|
-
def test_loop_non_input_task_input_from_element_group(null_config, tmp_path):
|
1226
|
+
def test_loop_non_input_task_input_from_element_group(null_config, tmp_path: Path):
|
1220
1227
|
"""Test correct sourcing of an element group input within a loop, for a task that is
|
1221
1228
|
not that loop's "input task" with respect to that parameter."""
|
1222
1229
|
s1 = hf.TaskSchema(
|
@@ -1,5 +1,6 @@
|
|
1
|
+
from __future__ import annotations
|
1
2
|
from dataclasses import dataclass
|
2
|
-
|
3
|
+
from typing_extensions import TypedDict
|
3
4
|
import pytest
|
4
5
|
|
5
6
|
from hpcflow.app import app as hf
|
@@ -18,43 +19,47 @@ class MyObj:
|
|
18
19
|
data: int
|
19
20
|
|
20
21
|
|
22
|
+
class SimpleObjectList(TypedDict):
|
23
|
+
objects: list[MyObj]
|
24
|
+
object_list: DotAccessObjectList
|
25
|
+
|
26
|
+
|
21
27
|
@pytest.fixture
|
22
|
-
def simple_object_list(null_config):
|
28
|
+
def simple_object_list(null_config) -> SimpleObjectList:
|
23
29
|
my_objs = [MyObj(name="A", data=1), MyObj(name="B", data=2)]
|
24
30
|
obj_list = DotAccessObjectList(my_objs, access_attribute="name")
|
25
|
-
|
26
|
-
return out
|
31
|
+
return {"objects": my_objs, "object_list": obj_list}
|
27
32
|
|
28
33
|
|
29
|
-
def test_get_item(simple_object_list):
|
34
|
+
def test_get_item(simple_object_list: SimpleObjectList):
|
30
35
|
objects = simple_object_list["objects"]
|
31
36
|
obj_list = simple_object_list["object_list"]
|
32
37
|
|
33
38
|
assert obj_list[0] == objects[0] and obj_list[1] == objects[1]
|
34
39
|
|
35
40
|
|
36
|
-
def test_get_dot_notation(simple_object_list):
|
41
|
+
def test_get_dot_notation(simple_object_list: SimpleObjectList):
|
37
42
|
objects = simple_object_list["objects"]
|
38
43
|
obj_list = simple_object_list["object_list"]
|
39
44
|
|
40
45
|
assert obj_list.A == objects[0] and obj_list.B == objects[1]
|
41
46
|
|
42
47
|
|
43
|
-
def test_add_obj_to_end(simple_object_list):
|
48
|
+
def test_add_obj_to_end(simple_object_list: SimpleObjectList):
|
44
49
|
obj_list = simple_object_list["object_list"]
|
45
50
|
new_obj = MyObj("C", 3)
|
46
51
|
obj_list.add_object(new_obj)
|
47
52
|
assert obj_list[-1] == new_obj
|
48
53
|
|
49
54
|
|
50
|
-
def test_add_obj_to_start(simple_object_list):
|
55
|
+
def test_add_obj_to_start(simple_object_list: SimpleObjectList):
|
51
56
|
obj_list = simple_object_list["object_list"]
|
52
57
|
new_obj = MyObj("C", 3)
|
53
58
|
obj_list.add_object(new_obj, 0)
|
54
59
|
assert obj_list[0] == new_obj
|
55
60
|
|
56
61
|
|
57
|
-
def test_add_obj_to_middle(simple_object_list):
|
62
|
+
def test_add_obj_to_middle(simple_object_list: SimpleObjectList):
|
58
63
|
obj_list = simple_object_list["object_list"]
|
59
64
|
new_obj = MyObj("C", 3)
|
60
65
|
obj_list.add_object(new_obj, 1)
|
@@ -80,12 +85,12 @@ def test_get_obj_attr_custom_callable(null_config):
|
|
80
85
|
assert o1.get(c1=2) == o1[0]
|
81
86
|
|
82
87
|
|
83
|
-
def test_get_with_missing_key(null_config):
|
88
|
+
def test_get_with_missing_key(null_config) -> None:
|
84
89
|
o1 = ObjectList([{"a": 1}, {"b": 2}])
|
85
90
|
assert o1.get(a=1) == {"a": 1}
|
86
91
|
|
87
92
|
|
88
|
-
def test_parameters_list_get_equivalence(null_config):
|
93
|
+
def test_parameters_list_get_equivalence(null_config) -> None:
|
89
94
|
p_name = "p12334567898765432101"
|
90
95
|
hf.parameters.add_object(hf.Parameter(p_name))
|
91
96
|
assert p_name in hf.parameters.list_attrs()
|
@@ -97,7 +102,7 @@ def test_parameters_list_get_equivalence(null_config):
|
|
97
102
|
)
|
98
103
|
|
99
104
|
|
100
|
-
def test_parameters_list_get_equivalence_non_existent(new_null_config):
|
105
|
+
def test_parameters_list_get_equivalence_non_existent(new_null_config) -> None:
|
101
106
|
# non-existent parameters should be created, unlike other ObjectList sub-classes,
|
102
107
|
# which raise
|
103
108
|
hf.reload_template_components()
|
@@ -1,19 +1,24 @@
|
|
1
|
+
from __future__ import annotations
|
1
2
|
from dataclasses import dataclass
|
3
|
+
from pathlib import Path
|
2
4
|
import random
|
3
5
|
import string
|
4
6
|
import sys
|
5
7
|
from textwrap import dedent
|
8
|
+
from typing import ClassVar
|
6
9
|
|
7
10
|
import pytest
|
8
11
|
import requests
|
9
12
|
|
10
13
|
from hpcflow.app import app as hf
|
14
|
+
from hpcflow.sdk.typing import hydrate
|
11
15
|
from hpcflow.sdk.core.parameters import ParameterValue
|
12
16
|
|
13
17
|
|
14
18
|
@dataclass
|
19
|
+
@hydrate
|
15
20
|
class MyParameterP1(ParameterValue):
|
16
|
-
_typ = "p1_test"
|
21
|
+
_typ: ClassVar[str] = "p1_test"
|
17
22
|
a: int
|
18
23
|
|
19
24
|
def CLI_format(self):
|
@@ -22,7 +27,9 @@ class MyParameterP1(ParameterValue):
|
|
22
27
|
|
23
28
|
@pytest.mark.integration
|
24
29
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
25
|
-
def test_submission_with_specified_parameter_class_module(
|
30
|
+
def test_submission_with_specified_parameter_class_module(
|
31
|
+
null_config, tmp_path: Path, store: str
|
32
|
+
):
|
26
33
|
"""Test we can use a ParameterValue subclass that is defined separately from the main
|
27
34
|
code (i.e. not automatically imported on app init)."""
|
28
35
|
|
@@ -84,7 +91,7 @@ def test_submission_with_specified_parameter_class_module(null_config, tmp_path,
|
|
84
91
|
|
85
92
|
|
86
93
|
@pytest.mark.parametrize("store", ["json", "zarr"])
|
87
|
-
def test_unseen_parameter(null_config, tmp_path, store):
|
94
|
+
def test_unseen_parameter(null_config, tmp_path: Path, store: str):
|
88
95
|
"""Test we can generate a workflow that uses an unseen parameter type."""
|
89
96
|
|
90
97
|
random_str = "".join(random.choice(string.ascii_letters) for _ in range(10))
|
@@ -116,7 +123,7 @@ def test_unseen_parameter(null_config, tmp_path, store):
|
|
116
123
|
assert wk.tasks[0].elements[0].get(f"inputs.{p_type}") == 5
|
117
124
|
|
118
125
|
|
119
|
-
def test_iter(new_null_config, tmp_path):
|
126
|
+
def test_iter(new_null_config, tmp_path: Path):
|
120
127
|
values = [1, 2, 3]
|
121
128
|
wkt = hf.WorkflowTemplate(
|
122
129
|
name="test",
|
@@ -132,7 +139,7 @@ def test_iter(new_null_config, tmp_path):
|
|
132
139
|
assert param_p1_i.value == values[idx]
|
133
140
|
|
134
141
|
|
135
|
-
def test_slice(new_null_config, tmp_path):
|
142
|
+
def test_slice(new_null_config, tmp_path: Path):
|
136
143
|
values = [1, 2, 3]
|
137
144
|
wkt = hf.WorkflowTemplate(
|
138
145
|
name="test",
|
@@ -158,7 +165,7 @@ def test_slice(new_null_config, tmp_path):
|
|
158
165
|
"retrieving demo data from GitHub."
|
159
166
|
),
|
160
167
|
)
|
161
|
-
def test_demo_data_substitution_param_value_class_method(new_null_config, tmp_path):
|
168
|
+
def test_demo_data_substitution_param_value_class_method(new_null_config, tmp_path: Path):
|
162
169
|
yaml_str = dedent(
|
163
170
|
"""\
|
164
171
|
name: temp
|
@@ -189,7 +196,9 @@ def test_demo_data_substitution_param_value_class_method(new_null_config, tmp_pa
|
|
189
196
|
"retrieving demo data from GitHub."
|
190
197
|
),
|
191
198
|
)
|
192
|
-
def test_demo_data_substitution_value_sequence_class_method(
|
199
|
+
def test_demo_data_substitution_value_sequence_class_method(
|
200
|
+
new_null_config, tmp_path: Path
|
201
|
+
):
|
193
202
|
yaml_str = dedent(
|
194
203
|
"""\
|
195
204
|
name: temp
|