hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/bad_script.py +2 -0
- hpcflow/data/scripts/do_nothing.py +2 -0
- hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
- hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/input_file_generator_basic.py +3 -0
- hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
- hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
- hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
- hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
- hpcflow/data/scripts/output_file_parser_basic.py +3 -0
- hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
- hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/script_exit_test.py +5 -0
- hpcflow/data/template_components/environments.yaml +1 -1
- hpcflow/sdk/__init__.py +26 -15
- hpcflow/sdk/app.py +2192 -768
- hpcflow/sdk/cli.py +506 -296
- hpcflow/sdk/cli_common.py +105 -7
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +115 -43
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +674 -318
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +125 -84
- hpcflow/sdk/config/types.py +148 -0
- hpcflow/sdk/core/__init__.py +25 -1
- hpcflow/sdk/core/actions.py +1771 -1059
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +139 -79
- hpcflow/sdk/core/command_files.py +263 -287
- hpcflow/sdk/core/commands.py +145 -112
- hpcflow/sdk/core/element.py +828 -535
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +455 -52
- hpcflow/sdk/core/execute.py +207 -0
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +751 -347
- hpcflow/sdk/core/loop_cache.py +164 -47
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +1100 -627
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +21 -37
- hpcflow/sdk/core/skip_reason.py +7 -0
- hpcflow/sdk/core/task.py +1649 -1339
- hpcflow/sdk/core/task_schema.py +308 -196
- hpcflow/sdk/core/test_utils.py +191 -114
- hpcflow/sdk/core/types.py +440 -0
- hpcflow/sdk/core/utils.py +485 -309
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +2544 -1178
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
- hpcflow/sdk/demo/cli.py +53 -33
- hpcflow/sdk/helper/cli.py +18 -15
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +122 -71
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +1360 -606
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +568 -188
- hpcflow/sdk/persistence/pending.py +382 -179
- hpcflow/sdk/persistence/store_resource.py +39 -23
- hpcflow/sdk/persistence/types.py +318 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +1337 -433
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +1651 -692
- hpcflow/sdk/submission/schedulers/__init__.py +167 -39
- hpcflow/sdk/submission/schedulers/direct.py +121 -81
- hpcflow/sdk/submission/schedulers/sge.py +170 -129
- hpcflow/sdk/submission/schedulers/slurm.py +291 -268
- hpcflow/sdk/submission/schedulers/utils.py +12 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +150 -29
- hpcflow/sdk/submission/shells/bash.py +283 -173
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +228 -170
- hpcflow/sdk/submission/submission.py +1014 -335
- hpcflow/sdk/submission/types.py +140 -0
- hpcflow/sdk/typing.py +182 -12
- hpcflow/sdk/utils/arrays.py +71 -0
- hpcflow/sdk/utils/deferred_file.py +55 -0
- hpcflow/sdk/utils/hashing.py +16 -0
- hpcflow/sdk/utils/patches.py +12 -0
- hpcflow/sdk/utils/strings.py +33 -0
- hpcflow/tests/api/test_api.py +32 -0
- hpcflow/tests/conftest.py +27 -6
- hpcflow/tests/data/multi_path_sequences.yaml +29 -0
- hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
- hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_input_file_generators.py +282 -0
- hpcflow/tests/scripts/test_main_scripts.py +866 -85
- hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
- hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
- hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
- hpcflow/tests/unit/test_action.py +262 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +33 -6
- hpcflow/tests/unit/test_cache.py +46 -0
- hpcflow/tests/unit/test_cli.py +134 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +142 -16
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +50 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_jobscript_unit.py +757 -0
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +1396 -84
- hpcflow/tests/unit/test_meta_task.py +325 -0
- hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +29 -7
- hpcflow/tests/unit/test_persistence.py +237 -42
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +117 -6
- hpcflow/tests/unit/test_run_directories.py +29 -0
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +23 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +38 -89
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/unit/utils/test_arrays.py +40 -0
- hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
- hpcflow/tests/unit/utils/test_hashing.py +65 -0
- hpcflow/tests/unit/utils/test_patches.py +5 -0
- hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
- hpcflow/tests/workflows/__init__.py +0 -0
- hpcflow/tests/workflows/test_directory_structure.py +31 -0
- hpcflow/tests/workflows/test_jobscript.py +334 -1
- hpcflow/tests/workflows/test_run_status.py +198 -0
- hpcflow/tests/workflows/test_skip_downstream.py +696 -0
- hpcflow/tests/workflows/test_submission.py +140 -0
- hpcflow/tests/workflows/test_workflows.py +160 -15
- hpcflow/tests/workflows/test_zip.py +18 -0
- hpcflow/viz_demo.ipynb +6587 -3
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
- hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -1,81 +1,86 @@
|
|
1
|
+
from __future__ import annotations
|
1
2
|
import sys
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import TYPE_CHECKING
|
2
5
|
import pytest
|
3
6
|
import requests
|
4
7
|
|
5
8
|
from hpcflow.app import app as hf
|
6
|
-
from hpcflow.sdk.core.errors import InputValueDuplicateSequenceAddress
|
7
9
|
from hpcflow.sdk.core.test_utils import P1_parameter_cls as P1
|
8
10
|
|
11
|
+
if TYPE_CHECKING:
|
12
|
+
from hpcflow.sdk.core.parameters import Parameter
|
13
|
+
|
9
14
|
|
10
15
|
@pytest.fixture
|
11
|
-
def null_config(tmp_path):
|
16
|
+
def null_config(tmp_path: Path):
|
12
17
|
if not hf.is_config_loaded:
|
13
18
|
hf.load_config(config_dir=tmp_path)
|
14
19
|
|
15
20
|
|
16
21
|
@pytest.fixture
|
17
|
-
def param_p1(null_config):
|
22
|
+
def param_p1(null_config) -> Parameter:
|
18
23
|
return hf.Parameter("p1")
|
19
24
|
|
20
25
|
|
21
|
-
def test_fix_trailing_path_delimiter(param_p1):
|
26
|
+
def test_fix_trailing_path_delimiter(param_p1: Parameter):
|
22
27
|
iv1 = hf.InputValue(parameter=param_p1, value=101, path="a.")
|
23
28
|
iv2 = hf.InputValue(parameter=param_p1, value=101, path="a")
|
24
29
|
assert iv1.path == iv2.path
|
25
30
|
|
26
31
|
|
27
|
-
def test_fix_single_path_delimiter(param_p1):
|
32
|
+
def test_fix_single_path_delimiter(param_p1: Parameter):
|
28
33
|
iv1 = hf.InputValue(parameter=param_p1, value=101, path=".")
|
29
34
|
iv2 = hf.InputValue(parameter=param_p1, value=101)
|
30
35
|
assert iv1.path == iv2.path
|
31
36
|
|
32
37
|
|
33
|
-
def test_normalised_path_without_path(param_p1):
|
38
|
+
def test_normalised_path_without_path(param_p1: Parameter):
|
34
39
|
iv1 = hf.InputValue(parameter=param_p1, value=101)
|
35
40
|
assert iv1.normalised_path == "inputs.p1"
|
36
41
|
|
37
42
|
|
38
|
-
def test_normalised_path_with_single_element_path(param_p1):
|
43
|
+
def test_normalised_path_with_single_element_path(param_p1: Parameter):
|
39
44
|
iv1 = hf.InputValue(parameter=param_p1, value=101, path="a")
|
40
45
|
assert iv1.normalised_path == "inputs.p1.a"
|
41
46
|
|
42
47
|
|
43
|
-
def test_normalised_path_with_multi_element_path(param_p1):
|
48
|
+
def test_normalised_path_with_multi_element_path(param_p1: Parameter):
|
44
49
|
iv1 = hf.InputValue(parameter=param_p1, value=101, path="a.b")
|
45
50
|
assert iv1.normalised_path == "inputs.p1.a.b"
|
46
51
|
|
47
52
|
|
48
|
-
def test_normalised_path_with_empty_path(param_p1):
|
53
|
+
def test_normalised_path_with_empty_path(param_p1: Parameter):
|
49
54
|
iv1 = hf.InputValue(parameter=param_p1, value=101, path="")
|
50
55
|
assert iv1.normalised_path == "inputs.p1"
|
51
56
|
|
52
57
|
|
53
|
-
def test_resource_spec_get_param_path(null_config):
|
58
|
+
def test_resource_spec_get_param_path(null_config) -> None:
|
54
59
|
rs1 = hf.ResourceSpec()
|
55
60
|
assert rs1.normalised_path == "resources.any"
|
56
61
|
|
57
62
|
|
58
|
-
def test_resource_spec_get_param_path_scope_any_with_single_kwarg(null_config):
|
63
|
+
def test_resource_spec_get_param_path_scope_any_with_single_kwarg(null_config) -> None:
|
59
64
|
rs1 = hf.ResourceSpec(scratch="local")
|
60
65
|
assert rs1.normalised_path == "resources.any"
|
61
66
|
|
62
67
|
|
63
|
-
def test_resources_spec_get_param_path_scope_main(null_config):
|
68
|
+
def test_resources_spec_get_param_path_scope_main(null_config) -> None:
|
64
69
|
rs1 = hf.ResourceSpec(scope=hf.ActionScope.main())
|
65
70
|
assert rs1.normalised_path == "resources.main"
|
66
71
|
|
67
72
|
|
68
|
-
def test_resources_spec_get_param_path_scope_with_kwargs(null_config):
|
73
|
+
def test_resources_spec_get_param_path_scope_with_kwargs(null_config) -> None:
|
69
74
|
rs1 = hf.ResourceSpec(scope=hf.ActionScope.input_file_generator(file="file1"))
|
70
75
|
assert rs1.normalised_path == "resources.input_file_generator[file=file1]"
|
71
76
|
|
72
77
|
|
73
|
-
def test_resources_spec_get_param_path_scope_with_no_kwargs(null_config):
|
78
|
+
def test_resources_spec_get_param_path_scope_with_no_kwargs(null_config) -> None:
|
74
79
|
rs1 = hf.ResourceSpec(scope=hf.ActionScope.input_file_generator())
|
75
80
|
assert rs1.normalised_path == "resources.input_file_generator"
|
76
81
|
|
77
82
|
|
78
|
-
def test_input_value_from_json_like_class_method_attribute_is_set(null_config):
|
83
|
+
def test_input_value_from_json_like_class_method_attribute_is_set(null_config) -> None:
|
79
84
|
parameter_typ = "p1"
|
80
85
|
cls_method = "from_data"
|
81
86
|
json_like = {"parameter": f"{parameter_typ}::{cls_method}", "value": 101}
|
@@ -84,7 +89,7 @@ def test_input_value_from_json_like_class_method_attribute_is_set(null_config):
|
|
84
89
|
assert inp_val.value_class_method == cls_method
|
85
90
|
|
86
91
|
|
87
|
-
def test_value_sequence_from_json_like_class_method_attribute_is_set(null_config):
|
92
|
+
def test_value_sequence_from_json_like_class_method_attribute_is_set(null_config) -> None:
|
88
93
|
parameter_typ = "p1"
|
89
94
|
cls_method = "from_data"
|
90
95
|
json_like = {
|
@@ -99,28 +104,28 @@ def test_value_sequence_from_json_like_class_method_attribute_is_set(null_config
|
|
99
104
|
assert val_seq.value_class_method == cls_method
|
100
105
|
|
101
106
|
|
102
|
-
def test_path_attributes(null_config):
|
107
|
+
def test_path_attributes(null_config) -> None:
|
103
108
|
inp = hf.InputValue(parameter="p1", value=101, path="a.b")
|
104
109
|
assert inp.labelled_type == "p1"
|
105
110
|
assert inp.normalised_path == "inputs.p1.a.b"
|
106
111
|
assert inp.normalised_inputs_path == "p1.a.b"
|
107
112
|
|
108
113
|
|
109
|
-
def test_path_attributes_with_label_arg(null_config):
|
114
|
+
def test_path_attributes_with_label_arg(null_config) -> None:
|
110
115
|
inp = hf.InputValue(parameter="p1", value=101, path="a.b", label="1")
|
111
116
|
assert inp.labelled_type == "p1[1]"
|
112
117
|
assert inp.normalised_path == "inputs.p1[1].a.b"
|
113
118
|
assert inp.normalised_inputs_path == "p1[1].a.b"
|
114
119
|
|
115
120
|
|
116
|
-
def test_path_attributes_with_label_arg_cast(null_config):
|
121
|
+
def test_path_attributes_with_label_arg_cast(null_config) -> None:
|
117
122
|
inp = hf.InputValue(parameter="p1", value=101, path="a.b", label=1)
|
118
123
|
assert inp.labelled_type == "p1[1]"
|
119
124
|
assert inp.normalised_path == "inputs.p1[1].a.b"
|
120
125
|
assert inp.normalised_inputs_path == "p1[1].a.b"
|
121
126
|
|
122
127
|
|
123
|
-
def test_from_json_like(null_config):
|
128
|
+
def test_from_json_like(null_config) -> None:
|
124
129
|
inp = hf.InputValue.from_json_like(
|
125
130
|
json_like={"parameter": "p1", "value": 101},
|
126
131
|
shared_data=hf.template_components,
|
@@ -130,7 +135,7 @@ def test_from_json_like(null_config):
|
|
130
135
|
assert inp.label == ""
|
131
136
|
|
132
137
|
|
133
|
-
def test_from_json_like_with_label(null_config):
|
138
|
+
def test_from_json_like_with_label(null_config) -> None:
|
134
139
|
inp = hf.InputValue.from_json_like(
|
135
140
|
json_like={"parameter": "p1[1]", "value": 101},
|
136
141
|
shared_data=hf.template_components,
|
@@ -140,12 +145,12 @@ def test_from_json_like_with_label(null_config):
|
|
140
145
|
assert inp.label == "1"
|
141
146
|
|
142
147
|
|
143
|
-
def test_value_is_dict_check_success(null_config):
|
148
|
+
def test_value_is_dict_check_success(null_config) -> None:
|
144
149
|
# Parameter("p1c") has an associated `ParameterValue` class, so data should be a dict:
|
145
150
|
hf.InputValue("p1c", {"a": 101})
|
146
151
|
|
147
152
|
|
148
|
-
def test_value_is_dict_check_raise(null_config):
|
153
|
+
def test_value_is_dict_check_raise(null_config) -> None:
|
149
154
|
# Parameter("p1c") has an associated `ParameterValue` class so data should be a dict:
|
150
155
|
with pytest.raises(ValueError):
|
151
156
|
hf.InputValue("p1c", 101)
|
@@ -165,7 +170,7 @@ def test_value_is_dict_check_no_raise_if_sub_parameter(null_config):
|
|
165
170
|
"retrieving demo data from GitHub."
|
166
171
|
),
|
167
172
|
)
|
168
|
-
def test_demo_data_value(null_config):
|
173
|
+
def test_demo_data_value(null_config) -> None:
|
169
174
|
name = "text_file.txt"
|
170
175
|
assert hf.InputValue("p1", value=f"<<demo_data_file:{name}>>").value == str(
|
171
176
|
hf.demo_data_cache_dir.joinpath(name)
|