hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/bad_script.py +2 -0
  5. hpcflow/data/scripts/do_nothing.py +2 -0
  6. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  7. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  8. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  9. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  10. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  11. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  12. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  13. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  14. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  15. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  16. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  17. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  18. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  19. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  20. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  21. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  22. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  23. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  24. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  25. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  26. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  27. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  28. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  29. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  30. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  31. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  32. hpcflow/data/scripts/script_exit_test.py +5 -0
  33. hpcflow/data/template_components/environments.yaml +1 -1
  34. hpcflow/sdk/__init__.py +26 -15
  35. hpcflow/sdk/app.py +2192 -768
  36. hpcflow/sdk/cli.py +506 -296
  37. hpcflow/sdk/cli_common.py +105 -7
  38. hpcflow/sdk/config/__init__.py +1 -1
  39. hpcflow/sdk/config/callbacks.py +115 -43
  40. hpcflow/sdk/config/cli.py +126 -103
  41. hpcflow/sdk/config/config.py +674 -318
  42. hpcflow/sdk/config/config_file.py +131 -95
  43. hpcflow/sdk/config/errors.py +125 -84
  44. hpcflow/sdk/config/types.py +148 -0
  45. hpcflow/sdk/core/__init__.py +25 -1
  46. hpcflow/sdk/core/actions.py +1771 -1059
  47. hpcflow/sdk/core/app_aware.py +24 -0
  48. hpcflow/sdk/core/cache.py +139 -79
  49. hpcflow/sdk/core/command_files.py +263 -287
  50. hpcflow/sdk/core/commands.py +145 -112
  51. hpcflow/sdk/core/element.py +828 -535
  52. hpcflow/sdk/core/enums.py +192 -0
  53. hpcflow/sdk/core/environment.py +74 -93
  54. hpcflow/sdk/core/errors.py +455 -52
  55. hpcflow/sdk/core/execute.py +207 -0
  56. hpcflow/sdk/core/json_like.py +540 -272
  57. hpcflow/sdk/core/loop.py +751 -347
  58. hpcflow/sdk/core/loop_cache.py +164 -47
  59. hpcflow/sdk/core/object_list.py +370 -207
  60. hpcflow/sdk/core/parameters.py +1100 -627
  61. hpcflow/sdk/core/rule.py +59 -41
  62. hpcflow/sdk/core/run_dir_files.py +21 -37
  63. hpcflow/sdk/core/skip_reason.py +7 -0
  64. hpcflow/sdk/core/task.py +1649 -1339
  65. hpcflow/sdk/core/task_schema.py +308 -196
  66. hpcflow/sdk/core/test_utils.py +191 -114
  67. hpcflow/sdk/core/types.py +440 -0
  68. hpcflow/sdk/core/utils.py +485 -309
  69. hpcflow/sdk/core/validation.py +82 -9
  70. hpcflow/sdk/core/workflow.py +2544 -1178
  71. hpcflow/sdk/core/zarr_io.py +98 -137
  72. hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
  73. hpcflow/sdk/demo/cli.py +53 -33
  74. hpcflow/sdk/helper/cli.py +18 -15
  75. hpcflow/sdk/helper/helper.py +75 -63
  76. hpcflow/sdk/helper/watcher.py +61 -28
  77. hpcflow/sdk/log.py +122 -71
  78. hpcflow/sdk/persistence/__init__.py +8 -31
  79. hpcflow/sdk/persistence/base.py +1360 -606
  80. hpcflow/sdk/persistence/defaults.py +6 -0
  81. hpcflow/sdk/persistence/discovery.py +38 -0
  82. hpcflow/sdk/persistence/json.py +568 -188
  83. hpcflow/sdk/persistence/pending.py +382 -179
  84. hpcflow/sdk/persistence/store_resource.py +39 -23
  85. hpcflow/sdk/persistence/types.py +318 -0
  86. hpcflow/sdk/persistence/utils.py +14 -11
  87. hpcflow/sdk/persistence/zarr.py +1337 -433
  88. hpcflow/sdk/runtime.py +44 -41
  89. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  90. hpcflow/sdk/submission/jobscript.py +1651 -692
  91. hpcflow/sdk/submission/schedulers/__init__.py +167 -39
  92. hpcflow/sdk/submission/schedulers/direct.py +121 -81
  93. hpcflow/sdk/submission/schedulers/sge.py +170 -129
  94. hpcflow/sdk/submission/schedulers/slurm.py +291 -268
  95. hpcflow/sdk/submission/schedulers/utils.py +12 -2
  96. hpcflow/sdk/submission/shells/__init__.py +14 -15
  97. hpcflow/sdk/submission/shells/base.py +150 -29
  98. hpcflow/sdk/submission/shells/bash.py +283 -173
  99. hpcflow/sdk/submission/shells/os_version.py +31 -30
  100. hpcflow/sdk/submission/shells/powershell.py +228 -170
  101. hpcflow/sdk/submission/submission.py +1014 -335
  102. hpcflow/sdk/submission/types.py +140 -0
  103. hpcflow/sdk/typing.py +182 -12
  104. hpcflow/sdk/utils/arrays.py +71 -0
  105. hpcflow/sdk/utils/deferred_file.py +55 -0
  106. hpcflow/sdk/utils/hashing.py +16 -0
  107. hpcflow/sdk/utils/patches.py +12 -0
  108. hpcflow/sdk/utils/strings.py +33 -0
  109. hpcflow/tests/api/test_api.py +32 -0
  110. hpcflow/tests/conftest.py +27 -6
  111. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  112. hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
  113. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  114. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  115. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  116. hpcflow/tests/scripts/test_main_scripts.py +866 -85
  117. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  118. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  119. hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
  120. hpcflow/tests/unit/test_action.py +262 -75
  121. hpcflow/tests/unit/test_action_rule.py +9 -4
  122. hpcflow/tests/unit/test_app.py +33 -6
  123. hpcflow/tests/unit/test_cache.py +46 -0
  124. hpcflow/tests/unit/test_cli.py +134 -1
  125. hpcflow/tests/unit/test_command.py +71 -54
  126. hpcflow/tests/unit/test_config.py +142 -16
  127. hpcflow/tests/unit/test_config_file.py +21 -18
  128. hpcflow/tests/unit/test_element.py +58 -62
  129. hpcflow/tests/unit/test_element_iteration.py +50 -1
  130. hpcflow/tests/unit/test_element_set.py +29 -19
  131. hpcflow/tests/unit/test_group.py +4 -2
  132. hpcflow/tests/unit/test_input_source.py +116 -93
  133. hpcflow/tests/unit/test_input_value.py +29 -24
  134. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  135. hpcflow/tests/unit/test_json_like.py +44 -35
  136. hpcflow/tests/unit/test_loop.py +1396 -84
  137. hpcflow/tests/unit/test_meta_task.py +325 -0
  138. hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
  139. hpcflow/tests/unit/test_object_list.py +17 -12
  140. hpcflow/tests/unit/test_parameter.py +29 -7
  141. hpcflow/tests/unit/test_persistence.py +237 -42
  142. hpcflow/tests/unit/test_resources.py +20 -18
  143. hpcflow/tests/unit/test_run.py +117 -6
  144. hpcflow/tests/unit/test_run_directories.py +29 -0
  145. hpcflow/tests/unit/test_runtime.py +2 -1
  146. hpcflow/tests/unit/test_schema_input.py +23 -15
  147. hpcflow/tests/unit/test_shell.py +23 -2
  148. hpcflow/tests/unit/test_slurm.py +8 -7
  149. hpcflow/tests/unit/test_submission.py +38 -89
  150. hpcflow/tests/unit/test_task.py +352 -247
  151. hpcflow/tests/unit/test_task_schema.py +33 -20
  152. hpcflow/tests/unit/test_utils.py +9 -11
  153. hpcflow/tests/unit/test_value_sequence.py +15 -12
  154. hpcflow/tests/unit/test_workflow.py +114 -83
  155. hpcflow/tests/unit/test_workflow_template.py +0 -1
  156. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  157. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  158. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  159. hpcflow/tests/unit/utils/test_patches.py +5 -0
  160. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  161. hpcflow/tests/workflows/__init__.py +0 -0
  162. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  163. hpcflow/tests/workflows/test_jobscript.py +334 -1
  164. hpcflow/tests/workflows/test_run_status.py +198 -0
  165. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  166. hpcflow/tests/workflows/test_submission.py +140 -0
  167. hpcflow/tests/workflows/test_workflows.py +160 -15
  168. hpcflow/tests/workflows/test_zip.py +18 -0
  169. hpcflow/viz_demo.ipynb +6587 -3
  170. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
  171. hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
  172. hpcflow/sdk/core/parallel.py +0 -21
  173. hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
  174. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
  175. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
  176. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -1,81 +1,86 @@
1
+ from __future__ import annotations
1
2
  import sys
3
+ from pathlib import Path
4
+ from typing import TYPE_CHECKING
2
5
  import pytest
3
6
  import requests
4
7
 
5
8
  from hpcflow.app import app as hf
6
- from hpcflow.sdk.core.errors import InputValueDuplicateSequenceAddress
7
9
  from hpcflow.sdk.core.test_utils import P1_parameter_cls as P1
8
10
 
11
+ if TYPE_CHECKING:
12
+ from hpcflow.sdk.core.parameters import Parameter
13
+
9
14
 
10
15
  @pytest.fixture
11
- def null_config(tmp_path):
16
+ def null_config(tmp_path: Path):
12
17
  if not hf.is_config_loaded:
13
18
  hf.load_config(config_dir=tmp_path)
14
19
 
15
20
 
16
21
  @pytest.fixture
17
- def param_p1(null_config):
22
+ def param_p1(null_config) -> Parameter:
18
23
  return hf.Parameter("p1")
19
24
 
20
25
 
21
- def test_fix_trailing_path_delimiter(param_p1):
26
+ def test_fix_trailing_path_delimiter(param_p1: Parameter):
22
27
  iv1 = hf.InputValue(parameter=param_p1, value=101, path="a.")
23
28
  iv2 = hf.InputValue(parameter=param_p1, value=101, path="a")
24
29
  assert iv1.path == iv2.path
25
30
 
26
31
 
27
- def test_fix_single_path_delimiter(param_p1):
32
+ def test_fix_single_path_delimiter(param_p1: Parameter):
28
33
  iv1 = hf.InputValue(parameter=param_p1, value=101, path=".")
29
34
  iv2 = hf.InputValue(parameter=param_p1, value=101)
30
35
  assert iv1.path == iv2.path
31
36
 
32
37
 
33
- def test_normalised_path_without_path(param_p1):
38
+ def test_normalised_path_without_path(param_p1: Parameter):
34
39
  iv1 = hf.InputValue(parameter=param_p1, value=101)
35
40
  assert iv1.normalised_path == "inputs.p1"
36
41
 
37
42
 
38
- def test_normalised_path_with_single_element_path(param_p1):
43
+ def test_normalised_path_with_single_element_path(param_p1: Parameter):
39
44
  iv1 = hf.InputValue(parameter=param_p1, value=101, path="a")
40
45
  assert iv1.normalised_path == "inputs.p1.a"
41
46
 
42
47
 
43
- def test_normalised_path_with_multi_element_path(param_p1):
48
+ def test_normalised_path_with_multi_element_path(param_p1: Parameter):
44
49
  iv1 = hf.InputValue(parameter=param_p1, value=101, path="a.b")
45
50
  assert iv1.normalised_path == "inputs.p1.a.b"
46
51
 
47
52
 
48
- def test_normalised_path_with_empty_path(param_p1):
53
+ def test_normalised_path_with_empty_path(param_p1: Parameter):
49
54
  iv1 = hf.InputValue(parameter=param_p1, value=101, path="")
50
55
  assert iv1.normalised_path == "inputs.p1"
51
56
 
52
57
 
53
- def test_resource_spec_get_param_path(null_config):
58
+ def test_resource_spec_get_param_path(null_config) -> None:
54
59
  rs1 = hf.ResourceSpec()
55
60
  assert rs1.normalised_path == "resources.any"
56
61
 
57
62
 
58
- def test_resource_spec_get_param_path_scope_any_with_single_kwarg(null_config):
63
+ def test_resource_spec_get_param_path_scope_any_with_single_kwarg(null_config) -> None:
59
64
  rs1 = hf.ResourceSpec(scratch="local")
60
65
  assert rs1.normalised_path == "resources.any"
61
66
 
62
67
 
63
- def test_resources_spec_get_param_path_scope_main(null_config):
68
+ def test_resources_spec_get_param_path_scope_main(null_config) -> None:
64
69
  rs1 = hf.ResourceSpec(scope=hf.ActionScope.main())
65
70
  assert rs1.normalised_path == "resources.main"
66
71
 
67
72
 
68
- def test_resources_spec_get_param_path_scope_with_kwargs(null_config):
73
+ def test_resources_spec_get_param_path_scope_with_kwargs(null_config) -> None:
69
74
  rs1 = hf.ResourceSpec(scope=hf.ActionScope.input_file_generator(file="file1"))
70
75
  assert rs1.normalised_path == "resources.input_file_generator[file=file1]"
71
76
 
72
77
 
73
- def test_resources_spec_get_param_path_scope_with_no_kwargs(null_config):
78
+ def test_resources_spec_get_param_path_scope_with_no_kwargs(null_config) -> None:
74
79
  rs1 = hf.ResourceSpec(scope=hf.ActionScope.input_file_generator())
75
80
  assert rs1.normalised_path == "resources.input_file_generator"
76
81
 
77
82
 
78
- def test_input_value_from_json_like_class_method_attribute_is_set(null_config):
83
+ def test_input_value_from_json_like_class_method_attribute_is_set(null_config) -> None:
79
84
  parameter_typ = "p1"
80
85
  cls_method = "from_data"
81
86
  json_like = {"parameter": f"{parameter_typ}::{cls_method}", "value": 101}
@@ -84,7 +89,7 @@ def test_input_value_from_json_like_class_method_attribute_is_set(null_config):
84
89
  assert inp_val.value_class_method == cls_method
85
90
 
86
91
 
87
- def test_value_sequence_from_json_like_class_method_attribute_is_set(null_config):
92
+ def test_value_sequence_from_json_like_class_method_attribute_is_set(null_config) -> None:
88
93
  parameter_typ = "p1"
89
94
  cls_method = "from_data"
90
95
  json_like = {
@@ -99,28 +104,28 @@ def test_value_sequence_from_json_like_class_method_attribute_is_set(null_config
99
104
  assert val_seq.value_class_method == cls_method
100
105
 
101
106
 
102
- def test_path_attributes(null_config):
107
+ def test_path_attributes(null_config) -> None:
103
108
  inp = hf.InputValue(parameter="p1", value=101, path="a.b")
104
109
  assert inp.labelled_type == "p1"
105
110
  assert inp.normalised_path == "inputs.p1.a.b"
106
111
  assert inp.normalised_inputs_path == "p1.a.b"
107
112
 
108
113
 
109
- def test_path_attributes_with_label_arg(null_config):
114
+ def test_path_attributes_with_label_arg(null_config) -> None:
110
115
  inp = hf.InputValue(parameter="p1", value=101, path="a.b", label="1")
111
116
  assert inp.labelled_type == "p1[1]"
112
117
  assert inp.normalised_path == "inputs.p1[1].a.b"
113
118
  assert inp.normalised_inputs_path == "p1[1].a.b"
114
119
 
115
120
 
116
- def test_path_attributes_with_label_arg_cast(null_config):
121
+ def test_path_attributes_with_label_arg_cast(null_config) -> None:
117
122
  inp = hf.InputValue(parameter="p1", value=101, path="a.b", label=1)
118
123
  assert inp.labelled_type == "p1[1]"
119
124
  assert inp.normalised_path == "inputs.p1[1].a.b"
120
125
  assert inp.normalised_inputs_path == "p1[1].a.b"
121
126
 
122
127
 
123
- def test_from_json_like(null_config):
128
+ def test_from_json_like(null_config) -> None:
124
129
  inp = hf.InputValue.from_json_like(
125
130
  json_like={"parameter": "p1", "value": 101},
126
131
  shared_data=hf.template_components,
@@ -130,7 +135,7 @@ def test_from_json_like(null_config):
130
135
  assert inp.label == ""
131
136
 
132
137
 
133
- def test_from_json_like_with_label(null_config):
138
+ def test_from_json_like_with_label(null_config) -> None:
134
139
  inp = hf.InputValue.from_json_like(
135
140
  json_like={"parameter": "p1[1]", "value": 101},
136
141
  shared_data=hf.template_components,
@@ -140,12 +145,12 @@ def test_from_json_like_with_label(null_config):
140
145
  assert inp.label == "1"
141
146
 
142
147
 
143
- def test_value_is_dict_check_success(null_config):
148
+ def test_value_is_dict_check_success(null_config) -> None:
144
149
  # Parameter("p1c") has an associated `ParameterValue` class, so data should be a dict:
145
150
  hf.InputValue("p1c", {"a": 101})
146
151
 
147
152
 
148
- def test_value_is_dict_check_raise(null_config):
153
+ def test_value_is_dict_check_raise(null_config) -> None:
149
154
  # Parameter("p1c") has an associated `ParameterValue` class so data should be a dict:
150
155
  with pytest.raises(ValueError):
151
156
  hf.InputValue("p1c", 101)
@@ -165,7 +170,7 @@ def test_value_is_dict_check_no_raise_if_sub_parameter(null_config):
165
170
  "retrieving demo data from GitHub."
166
171
  ),
167
172
  )
168
- def test_demo_data_value(null_config):
173
+ def test_demo_data_value(null_config) -> None:
169
174
  name = "text_file.txt"
170
175
  assert hf.InputValue("p1", value=f"<<demo_data_file:{name}>>").value == str(
171
176
  hf.demo_data_cache_dir.joinpath(name)