hpcflow 0.1.15__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. hpcflow/__init__.py +2 -11
  2. hpcflow/__pyinstaller/__init__.py +5 -0
  3. hpcflow/__pyinstaller/hook-hpcflow.py +40 -0
  4. hpcflow/_version.py +1 -1
  5. hpcflow/app.py +43 -0
  6. hpcflow/cli.py +2 -461
  7. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  8. hpcflow/data/demo_data_manifest/demo_data_manifest.json +6 -0
  9. hpcflow/data/jinja_templates/test/test_template.txt +8 -0
  10. hpcflow/data/programs/hello_world/README.md +1 -0
  11. hpcflow/data/programs/hello_world/hello_world.c +87 -0
  12. hpcflow/data/programs/hello_world/linux/hello_world +0 -0
  13. hpcflow/data/programs/hello_world/macos/hello_world +0 -0
  14. hpcflow/data/programs/hello_world/win/hello_world.exe +0 -0
  15. hpcflow/data/scripts/__init__.py +1 -0
  16. hpcflow/data/scripts/bad_script.py +2 -0
  17. hpcflow/data/scripts/demo_task_1_generate_t1_infile_1.py +8 -0
  18. hpcflow/data/scripts/demo_task_1_generate_t1_infile_2.py +8 -0
  19. hpcflow/data/scripts/demo_task_1_parse_p3.py +7 -0
  20. hpcflow/data/scripts/do_nothing.py +2 -0
  21. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  22. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  23. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  24. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  25. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  26. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  27. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  28. hpcflow/data/scripts/generate_t1_file_01.py +7 -0
  29. hpcflow/data/scripts/import_future_script.py +7 -0
  30. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  31. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  32. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  33. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  34. hpcflow/data/scripts/main_script_test_direct_in_direct_out.py +6 -0
  35. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  36. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  37. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  38. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  39. hpcflow/data/scripts/main_script_test_direct_in_direct_out_all_iters_test.py +15 -0
  40. hpcflow/data/scripts/main_script_test_direct_in_direct_out_env_spec.py +7 -0
  41. hpcflow/data/scripts/main_script_test_direct_in_direct_out_labels.py +8 -0
  42. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  43. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  44. hpcflow/data/scripts/main_script_test_direct_sub_param_in_direct_out.py +6 -0
  45. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +12 -0
  46. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  47. hpcflow/data/scripts/main_script_test_hdf5_in_obj_group.py +12 -0
  48. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +11 -0
  49. hpcflow/data/scripts/main_script_test_json_and_direct_in_json_out.py +14 -0
  50. hpcflow/data/scripts/main_script_test_json_in_json_and_direct_out.py +17 -0
  51. hpcflow/data/scripts/main_script_test_json_in_json_out.py +14 -0
  52. hpcflow/data/scripts/main_script_test_json_in_json_out_labels.py +16 -0
  53. hpcflow/data/scripts/main_script_test_json_in_obj.py +12 -0
  54. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  55. hpcflow/data/scripts/main_script_test_json_out_obj.py +10 -0
  56. hpcflow/data/scripts/main_script_test_json_sub_param_in_json_out_labels.py +16 -0
  57. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  58. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  59. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  60. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  61. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  62. hpcflow/data/scripts/parse_t1_file_01.py +4 -0
  63. hpcflow/data/scripts/script_exit_test.py +5 -0
  64. hpcflow/data/template_components/__init__.py +1 -0
  65. hpcflow/data/template_components/command_files.yaml +26 -0
  66. hpcflow/data/template_components/environments.yaml +13 -0
  67. hpcflow/data/template_components/parameters.yaml +14 -0
  68. hpcflow/data/template_components/task_schemas.yaml +139 -0
  69. hpcflow/data/workflows/workflow_1.yaml +5 -0
  70. hpcflow/examples.ipynb +1037 -0
  71. hpcflow/sdk/__init__.py +149 -0
  72. hpcflow/sdk/app.py +4266 -0
  73. hpcflow/sdk/cli.py +1479 -0
  74. hpcflow/sdk/cli_common.py +385 -0
  75. hpcflow/sdk/config/__init__.py +5 -0
  76. hpcflow/sdk/config/callbacks.py +246 -0
  77. hpcflow/sdk/config/cli.py +388 -0
  78. hpcflow/sdk/config/config.py +1410 -0
  79. hpcflow/sdk/config/config_file.py +501 -0
  80. hpcflow/sdk/config/errors.py +272 -0
  81. hpcflow/sdk/config/types.py +150 -0
  82. hpcflow/sdk/core/__init__.py +38 -0
  83. hpcflow/sdk/core/actions.py +3857 -0
  84. hpcflow/sdk/core/app_aware.py +25 -0
  85. hpcflow/sdk/core/cache.py +224 -0
  86. hpcflow/sdk/core/command_files.py +814 -0
  87. hpcflow/sdk/core/commands.py +424 -0
  88. hpcflow/sdk/core/element.py +2071 -0
  89. hpcflow/sdk/core/enums.py +221 -0
  90. hpcflow/sdk/core/environment.py +256 -0
  91. hpcflow/sdk/core/errors.py +1043 -0
  92. hpcflow/sdk/core/execute.py +207 -0
  93. hpcflow/sdk/core/json_like.py +809 -0
  94. hpcflow/sdk/core/loop.py +1320 -0
  95. hpcflow/sdk/core/loop_cache.py +282 -0
  96. hpcflow/sdk/core/object_list.py +933 -0
  97. hpcflow/sdk/core/parameters.py +3371 -0
  98. hpcflow/sdk/core/rule.py +196 -0
  99. hpcflow/sdk/core/run_dir_files.py +57 -0
  100. hpcflow/sdk/core/skip_reason.py +7 -0
  101. hpcflow/sdk/core/task.py +3792 -0
  102. hpcflow/sdk/core/task_schema.py +993 -0
  103. hpcflow/sdk/core/test_utils.py +538 -0
  104. hpcflow/sdk/core/types.py +447 -0
  105. hpcflow/sdk/core/utils.py +1207 -0
  106. hpcflow/sdk/core/validation.py +87 -0
  107. hpcflow/sdk/core/values.py +477 -0
  108. hpcflow/sdk/core/workflow.py +4820 -0
  109. hpcflow/sdk/core/zarr_io.py +206 -0
  110. hpcflow/sdk/data/__init__.py +13 -0
  111. hpcflow/sdk/data/config_file_schema.yaml +34 -0
  112. hpcflow/sdk/data/config_schema.yaml +260 -0
  113. hpcflow/sdk/data/environments_spec_schema.yaml +21 -0
  114. hpcflow/sdk/data/files_spec_schema.yaml +5 -0
  115. hpcflow/sdk/data/parameters_spec_schema.yaml +7 -0
  116. hpcflow/sdk/data/task_schema_spec_schema.yaml +3 -0
  117. hpcflow/sdk/data/workflow_spec_schema.yaml +22 -0
  118. hpcflow/sdk/demo/__init__.py +3 -0
  119. hpcflow/sdk/demo/cli.py +242 -0
  120. hpcflow/sdk/helper/__init__.py +3 -0
  121. hpcflow/sdk/helper/cli.py +137 -0
  122. hpcflow/sdk/helper/helper.py +300 -0
  123. hpcflow/sdk/helper/watcher.py +192 -0
  124. hpcflow/sdk/log.py +288 -0
  125. hpcflow/sdk/persistence/__init__.py +18 -0
  126. hpcflow/sdk/persistence/base.py +2817 -0
  127. hpcflow/sdk/persistence/defaults.py +6 -0
  128. hpcflow/sdk/persistence/discovery.py +39 -0
  129. hpcflow/sdk/persistence/json.py +954 -0
  130. hpcflow/sdk/persistence/pending.py +948 -0
  131. hpcflow/sdk/persistence/store_resource.py +203 -0
  132. hpcflow/sdk/persistence/types.py +309 -0
  133. hpcflow/sdk/persistence/utils.py +73 -0
  134. hpcflow/sdk/persistence/zarr.py +2388 -0
  135. hpcflow/sdk/runtime.py +320 -0
  136. hpcflow/sdk/submission/__init__.py +3 -0
  137. hpcflow/sdk/submission/enums.py +70 -0
  138. hpcflow/sdk/submission/jobscript.py +2379 -0
  139. hpcflow/sdk/submission/schedulers/__init__.py +281 -0
  140. hpcflow/sdk/submission/schedulers/direct.py +233 -0
  141. hpcflow/sdk/submission/schedulers/sge.py +376 -0
  142. hpcflow/sdk/submission/schedulers/slurm.py +598 -0
  143. hpcflow/sdk/submission/schedulers/utils.py +25 -0
  144. hpcflow/sdk/submission/shells/__init__.py +52 -0
  145. hpcflow/sdk/submission/shells/base.py +229 -0
  146. hpcflow/sdk/submission/shells/bash.py +504 -0
  147. hpcflow/sdk/submission/shells/os_version.py +115 -0
  148. hpcflow/sdk/submission/shells/powershell.py +352 -0
  149. hpcflow/sdk/submission/submission.py +1402 -0
  150. hpcflow/sdk/submission/types.py +140 -0
  151. hpcflow/sdk/typing.py +194 -0
  152. hpcflow/sdk/utils/arrays.py +69 -0
  153. hpcflow/sdk/utils/deferred_file.py +55 -0
  154. hpcflow/sdk/utils/hashing.py +16 -0
  155. hpcflow/sdk/utils/patches.py +31 -0
  156. hpcflow/sdk/utils/strings.py +69 -0
  157. hpcflow/tests/api/test_api.py +32 -0
  158. hpcflow/tests/conftest.py +123 -0
  159. hpcflow/tests/data/__init__.py +0 -0
  160. hpcflow/tests/data/benchmark_N_elements.yaml +6 -0
  161. hpcflow/tests/data/benchmark_script_runner.yaml +26 -0
  162. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  163. hpcflow/tests/data/workflow_1.json +10 -0
  164. hpcflow/tests/data/workflow_1.yaml +5 -0
  165. hpcflow/tests/data/workflow_1_slurm.yaml +8 -0
  166. hpcflow/tests/data/workflow_1_wsl.yaml +8 -0
  167. hpcflow/tests/data/workflow_test_run_abort.yaml +42 -0
  168. hpcflow/tests/jinja_templates/test_jinja_templates.py +161 -0
  169. hpcflow/tests/programs/test_programs.py +180 -0
  170. hpcflow/tests/schedulers/direct_linux/test_direct_linux_submission.py +12 -0
  171. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  172. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +14 -0
  173. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  174. hpcflow/tests/scripts/test_main_scripts.py +1361 -0
  175. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  176. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  177. hpcflow/tests/shells/wsl/test_wsl_submission.py +14 -0
  178. hpcflow/tests/unit/test_action.py +1066 -0
  179. hpcflow/tests/unit/test_action_rule.py +24 -0
  180. hpcflow/tests/unit/test_app.py +132 -0
  181. hpcflow/tests/unit/test_cache.py +46 -0
  182. hpcflow/tests/unit/test_cli.py +172 -0
  183. hpcflow/tests/unit/test_command.py +377 -0
  184. hpcflow/tests/unit/test_config.py +195 -0
  185. hpcflow/tests/unit/test_config_file.py +162 -0
  186. hpcflow/tests/unit/test_element.py +666 -0
  187. hpcflow/tests/unit/test_element_iteration.py +88 -0
  188. hpcflow/tests/unit/test_element_set.py +158 -0
  189. hpcflow/tests/unit/test_group.py +115 -0
  190. hpcflow/tests/unit/test_input_source.py +1479 -0
  191. hpcflow/tests/unit/test_input_value.py +398 -0
  192. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  193. hpcflow/tests/unit/test_json_like.py +1247 -0
  194. hpcflow/tests/unit/test_loop.py +2674 -0
  195. hpcflow/tests/unit/test_meta_task.py +325 -0
  196. hpcflow/tests/unit/test_multi_path_sequences.py +259 -0
  197. hpcflow/tests/unit/test_object_list.py +116 -0
  198. hpcflow/tests/unit/test_parameter.py +243 -0
  199. hpcflow/tests/unit/test_persistence.py +664 -0
  200. hpcflow/tests/unit/test_resources.py +243 -0
  201. hpcflow/tests/unit/test_run.py +286 -0
  202. hpcflow/tests/unit/test_run_directories.py +29 -0
  203. hpcflow/tests/unit/test_runtime.py +9 -0
  204. hpcflow/tests/unit/test_schema_input.py +372 -0
  205. hpcflow/tests/unit/test_shell.py +129 -0
  206. hpcflow/tests/unit/test_slurm.py +39 -0
  207. hpcflow/tests/unit/test_submission.py +502 -0
  208. hpcflow/tests/unit/test_task.py +2560 -0
  209. hpcflow/tests/unit/test_task_schema.py +182 -0
  210. hpcflow/tests/unit/test_utils.py +616 -0
  211. hpcflow/tests/unit/test_value_sequence.py +549 -0
  212. hpcflow/tests/unit/test_values.py +91 -0
  213. hpcflow/tests/unit/test_workflow.py +827 -0
  214. hpcflow/tests/unit/test_workflow_template.py +186 -0
  215. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  216. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  217. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  218. hpcflow/tests/unit/utils/test_patches.py +5 -0
  219. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  220. hpcflow/tests/unit/utils/test_strings.py +97 -0
  221. hpcflow/tests/workflows/__init__.py +0 -0
  222. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  223. hpcflow/tests/workflows/test_jobscript.py +355 -0
  224. hpcflow/tests/workflows/test_run_status.py +198 -0
  225. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  226. hpcflow/tests/workflows/test_submission.py +140 -0
  227. hpcflow/tests/workflows/test_workflows.py +564 -0
  228. hpcflow/tests/workflows/test_zip.py +18 -0
  229. hpcflow/viz_demo.ipynb +6794 -0
  230. hpcflow-0.2.0a271.dist-info/LICENSE +375 -0
  231. hpcflow-0.2.0a271.dist-info/METADATA +65 -0
  232. hpcflow-0.2.0a271.dist-info/RECORD +237 -0
  233. {hpcflow-0.1.15.dist-info → hpcflow-0.2.0a271.dist-info}/WHEEL +4 -5
  234. hpcflow-0.2.0a271.dist-info/entry_points.txt +6 -0
  235. hpcflow/api.py +0 -490
  236. hpcflow/archive/archive.py +0 -307
  237. hpcflow/archive/cloud/cloud.py +0 -45
  238. hpcflow/archive/cloud/errors.py +0 -9
  239. hpcflow/archive/cloud/providers/dropbox.py +0 -427
  240. hpcflow/archive/errors.py +0 -5
  241. hpcflow/base_db.py +0 -4
  242. hpcflow/config.py +0 -233
  243. hpcflow/copytree.py +0 -66
  244. hpcflow/data/examples/_config.yml +0 -14
  245. hpcflow/data/examples/damask/demo/1.run.yml +0 -4
  246. hpcflow/data/examples/damask/demo/2.process.yml +0 -29
  247. hpcflow/data/examples/damask/demo/geom.geom +0 -2052
  248. hpcflow/data/examples/damask/demo/load.load +0 -1
  249. hpcflow/data/examples/damask/demo/material.config +0 -185
  250. hpcflow/data/examples/damask/inputs/geom.geom +0 -2052
  251. hpcflow/data/examples/damask/inputs/load.load +0 -1
  252. hpcflow/data/examples/damask/inputs/material.config +0 -185
  253. hpcflow/data/examples/damask/profiles/_variable_lookup.yml +0 -21
  254. hpcflow/data/examples/damask/profiles/damask.yml +0 -4
  255. hpcflow/data/examples/damask/profiles/damask_process.yml +0 -8
  256. hpcflow/data/examples/damask/profiles/damask_run.yml +0 -5
  257. hpcflow/data/examples/damask/profiles/default.yml +0 -6
  258. hpcflow/data/examples/thinking.yml +0 -177
  259. hpcflow/errors.py +0 -2
  260. hpcflow/init_db.py +0 -37
  261. hpcflow/models.py +0 -2595
  262. hpcflow/nesting.py +0 -9
  263. hpcflow/profiles.py +0 -455
  264. hpcflow/project.py +0 -81
  265. hpcflow/scheduler.py +0 -322
  266. hpcflow/utils.py +0 -103
  267. hpcflow/validation.py +0 -166
  268. hpcflow/variables.py +0 -543
  269. hpcflow-0.1.15.dist-info/METADATA +0 -168
  270. hpcflow-0.1.15.dist-info/RECORD +0 -45
  271. hpcflow-0.1.15.dist-info/entry_points.txt +0 -8
  272. hpcflow-0.1.15.dist-info/top_level.txt +0 -1
  273. /hpcflow/{archive → data/jinja_templates}/__init__.py +0 -0
  274. /hpcflow/{archive/cloud → data/programs}/__init__.py +0 -0
  275. /hpcflow/{archive/cloud/providers → data/workflows}/__init__.py +0 -0
@@ -0,0 +1,549 @@
1
+ import sys
2
+ from textwrap import dedent
3
+ import numpy as np
4
+ from pathlib import Path
5
+ import pytest
6
+ import requests
7
+
8
+ from hpcflow.app import app as hf
9
+ from hpcflow.sdk.core.utils import read_YAML_str
10
+ from hpcflow.sdk.core.test_utils import P1_parameter_cls as P1
11
+
12
+
13
+ @pytest.fixture
14
+ def null_config(tmp_path: Path):
15
+ if not hf.is_config_loaded:
16
+ hf.load_config(config_dir=tmp_path)
17
+
18
+
19
+ def test_inputs_value_sequence_label_added_to_path():
20
+ seq = hf.ValueSequence(path="inputs.p1.a", values=[0, 1], nesting_order=0, label=0)
21
+ assert seq.path == "inputs.p1[0].a"
22
+
23
+
24
+ def test_inputs_value_sequence_no_label_added_to_path():
25
+ seq = hf.ValueSequence(path="inputs.p1.a", values=[0, 1], nesting_order=0, label="")
26
+ assert seq.path == "inputs.p1.a"
27
+
28
+
29
+ def test_inputs_value_sequence_label_attr_added():
30
+ seq = hf.ValueSequence(path="inputs.p1[1].a", values=[0, 1], nesting_order=0)
31
+ assert seq.label == "1"
32
+
33
+
34
+ def test_inputs_value_sequence_label_path_unmodified():
35
+ path = "inputs.p1[1].a"
36
+ seq = hf.ValueSequence(path=path, values=[0, 1], nesting_order=0)
37
+ assert seq.path == path
38
+
39
+
40
+ def test_raise_on_inputs_value_sequence_label_path_unequal():
41
+ with pytest.raises(ValueError):
42
+ hf.ValueSequence(path="inputs.p1[1].a", values=[0, 1], nesting_order=0, label="2")
43
+
44
+
45
+ def test_no_raise_on_inputs_value_sequence_label_path_equal():
46
+ hf.ValueSequence(path="inputs.p1[1].a", values=[0, 1], nesting_order=0, label="1")
47
+
48
+
49
+ def test_no_raise_on_inputs_value_sequence_label_path_cast_equal():
50
+ hf.ValueSequence(path="inputs.p1[1].a", values=[0, 1], nesting_order=0, label=1)
51
+
52
+
53
+ def test_raise_on_resources_value_sequence_with_path_label():
54
+ with pytest.raises(ValueError):
55
+ hf.ValueSequence(path="resources.main[1]", values=[0, 1], nesting_order=0)
56
+
57
+
58
+ def test_raise_on_resources_value_sequence_with_label_arg():
59
+ with pytest.raises(ValueError):
60
+ hf.ValueSequence(path="resources.main", values=[0, 1], nesting_order=0, label=1)
61
+
62
+
63
+ def test_inputs_value_sequence_simple_path_attributes():
64
+ path = "inputs.p1"
65
+ seq = hf.ValueSequence(path=path, values=[0, 1], nesting_order=0)
66
+ assert seq.path == path
67
+ assert seq.labelled_type == "p1"
68
+ assert seq.normalised_path == "inputs.p1"
69
+ assert seq.normalised_inputs_path == "p1"
70
+ assert seq.path_type == "inputs"
71
+ assert seq.input_type == "p1"
72
+ assert seq.input_path == ""
73
+ assert seq.resource_scope is None
74
+
75
+
76
+ def test_inputs_value_sequence_path_attributes():
77
+ path = "inputs.p1.a.b"
78
+ seq = hf.ValueSequence(path=path, values=[0, 1], nesting_order=0)
79
+ assert seq.path == path
80
+ assert seq.labelled_type == "p1"
81
+ assert seq.normalised_path == "inputs.p1.a.b"
82
+ assert seq.normalised_inputs_path == "p1.a.b"
83
+ assert seq.path_type == "inputs"
84
+ assert seq.input_type == "p1"
85
+ assert seq.input_path == "a.b"
86
+ assert seq.resource_scope is None
87
+
88
+
89
+ def test_inputs_value_sequence_with_path_label_path_attributes():
90
+ path = "inputs.p1[1].a.b"
91
+ seq = hf.ValueSequence(path=path, values=[0, 1], nesting_order=0)
92
+ assert seq.path == path
93
+ assert seq.labelled_type == "p1[1]"
94
+ assert seq.normalised_path == "inputs.p1[1].a.b"
95
+ assert seq.normalised_inputs_path == "p1[1].a.b"
96
+ assert seq.path_type == "inputs"
97
+ assert seq.input_type == "p1"
98
+ assert seq.input_path == "a.b"
99
+ assert seq.resource_scope is None
100
+
101
+
102
+ def test_inputs_value_sequence_with_arg_label_path_attributes():
103
+ path = "inputs.p1.a.b"
104
+ new_path = "inputs.p1[1].a.b"
105
+ seq = hf.ValueSequence(path=path, values=[0, 1], nesting_order=0, label=1)
106
+ assert seq.path == new_path
107
+ assert seq.labelled_type == "p1[1]"
108
+ assert seq.normalised_path == "inputs.p1[1].a.b"
109
+ assert seq.normalised_inputs_path == "p1[1].a.b"
110
+ assert seq.path_type == "inputs"
111
+ assert seq.input_type == "p1"
112
+ assert seq.input_path == "a.b"
113
+ assert seq.resource_scope is None
114
+
115
+
116
+ def test_resources_value_sequence_path_attributes():
117
+ path = "resources.main.num_cores"
118
+ seq = hf.ValueSequence(path=path, values=[0, 1], nesting_order=0)
119
+ assert seq.path == path
120
+ assert seq.labelled_type is None
121
+ assert seq.normalised_path == "resources.main.num_cores"
122
+ assert seq.normalised_inputs_path is None
123
+ assert seq.path_type == "resources"
124
+ assert seq.input_type is None
125
+ assert seq.input_path is None
126
+ assert seq.resource_scope == "main"
127
+
128
+
129
+ @pytest.mark.parametrize("store", ["json", "zarr"])
130
+ def test_value_sequence_object_values_during_workflow_init(
131
+ null_config, tmp_path: Path, store: str
132
+ ):
133
+ p1 = hf.Parameter("p1c")
134
+ s1 = hf.TaskSchema(objective="t1", inputs=[hf.SchemaInput(parameter=p1)])
135
+ obj = P1(a=101)
136
+ seq = hf.ValueSequence(path="inputs.p1c", values=[obj], nesting_order=0)
137
+ values_exp = [P1(a=101, d=None)]
138
+
139
+ t1 = hf.Task(
140
+ schema=[s1],
141
+ sequences=[seq],
142
+ )
143
+ # before workflow initialisation:
144
+ assert seq.values == values_exp
145
+
146
+ wk = hf.Workflow.from_template_data(
147
+ tasks=[],
148
+ path=tmp_path,
149
+ template_name="temp",
150
+ store=store,
151
+ )
152
+
153
+ with wk.batch_update():
154
+ wk.add_task(t1)
155
+ # after workflow initialisation but before store commit:
156
+ assert wk.tasks[0].template.element_sets[0].sequences[0].values == values_exp
157
+
158
+ # after initialisation and store commit:
159
+ assert wk.tasks[0].template.element_sets[0].sequences[0].values == values_exp
160
+
161
+
162
+ @pytest.mark.parametrize("store", ["json", "zarr"])
163
+ def test_value_sequence_object_values_class_method_during_workflow_init(
164
+ null_config, tmp_path: Path, store: str
165
+ ):
166
+ p1 = hf.Parameter("p1c")
167
+ s1 = hf.TaskSchema(objective="t1", inputs=[hf.SchemaInput(parameter=p1)])
168
+ obj = P1.from_data(b=50, c=51)
169
+ seq = hf.ValueSequence(path="inputs.p1c", values=[obj], nesting_order=0)
170
+ values_exp = [P1(a=101, d=None)]
171
+
172
+ t1 = hf.Task(
173
+ schema=[s1],
174
+ sequences=[seq],
175
+ )
176
+ # before workflow initialisation:
177
+ assert seq.values == values_exp
178
+
179
+ wk = hf.Workflow.from_template_data(
180
+ tasks=[],
181
+ path=tmp_path,
182
+ template_name="temp",
183
+ store=store,
184
+ )
185
+
186
+ with wk.batch_update():
187
+ wk.add_task(t1)
188
+ # after workflow initialisation but before store commit:
189
+ assert wk.tasks[0].template.element_sets[0].sequences[0].values == values_exp
190
+
191
+ # after initialisation and store commit:
192
+ assert wk.tasks[0].template.element_sets[0].sequences[0].values == values_exp
193
+
194
+
195
+ @pytest.mark.parametrize("store", ["json", "zarr"])
196
+ def test_value_sequence_object_values_named_class_method_during_workflow_init(
197
+ null_config, tmp_path: Path, store: str
198
+ ):
199
+ p1 = hf.Parameter("p1c")
200
+ s1 = hf.TaskSchema(objective="t1", inputs=[hf.SchemaInput(parameter=p1)])
201
+ data = {"b": 50, "c": 51}
202
+ seq = hf.ValueSequence(
203
+ path="inputs.p1c", values=[data], nesting_order=0, value_class_method="from_data"
204
+ )
205
+ values_exp = [data]
206
+
207
+ t1 = hf.Task(
208
+ schema=[s1],
209
+ sequences=[seq],
210
+ )
211
+ # before workflow initialisation:
212
+ assert seq.values == values_exp
213
+
214
+ wk = hf.Workflow.from_template_data(
215
+ tasks=[],
216
+ path=tmp_path,
217
+ template_name="temp",
218
+ store=store,
219
+ )
220
+
221
+ with wk.batch_update():
222
+ wk.add_task(t1)
223
+ # after workflow initialisation but before store commit:
224
+ assert wk.tasks[0].template.element_sets[0].sequences[0].values == values_exp
225
+
226
+ # after initialisation and store commit:
227
+ assert wk.tasks[0].template.element_sets[0].sequences[0].values == values_exp
228
+
229
+
230
+ def test_nesting_order_two_seqs_parallel(null_config, tmp_path: Path):
231
+ ts = hf.TaskSchema(
232
+ objective="test", inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")]
233
+ )
234
+ t1 = hf.Task(
235
+ schema=ts,
236
+ sequences=[
237
+ hf.ValueSequence("inputs.p1", values=["a", "b"], nesting_order=0),
238
+ hf.ValueSequence("inputs.p2", values=["c", "d"], nesting_order=0),
239
+ ],
240
+ )
241
+ wk = hf.Workflow.from_template_data(
242
+ template_name="test",
243
+ tasks=[t1],
244
+ path=tmp_path,
245
+ )
246
+ assert wk.tasks.test.num_elements == 2
247
+ assert wk.tasks.test.elements[0].get("inputs") == {"p1": "a", "p2": "c"}
248
+ assert wk.tasks.test.elements[1].get("inputs") == {"p1": "b", "p2": "d"}
249
+
250
+
251
+ def test_nesting_order_two_seqs_parallel_decimal_equiv(null_config, tmp_path: Path):
252
+ ts = hf.TaskSchema(
253
+ objective="test", inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")]
254
+ )
255
+ t1 = hf.Task(
256
+ schema=ts,
257
+ sequences=[
258
+ hf.ValueSequence("inputs.p1", values=["a", "b"], nesting_order=1.0),
259
+ hf.ValueSequence("inputs.p2", values=["c", "d"], nesting_order=1.0),
260
+ ],
261
+ )
262
+ wk = hf.Workflow.from_template_data(
263
+ template_name="test",
264
+ tasks=[t1],
265
+ path=tmp_path,
266
+ )
267
+ assert wk.tasks.test.num_elements == 2
268
+ assert wk.tasks.test.elements[0].get("inputs") == {"p1": "a", "p2": "c"}
269
+ assert wk.tasks.test.elements[1].get("inputs") == {"p1": "b", "p2": "d"}
270
+
271
+
272
+ def test_nesting_order_two_seqs_nested(
273
+ null_config,
274
+ tmp_path: Path,
275
+ ):
276
+ ts = hf.TaskSchema(
277
+ objective="test", inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")]
278
+ )
279
+ t1 = hf.Task(
280
+ schema=ts,
281
+ sequences=[
282
+ hf.ValueSequence("inputs.p1", values=["a", "b"], nesting_order=0),
283
+ hf.ValueSequence("inputs.p2", values=["c", "d"], nesting_order=1),
284
+ ],
285
+ )
286
+ wk = hf.Workflow.from_template_data(
287
+ template_name="test",
288
+ tasks=[t1],
289
+ path=tmp_path,
290
+ )
291
+ assert wk.tasks.test.num_elements == 4
292
+ assert wk.tasks.test.elements[0].get("inputs") == {"p1": "a", "p2": "c"}
293
+ assert wk.tasks.test.elements[1].get("inputs") == {"p1": "a", "p2": "d"}
294
+ assert wk.tasks.test.elements[2].get("inputs") == {"p1": "b", "p2": "c"}
295
+ assert wk.tasks.test.elements[3].get("inputs") == {"p1": "b", "p2": "d"}
296
+
297
+
298
+ def test_nesting_order_two_seqs_default_nesting_order(null_config, tmp_path: Path):
299
+ ts = hf.TaskSchema(
300
+ objective="test", inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")]
301
+ )
302
+ t1 = hf.Task(
303
+ schema=ts,
304
+ sequences=[
305
+ hf.ValueSequence("inputs.p1", values=["a", "b"]),
306
+ hf.ValueSequence("inputs.p2", values=["c", "d"]),
307
+ ],
308
+ )
309
+ wk = hf.Workflow.from_template_data(
310
+ template_name="test",
311
+ tasks=[t1],
312
+ path=tmp_path,
313
+ )
314
+ assert wk.tasks.test.num_elements == 2
315
+ assert wk.tasks.test.elements[0].get("inputs") == {"p1": "a", "p2": "c"}
316
+ assert wk.tasks.test.elements[1].get("inputs") == {"p1": "b", "p2": "d"}
317
+
318
+
319
+ def test_raise_nesting_order_two_seqs_default_nesting_order(null_config, tmp_path: Path):
320
+ ts = hf.TaskSchema(
321
+ objective="test", inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")]
322
+ )
323
+ t1 = hf.Task(
324
+ schema=ts,
325
+ sequences=[
326
+ hf.ValueSequence("inputs.p1", values=["a", "b"]),
327
+ hf.ValueSequence("inputs.p2", values=["c", "d", "e"]),
328
+ ],
329
+ )
330
+ with pytest.raises(ValueError):
331
+ wk = hf.Workflow.from_template_data(
332
+ template_name="test",
333
+ tasks=[t1],
334
+ path=tmp_path,
335
+ )
336
+
337
+
338
+ def test_raise_nesting_order_two_seqs_default_nesting_order_decimal(
339
+ null_config, tmp_path: Path
340
+ ):
341
+ ts = hf.TaskSchema(
342
+ objective="test", inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")]
343
+ )
344
+ t1 = hf.Task(
345
+ schema=ts,
346
+ sequences=[
347
+ hf.ValueSequence("inputs.p1", values=["a", "b"], nesting_order=0.0),
348
+ hf.ValueSequence("inputs.p2", values=["c", "d", "e"], nesting_order=0.0),
349
+ ],
350
+ )
351
+ with pytest.raises(ValueError):
352
+ wk = hf.Workflow.from_template_data(
353
+ template_name="test",
354
+ tasks=[t1],
355
+ path=tmp_path,
356
+ )
357
+
358
+
359
+ def test_nesting_order_three_seqs_decimal(null_config, tmp_path: Path):
360
+ ts = hf.TaskSchema(
361
+ objective="test",
362
+ inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2"), hf.SchemaInput("p3")],
363
+ )
364
+ t1 = hf.Task(
365
+ schema=ts,
366
+ sequences=[
367
+ hf.ValueSequence("inputs.p1", values=["a", "b"], nesting_order=0),
368
+ hf.ValueSequence("inputs.p2", values=["c", "d", "e"], nesting_order=1),
369
+ hf.ValueSequence(
370
+ "inputs.p3", values=["f", "g", "h", "i", "j", "k"], nesting_order=1.5
371
+ ),
372
+ ],
373
+ )
374
+ wk = hf.Workflow.from_template_data(
375
+ template_name="test",
376
+ tasks=[t1],
377
+ path=tmp_path,
378
+ )
379
+ assert wk.tasks.test.num_elements == 6
380
+ assert wk.tasks.test.elements[0].get("inputs") == {"p1": "a", "p2": "c", "p3": "f"}
381
+ assert wk.tasks.test.elements[1].get("inputs") == {"p1": "a", "p2": "d", "p3": "g"}
382
+ assert wk.tasks.test.elements[2].get("inputs") == {"p1": "a", "p2": "e", "p3": "h"}
383
+ assert wk.tasks.test.elements[3].get("inputs") == {"p1": "b", "p2": "c", "p3": "i"}
384
+ assert wk.tasks.test.elements[4].get("inputs") == {"p1": "b", "p2": "d", "p3": "j"}
385
+ assert wk.tasks.test.elements[5].get("inputs") == {"p1": "b", "p2": "e", "p3": "k"}
386
+
387
+
388
+ def test_nesting_order_three_seqs_all_decimal(null_config, tmp_path: Path):
389
+ ts = hf.TaskSchema(
390
+ objective="test",
391
+ inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2"), hf.SchemaInput("p3")],
392
+ )
393
+ t1 = hf.Task(
394
+ schema=ts,
395
+ sequences=[
396
+ hf.ValueSequence("inputs.p1", values=["a", "b"], nesting_order=0.5),
397
+ hf.ValueSequence("inputs.p2", values=["c", "d", "e"], nesting_order=1.2),
398
+ hf.ValueSequence(
399
+ "inputs.p3", values=["f", "g", "h", "i", "j", "k"], nesting_order=1.5
400
+ ),
401
+ ],
402
+ )
403
+ wk = hf.Workflow.from_template_data(
404
+ template_name="test",
405
+ tasks=[t1],
406
+ path=tmp_path,
407
+ )
408
+ assert wk.tasks.test.num_elements == 6
409
+ assert wk.tasks.test.elements[0].get("inputs") == {"p1": "a", "p2": "c", "p3": "f"}
410
+ assert wk.tasks.test.elements[1].get("inputs") == {"p1": "a", "p2": "d", "p3": "g"}
411
+ assert wk.tasks.test.elements[2].get("inputs") == {"p1": "a", "p2": "e", "p3": "h"}
412
+ assert wk.tasks.test.elements[3].get("inputs") == {"p1": "b", "p2": "c", "p3": "i"}
413
+ assert wk.tasks.test.elements[4].get("inputs") == {"p1": "b", "p2": "d", "p3": "j"}
414
+ assert wk.tasks.test.elements[5].get("inputs") == {"p1": "b", "p2": "e", "p3": "k"}
415
+
416
+
417
+ @pytest.mark.xfail(
418
+ condition=sys.platform == "darwin",
419
+ raises=requests.exceptions.HTTPError,
420
+ reason=(
421
+ "GHA MacOS runners use the same IP address, so we get rate limited when "
422
+ "retrieving demo data from GitHub."
423
+ ),
424
+ )
425
+ def test_demo_data_values(null_config):
426
+ name = "text_file.txt"
427
+ assert hf.ValueSequence(
428
+ path="inputs.p1", values=[f"<<demo_data_file:{name}>>"]
429
+ ).values[0] == str(hf.demo_data_cache_dir.joinpath(name))
430
+
431
+
432
+ def test_from_linear_space(null_config):
433
+ seq = hf.ValueSequence.from_linear_space(path="inputs.p1", start=0, stop=1, num=6)
434
+ assert np.allclose(seq.values, [0, 0.2, 0.4, 0.6, 0.8, 1.0])
435
+
436
+
437
+ def test_from_rectangle(null_config):
438
+ kwargs = dict(
439
+ path="inputs.p1",
440
+ start=[0, 0],
441
+ stop=[1, 1],
442
+ num=[2, 2],
443
+ )
444
+ seq_coord_0 = hf.ValueSequence.from_rectangle(**kwargs, coord=0)
445
+ seq_coord_1 = hf.ValueSequence.from_rectangle(**kwargs, coord=1)
446
+
447
+ assert np.allclose(seq_coord_0.values, [0, 1, 0, 1])
448
+ assert np.allclose(seq_coord_1.values, [1, 1, 0, 0])
449
+
450
+
451
+ def test_from_rectangle_coord_none(null_config):
452
+ kwargs = dict(
453
+ path="inputs.p1",
454
+ start=[0, 0],
455
+ stop=[1, 1],
456
+ num=[2, 2],
457
+ )
458
+ seq = hf.ValueSequence.from_rectangle(**kwargs)
459
+ assert np.allclose(seq.values, [[0, 1], [1, 1], [0, 0], [1, 0]])
460
+
461
+
462
+ def test_environments_sequence_to_resources(null_config):
463
+ seq = hf.ValueSequence(path="environments.my_env.version", values=[1, 2])
464
+ assert seq.path == "resources.any.environments.my_env.version"
465
+
466
+
467
+ def test_from_yaml_and_json_like_various(null_config):
468
+ seed = 13123
469
+ es_1 = dedent(
470
+ """\
471
+ sequences:
472
+ - path: inputs.p1c::from_data
473
+ nesting_order: 1
474
+ values: [100, 200]
475
+ """
476
+ )
477
+ es_2 = dedent(
478
+ f"""\
479
+ sequences:
480
+ - path: inputs.p1
481
+ nesting_order: 1
482
+ values::from_normal:
483
+ loc: 1.4
484
+ scale: 0.1
485
+ shape: 2
486
+ seed: {seed}
487
+ """
488
+ )
489
+
490
+ es_3 = dedent(
491
+ f"""\
492
+ sequences:
493
+ - path: inputs.p1c::from_data
494
+ nesting_order: 1
495
+ values::from_normal:
496
+ loc: 1.4
497
+ scale: 0.1
498
+ shape: 2
499
+ seed: {seed}
500
+ """
501
+ )
502
+
503
+ es_JSONs = [read_YAML_str(es_i) for es_i in (es_1, es_2, es_3)]
504
+ es = [hf.ElementSet.from_json_like(es_json_i) for es_json_i in es_JSONs]
505
+ seqs = [es_i.sequences[0] for es_i in es]
506
+
507
+ assert (
508
+ seqs[0]
509
+ == hf.ValueSequence.from_json_like(
510
+ {
511
+ "path": "inputs.p1c::from_data",
512
+ "values": [100, 200],
513
+ "nesting_order": 1,
514
+ }
515
+ )
516
+ == hf.ValueSequence(
517
+ "inputs.p1c",
518
+ values=[100, 200],
519
+ nesting_order=1,
520
+ value_class_method="from_data",
521
+ )
522
+ )
523
+
524
+ norm_args = {"loc": 1.4, "scale": 0.1, "shape": 2, "seed": seed}
525
+ assert (
526
+ seqs[1]
527
+ == hf.ValueSequence.from_json_like(
528
+ {
529
+ "path": "inputs.p1",
530
+ "values::from_normal": norm_args,
531
+ "nesting_order": 1,
532
+ }
533
+ )
534
+ == hf.ValueSequence.from_normal("inputs.p1", nesting_order=1, **norm_args)
535
+ )
536
+
537
+ assert (
538
+ seqs[2]
539
+ == hf.ValueSequence.from_json_like(
540
+ {
541
+ "path": "inputs.p1c::from_data",
542
+ "values::from_normal": norm_args,
543
+ "nesting_order": 1,
544
+ }
545
+ )
546
+ == hf.ValueSequence.from_normal(
547
+ "inputs.p1c", nesting_order=1, **norm_args, value_class_method="from_data"
548
+ )
549
+ )
@@ -0,0 +1,91 @@
1
+ """Testing of the ValuesMixin class"""
2
+
3
+ from pathlib import Path
4
+ from textwrap import dedent
5
+
6
+ from hpcflow.sdk.core.utils import read_YAML_str
7
+ from hpcflow.app import app as hf
8
+
9
+
10
+ def test_demo_data_paths_resolved_in_input_values(null_config):
11
+ assert Path(hf.InputValue("p1", "<<demo_data_file:text_file.txt>>").value).is_file()
12
+ assert Path(hf.InputValue("p1", "<<demo_data_file:zip_file.zip>>").value).is_file()
13
+
14
+
15
+ def test_demo_data_paths_resolved_in_input_values_class_methods(null_config):
16
+ assert hf.InputValue.from_file("p1", "<<demo_data_file:text_file.txt>>").value == [
17
+ str(i) for i in range(1, 11)
18
+ ]
19
+
20
+
21
+ def test_demo_data_paths_resolved_in_input_values_from_yaml(null_config):
22
+
23
+ es = dedent(
24
+ """\
25
+ inputs:
26
+ p1: <<demo_data_file:text_file.txt>>
27
+ p2: <<demo_data_file:zip_file.zip>>
28
+ """
29
+ )
30
+ es_JSON = read_YAML_str(es)
31
+ es = hf.ElementSet.from_json_like(es_JSON, shared_data=hf.template_components)
32
+
33
+ assert Path(es.inputs[0].value).is_file()
34
+ assert Path(es.inputs[1].value).is_file()
35
+
36
+
37
+ def test_demo_data_paths_resolved_in_input_values_class_methods_from_yaml(null_config):
38
+ es = dedent(
39
+ """\
40
+ inputs:
41
+ p1::from_file:
42
+ file_path: <<demo_data_file:text_file.txt>>
43
+ """
44
+ )
45
+ es_JSON = read_YAML_str(es)
46
+ es = hf.ElementSet.from_json_like(es_JSON, shared_data=hf.template_components)
47
+ assert es.inputs[0].value == [str(i) for i in range(1, 11)]
48
+
49
+
50
+ def test_demo_data_paths_resolved_in_value_sequences(null_config):
51
+ seqs = hf.ValueSequence(
52
+ "inputs.p1",
53
+ values=["<<demo_data_file:text_file.txt>>", "<<demo_data_file:zip_file.zip>>"],
54
+ )
55
+ assert all(Path(val_i).is_file() for val_i in seqs.values)
56
+
57
+
58
+ def test_demo_data_paths_resolved_in_value_sequences_class_methods(null_config):
59
+ assert hf.ValueSequence.from_file(
60
+ "inputs.p1",
61
+ file_path="<<demo_data_file:text_file.txt>>",
62
+ ).values == [str(i) for i in range(1, 11)]
63
+
64
+
65
+ def test_demo_data_paths_resolved_in_value_sequences_from_yaml(null_config):
66
+ es = dedent(
67
+ """\
68
+ sequences:
69
+ - path: inputs.p1
70
+ values:
71
+ - <<demo_data_file:text_file.txt>>
72
+ - <<demo_data_file:zip_file.zip>>
73
+ """
74
+ )
75
+ es_JSON = read_YAML_str(es)
76
+ es = hf.ElementSet.from_json_like(es_JSON, shared_data=hf.template_components)
77
+ assert all(Path(val_i).is_file() for val_i in es.sequences[0].values)
78
+
79
+
80
+ def test_demo_data_paths_resolved_in_value_sequences_from_yaml_class_methods(null_config):
81
+ es = dedent(
82
+ """\
83
+ sequences:
84
+ - path: inputs.p1
85
+ values::from_file:
86
+ file_path: <<demo_data_file:text_file.txt>>
87
+ """
88
+ )
89
+ es_JSON = read_YAML_str(es)
90
+ es = hf.ElementSet.from_json_like(es_JSON, shared_data=hf.template_components)
91
+ assert es.sequences[0].values == [str(i) for i in range(1, 11)]