hpcflow 0.1.15__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. hpcflow/__init__.py +2 -11
  2. hpcflow/__pyinstaller/__init__.py +5 -0
  3. hpcflow/__pyinstaller/hook-hpcflow.py +40 -0
  4. hpcflow/_version.py +1 -1
  5. hpcflow/app.py +43 -0
  6. hpcflow/cli.py +2 -461
  7. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  8. hpcflow/data/demo_data_manifest/demo_data_manifest.json +6 -0
  9. hpcflow/data/jinja_templates/test/test_template.txt +8 -0
  10. hpcflow/data/programs/hello_world/README.md +1 -0
  11. hpcflow/data/programs/hello_world/hello_world.c +87 -0
  12. hpcflow/data/programs/hello_world/linux/hello_world +0 -0
  13. hpcflow/data/programs/hello_world/macos/hello_world +0 -0
  14. hpcflow/data/programs/hello_world/win/hello_world.exe +0 -0
  15. hpcflow/data/scripts/__init__.py +1 -0
  16. hpcflow/data/scripts/bad_script.py +2 -0
  17. hpcflow/data/scripts/demo_task_1_generate_t1_infile_1.py +8 -0
  18. hpcflow/data/scripts/demo_task_1_generate_t1_infile_2.py +8 -0
  19. hpcflow/data/scripts/demo_task_1_parse_p3.py +7 -0
  20. hpcflow/data/scripts/do_nothing.py +2 -0
  21. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  22. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  23. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  24. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  25. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  26. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  27. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  28. hpcflow/data/scripts/generate_t1_file_01.py +7 -0
  29. hpcflow/data/scripts/import_future_script.py +7 -0
  30. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  31. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  32. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  33. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  34. hpcflow/data/scripts/main_script_test_direct_in_direct_out.py +6 -0
  35. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  36. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  37. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  38. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  39. hpcflow/data/scripts/main_script_test_direct_in_direct_out_all_iters_test.py +15 -0
  40. hpcflow/data/scripts/main_script_test_direct_in_direct_out_env_spec.py +7 -0
  41. hpcflow/data/scripts/main_script_test_direct_in_direct_out_labels.py +8 -0
  42. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  43. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  44. hpcflow/data/scripts/main_script_test_direct_sub_param_in_direct_out.py +6 -0
  45. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +12 -0
  46. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  47. hpcflow/data/scripts/main_script_test_hdf5_in_obj_group.py +12 -0
  48. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +11 -0
  49. hpcflow/data/scripts/main_script_test_json_and_direct_in_json_out.py +14 -0
  50. hpcflow/data/scripts/main_script_test_json_in_json_and_direct_out.py +17 -0
  51. hpcflow/data/scripts/main_script_test_json_in_json_out.py +14 -0
  52. hpcflow/data/scripts/main_script_test_json_in_json_out_labels.py +16 -0
  53. hpcflow/data/scripts/main_script_test_json_in_obj.py +12 -0
  54. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  55. hpcflow/data/scripts/main_script_test_json_out_obj.py +10 -0
  56. hpcflow/data/scripts/main_script_test_json_sub_param_in_json_out_labels.py +16 -0
  57. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  58. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  59. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  60. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  61. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  62. hpcflow/data/scripts/parse_t1_file_01.py +4 -0
  63. hpcflow/data/scripts/script_exit_test.py +5 -0
  64. hpcflow/data/template_components/__init__.py +1 -0
  65. hpcflow/data/template_components/command_files.yaml +26 -0
  66. hpcflow/data/template_components/environments.yaml +13 -0
  67. hpcflow/data/template_components/parameters.yaml +14 -0
  68. hpcflow/data/template_components/task_schemas.yaml +139 -0
  69. hpcflow/data/workflows/workflow_1.yaml +5 -0
  70. hpcflow/examples.ipynb +1037 -0
  71. hpcflow/sdk/__init__.py +149 -0
  72. hpcflow/sdk/app.py +4266 -0
  73. hpcflow/sdk/cli.py +1479 -0
  74. hpcflow/sdk/cli_common.py +385 -0
  75. hpcflow/sdk/config/__init__.py +5 -0
  76. hpcflow/sdk/config/callbacks.py +246 -0
  77. hpcflow/sdk/config/cli.py +388 -0
  78. hpcflow/sdk/config/config.py +1410 -0
  79. hpcflow/sdk/config/config_file.py +501 -0
  80. hpcflow/sdk/config/errors.py +272 -0
  81. hpcflow/sdk/config/types.py +150 -0
  82. hpcflow/sdk/core/__init__.py +38 -0
  83. hpcflow/sdk/core/actions.py +3857 -0
  84. hpcflow/sdk/core/app_aware.py +25 -0
  85. hpcflow/sdk/core/cache.py +224 -0
  86. hpcflow/sdk/core/command_files.py +814 -0
  87. hpcflow/sdk/core/commands.py +424 -0
  88. hpcflow/sdk/core/element.py +2071 -0
  89. hpcflow/sdk/core/enums.py +221 -0
  90. hpcflow/sdk/core/environment.py +256 -0
  91. hpcflow/sdk/core/errors.py +1043 -0
  92. hpcflow/sdk/core/execute.py +207 -0
  93. hpcflow/sdk/core/json_like.py +809 -0
  94. hpcflow/sdk/core/loop.py +1320 -0
  95. hpcflow/sdk/core/loop_cache.py +282 -0
  96. hpcflow/sdk/core/object_list.py +933 -0
  97. hpcflow/sdk/core/parameters.py +3371 -0
  98. hpcflow/sdk/core/rule.py +196 -0
  99. hpcflow/sdk/core/run_dir_files.py +57 -0
  100. hpcflow/sdk/core/skip_reason.py +7 -0
  101. hpcflow/sdk/core/task.py +3792 -0
  102. hpcflow/sdk/core/task_schema.py +993 -0
  103. hpcflow/sdk/core/test_utils.py +538 -0
  104. hpcflow/sdk/core/types.py +447 -0
  105. hpcflow/sdk/core/utils.py +1207 -0
  106. hpcflow/sdk/core/validation.py +87 -0
  107. hpcflow/sdk/core/values.py +477 -0
  108. hpcflow/sdk/core/workflow.py +4820 -0
  109. hpcflow/sdk/core/zarr_io.py +206 -0
  110. hpcflow/sdk/data/__init__.py +13 -0
  111. hpcflow/sdk/data/config_file_schema.yaml +34 -0
  112. hpcflow/sdk/data/config_schema.yaml +260 -0
  113. hpcflow/sdk/data/environments_spec_schema.yaml +21 -0
  114. hpcflow/sdk/data/files_spec_schema.yaml +5 -0
  115. hpcflow/sdk/data/parameters_spec_schema.yaml +7 -0
  116. hpcflow/sdk/data/task_schema_spec_schema.yaml +3 -0
  117. hpcflow/sdk/data/workflow_spec_schema.yaml +22 -0
  118. hpcflow/sdk/demo/__init__.py +3 -0
  119. hpcflow/sdk/demo/cli.py +242 -0
  120. hpcflow/sdk/helper/__init__.py +3 -0
  121. hpcflow/sdk/helper/cli.py +137 -0
  122. hpcflow/sdk/helper/helper.py +300 -0
  123. hpcflow/sdk/helper/watcher.py +192 -0
  124. hpcflow/sdk/log.py +288 -0
  125. hpcflow/sdk/persistence/__init__.py +18 -0
  126. hpcflow/sdk/persistence/base.py +2817 -0
  127. hpcflow/sdk/persistence/defaults.py +6 -0
  128. hpcflow/sdk/persistence/discovery.py +39 -0
  129. hpcflow/sdk/persistence/json.py +954 -0
  130. hpcflow/sdk/persistence/pending.py +948 -0
  131. hpcflow/sdk/persistence/store_resource.py +203 -0
  132. hpcflow/sdk/persistence/types.py +309 -0
  133. hpcflow/sdk/persistence/utils.py +73 -0
  134. hpcflow/sdk/persistence/zarr.py +2388 -0
  135. hpcflow/sdk/runtime.py +320 -0
  136. hpcflow/sdk/submission/__init__.py +3 -0
  137. hpcflow/sdk/submission/enums.py +70 -0
  138. hpcflow/sdk/submission/jobscript.py +2379 -0
  139. hpcflow/sdk/submission/schedulers/__init__.py +281 -0
  140. hpcflow/sdk/submission/schedulers/direct.py +233 -0
  141. hpcflow/sdk/submission/schedulers/sge.py +376 -0
  142. hpcflow/sdk/submission/schedulers/slurm.py +598 -0
  143. hpcflow/sdk/submission/schedulers/utils.py +25 -0
  144. hpcflow/sdk/submission/shells/__init__.py +52 -0
  145. hpcflow/sdk/submission/shells/base.py +229 -0
  146. hpcflow/sdk/submission/shells/bash.py +504 -0
  147. hpcflow/sdk/submission/shells/os_version.py +115 -0
  148. hpcflow/sdk/submission/shells/powershell.py +352 -0
  149. hpcflow/sdk/submission/submission.py +1402 -0
  150. hpcflow/sdk/submission/types.py +140 -0
  151. hpcflow/sdk/typing.py +194 -0
  152. hpcflow/sdk/utils/arrays.py +69 -0
  153. hpcflow/sdk/utils/deferred_file.py +55 -0
  154. hpcflow/sdk/utils/hashing.py +16 -0
  155. hpcflow/sdk/utils/patches.py +31 -0
  156. hpcflow/sdk/utils/strings.py +69 -0
  157. hpcflow/tests/api/test_api.py +32 -0
  158. hpcflow/tests/conftest.py +123 -0
  159. hpcflow/tests/data/__init__.py +0 -0
  160. hpcflow/tests/data/benchmark_N_elements.yaml +6 -0
  161. hpcflow/tests/data/benchmark_script_runner.yaml +26 -0
  162. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  163. hpcflow/tests/data/workflow_1.json +10 -0
  164. hpcflow/tests/data/workflow_1.yaml +5 -0
  165. hpcflow/tests/data/workflow_1_slurm.yaml +8 -0
  166. hpcflow/tests/data/workflow_1_wsl.yaml +8 -0
  167. hpcflow/tests/data/workflow_test_run_abort.yaml +42 -0
  168. hpcflow/tests/jinja_templates/test_jinja_templates.py +161 -0
  169. hpcflow/tests/programs/test_programs.py +180 -0
  170. hpcflow/tests/schedulers/direct_linux/test_direct_linux_submission.py +12 -0
  171. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  172. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +14 -0
  173. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  174. hpcflow/tests/scripts/test_main_scripts.py +1361 -0
  175. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  176. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  177. hpcflow/tests/shells/wsl/test_wsl_submission.py +14 -0
  178. hpcflow/tests/unit/test_action.py +1066 -0
  179. hpcflow/tests/unit/test_action_rule.py +24 -0
  180. hpcflow/tests/unit/test_app.py +132 -0
  181. hpcflow/tests/unit/test_cache.py +46 -0
  182. hpcflow/tests/unit/test_cli.py +172 -0
  183. hpcflow/tests/unit/test_command.py +377 -0
  184. hpcflow/tests/unit/test_config.py +195 -0
  185. hpcflow/tests/unit/test_config_file.py +162 -0
  186. hpcflow/tests/unit/test_element.py +666 -0
  187. hpcflow/tests/unit/test_element_iteration.py +88 -0
  188. hpcflow/tests/unit/test_element_set.py +158 -0
  189. hpcflow/tests/unit/test_group.py +115 -0
  190. hpcflow/tests/unit/test_input_source.py +1479 -0
  191. hpcflow/tests/unit/test_input_value.py +398 -0
  192. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  193. hpcflow/tests/unit/test_json_like.py +1247 -0
  194. hpcflow/tests/unit/test_loop.py +2674 -0
  195. hpcflow/tests/unit/test_meta_task.py +325 -0
  196. hpcflow/tests/unit/test_multi_path_sequences.py +259 -0
  197. hpcflow/tests/unit/test_object_list.py +116 -0
  198. hpcflow/tests/unit/test_parameter.py +243 -0
  199. hpcflow/tests/unit/test_persistence.py +664 -0
  200. hpcflow/tests/unit/test_resources.py +243 -0
  201. hpcflow/tests/unit/test_run.py +286 -0
  202. hpcflow/tests/unit/test_run_directories.py +29 -0
  203. hpcflow/tests/unit/test_runtime.py +9 -0
  204. hpcflow/tests/unit/test_schema_input.py +372 -0
  205. hpcflow/tests/unit/test_shell.py +129 -0
  206. hpcflow/tests/unit/test_slurm.py +39 -0
  207. hpcflow/tests/unit/test_submission.py +502 -0
  208. hpcflow/tests/unit/test_task.py +2560 -0
  209. hpcflow/tests/unit/test_task_schema.py +182 -0
  210. hpcflow/tests/unit/test_utils.py +616 -0
  211. hpcflow/tests/unit/test_value_sequence.py +549 -0
  212. hpcflow/tests/unit/test_values.py +91 -0
  213. hpcflow/tests/unit/test_workflow.py +827 -0
  214. hpcflow/tests/unit/test_workflow_template.py +186 -0
  215. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  216. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  217. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  218. hpcflow/tests/unit/utils/test_patches.py +5 -0
  219. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  220. hpcflow/tests/unit/utils/test_strings.py +97 -0
  221. hpcflow/tests/workflows/__init__.py +0 -0
  222. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  223. hpcflow/tests/workflows/test_jobscript.py +355 -0
  224. hpcflow/tests/workflows/test_run_status.py +198 -0
  225. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  226. hpcflow/tests/workflows/test_submission.py +140 -0
  227. hpcflow/tests/workflows/test_workflows.py +564 -0
  228. hpcflow/tests/workflows/test_zip.py +18 -0
  229. hpcflow/viz_demo.ipynb +6794 -0
  230. hpcflow-0.2.0a271.dist-info/LICENSE +375 -0
  231. hpcflow-0.2.0a271.dist-info/METADATA +65 -0
  232. hpcflow-0.2.0a271.dist-info/RECORD +237 -0
  233. {hpcflow-0.1.15.dist-info → hpcflow-0.2.0a271.dist-info}/WHEEL +4 -5
  234. hpcflow-0.2.0a271.dist-info/entry_points.txt +6 -0
  235. hpcflow/api.py +0 -490
  236. hpcflow/archive/archive.py +0 -307
  237. hpcflow/archive/cloud/cloud.py +0 -45
  238. hpcflow/archive/cloud/errors.py +0 -9
  239. hpcflow/archive/cloud/providers/dropbox.py +0 -427
  240. hpcflow/archive/errors.py +0 -5
  241. hpcflow/base_db.py +0 -4
  242. hpcflow/config.py +0 -233
  243. hpcflow/copytree.py +0 -66
  244. hpcflow/data/examples/_config.yml +0 -14
  245. hpcflow/data/examples/damask/demo/1.run.yml +0 -4
  246. hpcflow/data/examples/damask/demo/2.process.yml +0 -29
  247. hpcflow/data/examples/damask/demo/geom.geom +0 -2052
  248. hpcflow/data/examples/damask/demo/load.load +0 -1
  249. hpcflow/data/examples/damask/demo/material.config +0 -185
  250. hpcflow/data/examples/damask/inputs/geom.geom +0 -2052
  251. hpcflow/data/examples/damask/inputs/load.load +0 -1
  252. hpcflow/data/examples/damask/inputs/material.config +0 -185
  253. hpcflow/data/examples/damask/profiles/_variable_lookup.yml +0 -21
  254. hpcflow/data/examples/damask/profiles/damask.yml +0 -4
  255. hpcflow/data/examples/damask/profiles/damask_process.yml +0 -8
  256. hpcflow/data/examples/damask/profiles/damask_run.yml +0 -5
  257. hpcflow/data/examples/damask/profiles/default.yml +0 -6
  258. hpcflow/data/examples/thinking.yml +0 -177
  259. hpcflow/errors.py +0 -2
  260. hpcflow/init_db.py +0 -37
  261. hpcflow/models.py +0 -2595
  262. hpcflow/nesting.py +0 -9
  263. hpcflow/profiles.py +0 -455
  264. hpcflow/project.py +0 -81
  265. hpcflow/scheduler.py +0 -322
  266. hpcflow/utils.py +0 -103
  267. hpcflow/validation.py +0 -166
  268. hpcflow/variables.py +0 -543
  269. hpcflow-0.1.15.dist-info/METADATA +0 -168
  270. hpcflow-0.1.15.dist-info/RECORD +0 -45
  271. hpcflow-0.1.15.dist-info/entry_points.txt +0 -8
  272. hpcflow-0.1.15.dist-info/top_level.txt +0 -1
  273. /hpcflow/{archive → data/jinja_templates}/__init__.py +0 -0
  274. /hpcflow/{archive/cloud → data/programs}/__init__.py +0 -0
  275. /hpcflow/{archive/cloud/providers → data/workflows}/__init__.py +0 -0
@@ -0,0 +1,325 @@
1
+ from textwrap import dedent
2
+
3
+ import pytest
4
+
5
+ from hpcflow.app import app as hf
6
+ from hpcflow.sdk.config.errors import UnknownMetaTaskConstitutiveSchema
7
+
8
+
9
+ def test_basic_meta_task_workflow(new_null_config, tmp_path):
10
+ wk_yaml = dedent(
11
+ """\
12
+ name: test_meta_task
13
+ template_components:
14
+ task_schemas:
15
+ - objective: s0
16
+ inputs:
17
+ - parameter: p1
18
+ outputs:
19
+ - parameter: p2
20
+ actions:
21
+ - commands:
22
+ - command: echo "$((<<parameter:p1>> + 1))"
23
+ stdout: <<int(parameter:p2)>>
24
+
25
+ - objective: s1
26
+ inputs:
27
+ - parameter: p2
28
+ - parameter: p2b
29
+ outputs:
30
+ - parameter: p3
31
+ actions:
32
+ - commands:
33
+ - command: echo "$((<<parameter:p2>> + <<parameter:p2b>>))"
34
+ stdout: <<int(parameter:p3)>>
35
+
36
+ - objective: s2
37
+ inputs:
38
+ - parameter: p3
39
+ outputs:
40
+ - parameter: p4
41
+ actions:
42
+ - commands:
43
+ - command: echo "$((<<parameter:p3>> + 1))"
44
+ stdout: <<int(parameter:p4)>>
45
+
46
+ - objective: s3
47
+ inputs:
48
+ - parameter: p4
49
+ outputs:
50
+ - parameter: p5
51
+ actions:
52
+ - commands:
53
+ - command: echo "$((<<parameter:p4>> + 1))"
54
+ stdout: <<int(parameter:p5)>>
55
+
56
+ meta_task_schemas:
57
+ - objective: system_analysis
58
+ inputs:
59
+ - parameter: p2
60
+ outputs:
61
+ - parameter: p4
62
+
63
+ meta_tasks:
64
+ system_analysis:
65
+ - schema: s1
66
+ inputs:
67
+ p2b: 220
68
+ - schema: s2
69
+
70
+ tasks:
71
+ - schema: s0
72
+ inputs:
73
+ p1: 100
74
+ - schema: system_analysis
75
+ - schema: s3
76
+ """
77
+ )
78
+ wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
79
+
80
+ # basic check of param dependendices
81
+ s0_di = wk.tasks.s0.elements[0].get_data_idx()
82
+ s1_di = wk.tasks.s1.elements[0].get_data_idx()
83
+ s2_di = wk.tasks.s2.elements[0].get_data_idx()
84
+ s3_di = wk.tasks.s3.elements[0].get_data_idx()
85
+
86
+ assert s0_di["outputs.p2"] == s1_di["inputs.p2"]
87
+ assert s1_di["outputs.p3"] == s2_di["inputs.p3"]
88
+ assert s2_di["outputs.p4"] == s3_di["inputs.p4"]
89
+
90
+
91
+ def test_basic_meta_task_workflow_API(new_null_config, tmp_path):
92
+ """as above but using Python API."""
93
+ # normal task schemas:
94
+ s0 = hf.TaskSchema(
95
+ objective="s0",
96
+ inputs=[hf.SchemaInput("p1")],
97
+ outputs=[hf.SchemaOutput("p2")],
98
+ actions=[
99
+ hf.Action(
100
+ commands=[
101
+ hf.Command(
102
+ command='echo "$((<<parameter:p1>> + 1))"',
103
+ stdout="<<int(parameter:p2)>>",
104
+ )
105
+ ]
106
+ )
107
+ ],
108
+ )
109
+ s1 = hf.TaskSchema(
110
+ objective="s1",
111
+ inputs=[hf.SchemaInput("p2")],
112
+ outputs=[hf.SchemaOutput("p3")],
113
+ actions=[
114
+ hf.Action(
115
+ commands=[
116
+ hf.Command(
117
+ command='echo "$((<<parameter:p2>> + 1))"',
118
+ stdout="<<int(parameter:p3)>>",
119
+ )
120
+ ]
121
+ )
122
+ ],
123
+ )
124
+ s2 = hf.TaskSchema(
125
+ objective="s2",
126
+ inputs=[hf.SchemaInput("p3")],
127
+ outputs=[hf.SchemaOutput("p4")],
128
+ actions=[
129
+ hf.Action(
130
+ commands=[
131
+ hf.Command(
132
+ command='echo "$((<<parameter:p3>> + 1))"',
133
+ stdout="<<int(parameter:p4)>>",
134
+ )
135
+ ]
136
+ )
137
+ ],
138
+ )
139
+ s3 = hf.TaskSchema(
140
+ objective="s3",
141
+ inputs=[hf.SchemaInput("p4")],
142
+ outputs=[hf.SchemaOutput("p5")],
143
+ actions=[
144
+ hf.Action(
145
+ commands=[
146
+ hf.Command(
147
+ command='echo "$((<<parameter:p4>> + 1))"',
148
+ stdout="<<int(parameter:p5)>>",
149
+ )
150
+ ]
151
+ )
152
+ ],
153
+ )
154
+
155
+ # meta=task schema:
156
+ ms = hf.MetaTaskSchema(
157
+ objective="system_analysis",
158
+ inputs=[hf.SchemaInput("p2")],
159
+ outputs=[hf.SchemaOutput("p4")],
160
+ )
161
+
162
+ # meta-task:
163
+ m1 = hf.MetaTask(
164
+ schema=ms,
165
+ tasks=[
166
+ hf.Task(schema=s1),
167
+ hf.Task(schema=s2),
168
+ ],
169
+ )
170
+
171
+ # workflow template tasks list:
172
+ tasks = [
173
+ hf.Task(schema=s0, inputs={"p1": 100}),
174
+ m1,
175
+ hf.Task(schema=s3),
176
+ ]
177
+
178
+ wk = hf.Workflow.from_template_data(
179
+ template_name="meta_task_workflow",
180
+ tasks=tasks,
181
+ path=tmp_path,
182
+ )
183
+
184
+ # basic check of param dependendices
185
+ s0_di = wk.tasks.s0.elements[0].get_data_idx()
186
+ s1_di = wk.tasks.s1.elements[0].get_data_idx()
187
+ s2_di = wk.tasks.s2.elements[0].get_data_idx()
188
+ s3_di = wk.tasks.s3.elements[0].get_data_idx()
189
+
190
+ assert s0_di["outputs.p2"] == s1_di["inputs.p2"]
191
+ assert s1_di["outputs.p3"] == s2_di["inputs.p3"]
192
+ assert s2_di["outputs.p4"] == s3_di["inputs.p4"]
193
+
194
+
195
+ def test_meta_task_custom_parametrisation(new_null_config, tmp_path):
196
+ """test customising the parametrisation of inputs, sequences, and resources within the
197
+ `tasks` list."""
198
+ wk_yaml = dedent(
199
+ """\
200
+ name: test_metatask_multi_element_sets_custom_parametrisation
201
+ template_components:
202
+ task_schemas:
203
+ - objective: s1
204
+ inputs:
205
+ - parameter: p1
206
+ - parameter: p2
207
+ outputs:
208
+ - parameter: p3
209
+ actions:
210
+ - commands:
211
+ - command: echo "$((<<parameter:p1>> + <<parameter:p2>>))"
212
+ stdout: <<int(parameter:p3)>>
213
+
214
+ meta_task_schemas:
215
+ - objective: system_analysis
216
+ inputs:
217
+ - parameter: p1
218
+ - parameter: p2
219
+ outputs:
220
+ - parameter: p3
221
+
222
+ meta_tasks:
223
+ system_analysis:
224
+ - schema: s1
225
+ element_sets:
226
+ - inputs:
227
+ p1: 100
228
+ p2: 200
229
+ - inputs:
230
+ p1: 100
231
+ sequences:
232
+ - path: inputs.p2
233
+ values: [200, 201]
234
+ tasks:
235
+ - schema: system_analysis
236
+ inputs:
237
+ s1: # should apply to first element set by default
238
+ p1: 101
239
+ resources:
240
+ s1: # should apply to first element set by default
241
+ any:
242
+ num_cores: 2
243
+ - schema: system_analysis
244
+ inputs:
245
+ s1.0: # applies to first element set of s1
246
+ p1: 102
247
+ s1.1: # applies to second element set of s1
248
+ p1: 103
249
+ sequences:
250
+ s1.1: # sequences list in second element set is replaced with this list:
251
+ - path: inputs.p2
252
+ values: [300, 301]
253
+ """
254
+ )
255
+ wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
256
+
257
+ assert wk.tasks.s1_1.template.element_sets[0].resources[0].num_cores == 2 # modified
258
+ assert (
259
+ wk.tasks.s1_2.template.element_sets[0].resources[0].num_cores is None
260
+ ) # unaffected
261
+
262
+ assert wk.tasks.s1_1.template.element_sets[0].inputs[0].value == 101 # modified
263
+ assert wk.tasks.s1_1.template.element_sets[0].inputs[1].value == 200 # unaffected
264
+ assert wk.tasks.s1_1.template.element_sets[1].sequences[0].values == [
265
+ 200,
266
+ 201,
267
+ ] # unaffected
268
+
269
+ assert wk.tasks.s1_2.template.element_sets[0].inputs[0].value == 102 # modified
270
+ assert wk.tasks.s1_2.template.element_sets[1].inputs[0].value == 103 # modified
271
+ assert wk.tasks.s1_2.template.element_sets[1].sequences[0].values == [
272
+ 300,
273
+ 301,
274
+ ] # modified
275
+
276
+
277
+ def test_meta_task_custom_parametrisation_raises_on_bad_schema_name(
278
+ new_null_config, tmp_path
279
+ ):
280
+ wk_yaml = dedent(
281
+ """\
282
+ name: test_metatask_raise_on_bad_schema_name
283
+ template_components:
284
+ task_schemas:
285
+ - objective: s1
286
+ inputs:
287
+ - parameter: p1
288
+ - parameter: p2
289
+ outputs:
290
+ - parameter: p3
291
+ actions:
292
+ - commands:
293
+ - command: echo "$((<<parameter:p1>> + <<parameter:p2>>))"
294
+ stdout: <<int(parameter:p3)>>
295
+
296
+ meta_task_schemas:
297
+ - objective: system_analysis
298
+ inputs:
299
+ - parameter: p1
300
+ - parameter: p2
301
+ outputs:
302
+ - parameter: p3
303
+
304
+ meta_tasks:
305
+ system_analysis:
306
+ - schema: s1
307
+ element_sets:
308
+ - inputs:
309
+ p1: 100
310
+ p2: 200
311
+ - inputs:
312
+ p1: 100
313
+ sequences:
314
+ - path: inputs.p2
315
+ values: [200, 201]
316
+ tasks:
317
+ - schema: system_analysis
318
+ resources:
319
+ BAD_SCHEMA_NAME: # should raise!
320
+ any:
321
+ num_cores: 2
322
+ """
323
+ )
324
+ with pytest.raises(UnknownMetaTaskConstitutiveSchema):
325
+ wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
@@ -0,0 +1,259 @@
1
+ from __future__ import annotations
2
+ from textwrap import dedent
3
+
4
+ import numpy as np
5
+ import pytest
6
+
7
+ from hpcflow.app import app as hf
8
+ from hpcflow.sdk.core.test_utils import make_schemas
9
+ from hpcflow.sdk.core.utils import get_file_context
10
+
11
+
12
+ def test_MPS_sequences():
13
+ mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
14
+ assert mps.sequences == [
15
+ hf.ValueSequence(path="inputs.p1", values=[0, 1]),
16
+ hf.ValueSequence(path="inputs.p2", values=[2, 3]),
17
+ ]
18
+
19
+
20
+ def test_MPS_sequences_moved_to_element_set():
21
+ mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
22
+ es = hf.ElementSet(multi_path_sequences=[mps])
23
+ expected_mps_seqs = [
24
+ hf.ValueSequence(path="inputs.p1", values=[0, 1]),
25
+ hf.ValueSequence(path="inputs.p2", values=[2, 3]),
26
+ ]
27
+ assert mps._sequences is None
28
+ assert mps.sequence_indices == [0, 2]
29
+ assert es.sequences == mps.sequences
30
+ assert es.sequences == expected_mps_seqs
31
+
32
+
33
+ def test_MPS_sequences_moved_to_element_set_with_existing_sequences():
34
+ mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
35
+ seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
36
+ expected_mps_seqs = [
37
+ hf.ValueSequence(path="inputs.p1", values=[0, 1]),
38
+ hf.ValueSequence(path="inputs.p2", values=[2, 3]),
39
+ ]
40
+ es = hf.ElementSet(
41
+ sequences=[seq],
42
+ multi_path_sequences=[mps],
43
+ )
44
+ assert mps._sequences is None
45
+ assert mps.sequence_indices == [1, 3]
46
+ assert mps.sequences == expected_mps_seqs
47
+ assert es.sequences == [seq, *expected_mps_seqs]
48
+
49
+
50
+ def test_MPS_sequences_moved_to_task_element_set():
51
+ mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
52
+ s1 = make_schemas(({"p1": None, "p2": None}, ()))
53
+ t1 = hf.Task(s1, multi_path_sequences=[mps])
54
+ expected_mps_seqs = [
55
+ hf.ValueSequence(path="inputs.p1", values=[0, 1]),
56
+ hf.ValueSequence(path="inputs.p2", values=[2, 3]),
57
+ ]
58
+ es = t1.element_sets[0]
59
+ assert mps._sequences is None
60
+ assert mps.sequence_indices == [0, 2]
61
+ assert es.sequences == mps.sequences
62
+ assert es.sequences == expected_mps_seqs
63
+
64
+
65
+ def test_MPS_sequences_moved_to_task_element_set_with_existing_sequences():
66
+ mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
67
+ seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
68
+ s1 = make_schemas(({"p0": None, "p1": None, "p2": None}, ()))
69
+ t1 = hf.Task(s1, sequences=[seq], multi_path_sequences=[mps])
70
+ expected_mps_seqs = [
71
+ hf.ValueSequence(path="inputs.p1", values=[0, 1]),
72
+ hf.ValueSequence(path="inputs.p2", values=[2, 3]),
73
+ ]
74
+ es = t1.element_sets[0]
75
+ assert mps._sequences is None
76
+ assert mps.sequence_indices == [1, 3]
77
+ assert mps.sequences == expected_mps_seqs
78
+ assert es.sequences == [seq, *expected_mps_seqs]
79
+
80
+
81
+ def test_MPS_sequence_element_inputs(null_config, tmp_path):
82
+ mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
83
+ s1 = make_schemas(({"p1": None, "p2": None}, ()))
84
+ t1 = hf.Task(s1, multi_path_sequences=[mps])
85
+ wf = hf.Workflow.from_template_data(
86
+ tasks=[t1],
87
+ template_name="test_multi_path_sequence",
88
+ path=tmp_path,
89
+ )
90
+ assert len(wf.template.tasks[0].element_sets[0].sequences) == 2
91
+ assert wf.tasks[0].num_elements == 2
92
+ assert wf.tasks[0].elements[0].get("inputs") == {"p1": 0, "p2": 2}
93
+ assert wf.tasks[0].elements[1].get("inputs") == {"p1": 1, "p2": 3}
94
+
95
+
96
+ def test_MPS_sequence_element_inputs_with_existing_sequence(null_config, tmp_path):
97
+ mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
98
+ seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
99
+ s1 = make_schemas(({"p0": None, "p1": None, "p2": None}, ()))
100
+ t1 = hf.Task(s1, sequences=[seq], multi_path_sequences=[mps])
101
+ wf = hf.Workflow.from_template_data(
102
+ tasks=[t1],
103
+ template_name="test_multi_path_sequence",
104
+ path=tmp_path,
105
+ )
106
+ assert len(wf.template.tasks[0].element_sets[0].sequences) == 3
107
+ assert wf.tasks[0].num_elements == 2
108
+ assert wf.tasks[0].elements[0].get("inputs") == {"p0": 0, "p1": 0, "p2": 2}
109
+ assert wf.tasks[0].elements[1].get("inputs") == {"p0": 1, "p1": 1, "p2": 3}
110
+
111
+ # check the same on reload:
112
+ wf = wf.reload()
113
+ assert len(wf.template.tasks[0].element_sets[0].sequences) == 3
114
+ assert wf.tasks[0].num_elements == 2
115
+ assert wf.tasks[0].elements[0].get("inputs") == {"p0": 0, "p1": 0, "p2": 2}
116
+ assert wf.tasks[0].elements[1].get("inputs") == {"p0": 1, "p1": 1, "p2": 3}
117
+
118
+
119
+ @pytest.mark.integration
120
+ def test_MPS_element_outputs(null_config, tmp_path):
121
+ with get_file_context("hpcflow.tests.data", "multi_path_sequences.yaml") as file_path:
122
+ wf = hf.make_and_submit_workflow(
123
+ file_path,
124
+ path=tmp_path,
125
+ status=False,
126
+ add_to_known=False,
127
+ wait=True,
128
+ )
129
+ assert wf.tasks[0].num_elements == 2
130
+
131
+ p2 = wf.tasks[0].elements[0].outputs.p2
132
+ assert isinstance(p2, hf.ElementParameter)
133
+ assert p2.value == 302
134
+
135
+ p2 = wf.tasks[0].elements[1].outputs.p2
136
+ assert isinstance(p2, hf.ElementParameter)
137
+ assert p2.value == 304
138
+
139
+
140
+ def test_MPS_latin_hypercube_sequence_values():
141
+ wft_yaml = dedent(
142
+ """\
143
+ name: test_latin_hypercube_sampling
144
+ template_components:
145
+ task_schemas:
146
+ - objective: define_p1
147
+ inputs:
148
+ - parameter: p1
149
+ tasks:
150
+ - schema: define_p1
151
+ inputs:
152
+ p1: {}
153
+ multi_path_sequences:
154
+ - paths: [inputs.p1.a, inputs.p1.b]
155
+ values::from_latin_hypercube:
156
+ num_samples: 5
157
+ """
158
+ )
159
+ wft = hf.WorkflowTemplate.from_YAML_string(wft_yaml)
160
+ es = wft.tasks[0].element_sets[0]
161
+ assert len(es.multi_path_sequences) == 1
162
+ mps_values = np.asarray(es.multi_path_sequences[0].values)
163
+ assert mps_values.shape == (2, 5)
164
+ assert len(es.sequences) == 2
165
+ seq_1 = es.sequences[0]
166
+ seq_2 = es.sequences[1]
167
+ assert seq_1.path == "inputs.p1.a"
168
+ assert seq_2.path == "inputs.p1.b"
169
+ assert np.array_equal(np.asarray(seq_1.values), mps_values[0])
170
+ assert np.array_equal(np.asarray(seq_2.values), mps_values[1])
171
+
172
+
173
+ def test_MPS_latin_hypercube_sequence_bounds():
174
+
175
+ bounds = {
176
+ "inputs.a": {"extent": [16789.2, 17812.5], "scaling": "linear"},
177
+ "inputs.c": {"extent": [1.0e-10, 1.0e-5], "scaling": "log"},
178
+ }
179
+
180
+ mps = hf.MultiPathSequence.from_latin_hypercube(
181
+ paths=["inputs.a", "inputs.b", "inputs.c"],
182
+ num_samples=10,
183
+ bounds=bounds,
184
+ )
185
+
186
+ vals_arr = np.array(mps.values)
187
+
188
+ assert vals_arr.shape == (3, 10)
189
+
190
+ vals_a = vals_arr[0]
191
+ vals_b = vals_arr[1]
192
+ vals_c = vals_arr[2]
193
+
194
+ extent_a = bounds["inputs.a"]["extent"]
195
+ extent_b = [0.0, 1.0]
196
+ extent_c = bounds["inputs.c"]["extent"]
197
+
198
+ assert np.logical_and(vals_a > extent_a[0], vals_a < extent_a[1]).all()
199
+ assert np.logical_and(vals_b > extent_b[0], vals_b < extent_b[1]).all()
200
+ assert np.logical_and(vals_c > extent_c[0], vals_c < extent_c[1]).all()
201
+
202
+
203
+ def test_MPS_move_from_sequences_list():
204
+ wft_yaml = dedent(
205
+ """\
206
+ name: test_latin_hypercube_sampling
207
+ template_components:
208
+ task_schemas:
209
+ - objective: define_p1_p2_p3_p4
210
+ inputs:
211
+ - parameter: p1
212
+ - parameter: p2
213
+ - parameter: p3
214
+ - parameter: p4
215
+ tasks:
216
+ - schema: define_p1_p2_p3_p4
217
+ inputs:
218
+ p1: {}
219
+ p2: {}
220
+ p3: {}
221
+
222
+ multi_path_sequences:
223
+ - paths: [inputs.p1.a, inputs.p1.b]
224
+ values::from_latin_hypercube:
225
+ num_samples: 4
226
+
227
+ sequences:
228
+ - paths: [inputs.p2.a, inputs.p2.b] # actually a multi-path sequence
229
+ values::from_latin_hypercube:
230
+ num_samples: 4
231
+
232
+ - path: inputs.p4 # a normal sequence
233
+ values: [0, 1, 2, 3]
234
+
235
+ - paths: [inputs.p3.a, inputs.p3.b] # actually a multi-path sequence
236
+ values::from_latin_hypercube:
237
+ num_samples: 4
238
+ """
239
+ )
240
+ wft = hf.WorkflowTemplate.from_YAML_string(wft_yaml)
241
+ es = wft.tasks[0].element_sets[0]
242
+ mps_lst = es.multi_path_sequences
243
+ seq_lst = es.sequences
244
+ assert len(mps_lst) == 3
245
+ assert len(seq_lst) == 7 # one original plus three multi-path with two paths each
246
+
247
+ # check ordering of multi-path sequences is preserved:
248
+ assert mps_lst[0].paths == ["inputs.p1.a", "inputs.p1.b"]
249
+ assert mps_lst[1].paths == ["inputs.p2.a", "inputs.p2.b"]
250
+ assert mps_lst[2].paths == ["inputs.p3.a", "inputs.p3.b"]
251
+
252
+ # check sensible ordering of sequences:
253
+ assert seq_lst[0].path == "inputs.p4"
254
+ assert seq_lst[1].path == "inputs.p1.a"
255
+ assert seq_lst[2].path == "inputs.p1.b"
256
+ assert seq_lst[3].path == "inputs.p2.a"
257
+ assert seq_lst[4].path == "inputs.p2.b"
258
+ assert seq_lst[5].path == "inputs.p3.a"
259
+ assert seq_lst[6].path == "inputs.p3.b"
@@ -0,0 +1,116 @@
1
+ from __future__ import annotations
2
+ from dataclasses import dataclass
3
+ from typing_extensions import TypedDict
4
+ import pytest
5
+
6
+ from hpcflow.app import app as hf
7
+ from hpcflow.sdk.core.object_list import ObjectList, DotAccessObjectList
8
+
9
+
10
+ @pytest.fixture
11
+ def null_config(tmp_path):
12
+ if not hf.is_config_loaded:
13
+ hf.load_config(config_dir=tmp_path)
14
+
15
+
16
+ @dataclass
17
+ class MyObj:
18
+ name: str
19
+ data: int
20
+
21
+
22
+ class SimpleObjectList(TypedDict):
23
+ objects: list[MyObj]
24
+ object_list: DotAccessObjectList
25
+
26
+
27
+ @pytest.fixture
28
+ def simple_object_list(null_config) -> SimpleObjectList:
29
+ my_objs = [MyObj(name="A", data=1), MyObj(name="B", data=2)]
30
+ obj_list = DotAccessObjectList(my_objs, access_attribute="name")
31
+ return {"objects": my_objs, "object_list": obj_list}
32
+
33
+
34
+ def test_get_item(simple_object_list: SimpleObjectList):
35
+ objects = simple_object_list["objects"]
36
+ obj_list = simple_object_list["object_list"]
37
+
38
+ assert obj_list[0] == objects[0] and obj_list[1] == objects[1]
39
+
40
+
41
+ def test_get_dot_notation(simple_object_list: SimpleObjectList):
42
+ objects = simple_object_list["objects"]
43
+ obj_list = simple_object_list["object_list"]
44
+
45
+ assert obj_list.A == objects[0] and obj_list.B == objects[1]
46
+
47
+
48
+ def test_add_obj_to_end(simple_object_list: SimpleObjectList):
49
+ obj_list = simple_object_list["object_list"]
50
+ new_obj = MyObj("C", 3)
51
+ obj_list.add_object(new_obj)
52
+ assert obj_list[-1] == new_obj
53
+
54
+
55
+ def test_add_obj_to_start(simple_object_list: SimpleObjectList):
56
+ obj_list = simple_object_list["object_list"]
57
+ new_obj = MyObj("C", 3)
58
+ obj_list.add_object(new_obj, 0)
59
+ assert obj_list[0] == new_obj
60
+
61
+
62
+ def test_add_obj_to_middle(simple_object_list: SimpleObjectList):
63
+ obj_list = simple_object_list["object_list"]
64
+ new_obj = MyObj("C", 3)
65
+ obj_list.add_object(new_obj, 1)
66
+ assert obj_list[1] == new_obj
67
+
68
+
69
+ def test_get_obj_attr_custom_callable(null_config):
70
+ def my_get_obj_attr(self, obj, attr):
71
+ if attr == "a":
72
+ return getattr(obj, attr)
73
+ else:
74
+ return getattr(obj, "b")[attr]
75
+
76
+ MyObjectList = type("MyObjectList", (ObjectList,), {})
77
+ MyObjectList._get_obj_attr = my_get_obj_attr
78
+
79
+ o1 = MyObjectList(
80
+ [
81
+ {"a": 1, "b": {"c1": 2}},
82
+ {"a": 2, "b": {"c1": 3}},
83
+ ]
84
+ )
85
+ assert o1.get(c1=2) == o1[0]
86
+
87
+
88
+ def test_get_with_missing_key(null_config) -> None:
89
+ o1 = ObjectList([{"a": 1}, {"b": 2}])
90
+ assert o1.get(a=1) == {"a": 1}
91
+
92
+
93
+ def test_parameters_list_get_equivalence(null_config) -> None:
94
+ p_name = "p12334567898765432101"
95
+ hf.parameters.add_object(hf.Parameter(p_name))
96
+ assert p_name in hf.parameters.list_attrs()
97
+ assert (
98
+ getattr(hf.parameters, p_name)
99
+ == hf.parameters.get(p_name)
100
+ == hf.parameters.get_all(p_name)[0]
101
+ == hf.parameters.get(typ=p_name)
102
+ )
103
+
104
+
105
+ def test_parameters_list_get_equivalence_non_existent(new_null_config) -> None:
106
+ # non-existent parameters should be created, unlike other ObjectList sub-classes,
107
+ # which raise
108
+ hf.reload_template_components()
109
+ p_name = "p12334567898765432101"
110
+ assert p_name not in hf.parameters.list_attrs()
111
+ assert (
112
+ getattr(hf.parameters, p_name)
113
+ == hf.parameters.get(p_name)
114
+ == hf.parameters.get_all(p_name)[0]
115
+ == hf.parameters.get(typ=p_name)
116
+ )